From 7f22458b2d4589c892aa2827050418421e7632db Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:22:15 -0400 Subject: [PATCH 01/49] Remove dead #if 0 b_element_op loop in gemm_multi_ABD example The disabled loop applying b_element_op was superseded by the active code path using ReferenceGemm with PassThrough ops. --- ...mm_multi_ABD_xdl_multiply_bias_fastgelu_bf16_i8.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/projects/composablekernel/example/60_gemm_multi_ABD/gemm_multi_ABD_xdl_multiply_bias_fastgelu_bf16_i8.cpp b/projects/composablekernel/example/60_gemm_multi_ABD/gemm_multi_ABD_xdl_multiply_bias_fastgelu_bf16_i8.cpp index b0b2d29d987b..2ceca3c877a0 100644 --- a/projects/composablekernel/example/60_gemm_multi_ABD/gemm_multi_ABD_xdl_multiply_bias_fastgelu_bf16_i8.cpp +++ b/projects/composablekernel/example/60_gemm_multi_ABD/gemm_multi_ABD_xdl_multiply_bias_fastgelu_bf16_i8.cpp @@ -238,16 +238,6 @@ int main(int argc, char* argv[]) Tensor b_k_n(f_host_tensor_descriptor(K, N, StrideB, B0Layout{})); -#if 0 - for(int n = 0; n < N; ++n) - { - for(int k = 0; k < K; ++k) - { - b_element_op(b_k_n(k, n), b0_k_n(k, n), b1_k_n(k, n)); - } - } -#endif - using ReferenceGemmInstance = ck::tensor_operation::host::ReferenceGemm Date: Thu, 9 Apr 2026 00:26:44 -0400 Subject: [PATCH 02/49] Remove dead #if 0 blocks in example/ck_tile/11_add_rmsnorm2d_rdquant/instances Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../add_rmsnorm2d_rdquant_fwd_bf16_n1024_instance.cpp | 8 -------- .../add_rmsnorm2d_rdquant_fwd_fp16_n1024_instance.cpp | 8 -------- 2 files changed, 16 deletions(-) diff --git a/projects/composablekernel/example/ck_tile/11_add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_bf16_n1024_instance.cpp b/projects/composablekernel/example/ck_tile/11_add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_bf16_n1024_instance.cpp index 8f4813a47eb7..ca491148446f 100644 --- a/projects/composablekernel/example/ck_tile/11_add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_bf16_n1024_instance.cpp +++ b/projects/composablekernel/example/ck_tile/11_add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_bf16_n1024_instance.cpp @@ -5,14 +5,6 @@ // clang-format off // rm rn tm tn vn pd x 3p -#if 0 -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); - -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -#endif template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); diff --git a/projects/composablekernel/example/ck_tile/11_add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_fp16_n1024_instance.cpp b/projects/composablekernel/example/ck_tile/11_add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_fp16_n1024_instance.cpp index e357d7e3acff..f754d8e959f9 100644 --- a/projects/composablekernel/example/ck_tile/11_add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_fp16_n1024_instance.cpp +++ b/projects/composablekernel/example/ck_tile/11_add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_fp16_n1024_instance.cpp @@ -5,14 +5,6 @@ // clang-format off // rm rn tm tn vn pd x 3p -#if 0 -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); - -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -#endif template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); From 65714890e10c8bc60b6ce97c09ea147cf435b81f Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:27:34 -0400 Subject: [PATCH 03/49] Remove dead #if 0 blocks in example/ck_tile/12_smoothquant/instances Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../instances/smoothquant_bf16_n1024_instance.cpp | 8 -------- .../instances/smoothquant_fp16_n1024_instance.cpp | 8 -------- 2 files changed, 16 deletions(-) diff --git a/projects/composablekernel/example/ck_tile/12_smoothquant/instances/smoothquant_bf16_n1024_instance.cpp b/projects/composablekernel/example/ck_tile/12_smoothquant/instances/smoothquant_bf16_n1024_instance.cpp index 8a5e0c74a0f9..66f427247a12 100644 --- a/projects/composablekernel/example/ck_tile/12_smoothquant/instances/smoothquant_bf16_n1024_instance.cpp +++ b/projects/composablekernel/example/ck_tile/12_smoothquant/instances/smoothquant_bf16_n1024_instance.cpp @@ -5,14 +5,6 @@ // clang-format off // rm rn tm tn vn pd 2p -#if 0 -template float smoothquant_>(const S&, A); -template float smoothquant_>(const S&, A); -template float smoothquant_>(const S&, A); -template float smoothquant_>(const S&, A); - -template float smoothquant_>(const S&, A); -#endif template float smoothquant_>(const S&, A); template float smoothquant_>(const S&, A); diff --git a/projects/composablekernel/example/ck_tile/12_smoothquant/instances/smoothquant_fp16_n1024_instance.cpp b/projects/composablekernel/example/ck_tile/12_smoothquant/instances/smoothquant_fp16_n1024_instance.cpp index 9c08cf64f091..103f7281b02a 100644 --- a/projects/composablekernel/example/ck_tile/12_smoothquant/instances/smoothquant_fp16_n1024_instance.cpp +++ b/projects/composablekernel/example/ck_tile/12_smoothquant/instances/smoothquant_fp16_n1024_instance.cpp @@ -5,14 +5,6 @@ // clang-format off // rm rn tm tn vn pd 2p -#if 0 -template float smoothquant_>(const S&, A); -template float smoothquant_>(const S&, A); -template float smoothquant_>(const S&, A); -template float smoothquant_>(const S&, A); - -template float smoothquant_>(const S&, A); -#endif template float smoothquant_>(const S&, A); template float smoothquant_>(const S&, A); From 4686201643a8a4716858fef8b83b73f474426145 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:28:16 -0400 Subject: [PATCH 04/49] Remove dead #if 0 blocks in example/ck_tile/14_moe_smoothquant/instances Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../instances/moe_smoothquant_bf16_n1024_instance.cpp | 8 -------- .../instances/moe_smoothquant_fp16_n1024_instance.cpp | 8 -------- 2 files changed, 16 deletions(-) diff --git a/projects/composablekernel/example/ck_tile/14_moe_smoothquant/instances/moe_smoothquant_bf16_n1024_instance.cpp b/projects/composablekernel/example/ck_tile/14_moe_smoothquant/instances/moe_smoothquant_bf16_n1024_instance.cpp index 8c72b81dc13b..56fcca3beb81 100644 --- a/projects/composablekernel/example/ck_tile/14_moe_smoothquant/instances/moe_smoothquant_bf16_n1024_instance.cpp +++ b/projects/composablekernel/example/ck_tile/14_moe_smoothquant/instances/moe_smoothquant_bf16_n1024_instance.cpp @@ -5,14 +5,6 @@ // clang-format off // rm rn tm tn vn pd 2p -#if 0 -template float moe_smoothquant_>(const S&, A); -template float moe_smoothquant_>(const S&, A); -template float moe_smoothquant_>(const S&, A); -template float moe_smoothquant_>(const S&, A); - -template float moe_smoothquant_>(const S&, A); -#endif template float moe_smoothquant_>(const S&, A); template float moe_smoothquant_>(const S&, A); diff --git a/projects/composablekernel/example/ck_tile/14_moe_smoothquant/instances/moe_smoothquant_fp16_n1024_instance.cpp b/projects/composablekernel/example/ck_tile/14_moe_smoothquant/instances/moe_smoothquant_fp16_n1024_instance.cpp index 6d7a5e7c1f59..2462cd218eb5 100644 --- a/projects/composablekernel/example/ck_tile/14_moe_smoothquant/instances/moe_smoothquant_fp16_n1024_instance.cpp +++ b/projects/composablekernel/example/ck_tile/14_moe_smoothquant/instances/moe_smoothquant_fp16_n1024_instance.cpp @@ -5,14 +5,6 @@ // clang-format off // rm rn tm tn vn pd 2p -#if 0 -template float moe_smoothquant_>(const S&, A); -template float moe_smoothquant_>(const S&, A); -template float moe_smoothquant_>(const S&, A); -template float moe_smoothquant_>(const S&, A); - -template float moe_smoothquant_>(const S&, A); -#endif template float moe_smoothquant_>(const S&, A); template float moe_smoothquant_>(const S&, A); From 0504842bb1df7b660554e970ac7955dc17a843c5 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:28:54 -0400 Subject: [PATCH 05/49] Remove dead #if 0 blocks in ck/tensor_operation/gpu/block Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- ...dlops_blockscale_b_preshuffle_selector.hpp | 26 ------------------- ...s_moe_blockscale_b_preshuffle_selector.hpp | 26 ------------------- 2 files changed, 52 deletions(-) diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_selector.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_selector.hpp index 8df23454a262..41ca5916cbed 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_selector.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_selector.hpp @@ -60,32 +60,6 @@ constexpr auto BlockGemmBlockScaleBPreshufflePipeline_Selector() NRepeat, KPack>{}; } -#if 0 - else if constexpr(BlkGemmPipelineVer == BlockGemmPipelineVersion::v2) - { - return BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v2< - BlkGemmPipeSche, - BlockSize, - ADataType, - BDataType, - ComputeDataType, - AccDataType, - ATileDesc, - BTileDesc, - AMmaTileDesc, - BMmaTileDesc, - ABlockTransferSrcScalarPerVector, - BBlockTransferSrcScalarPerVector, - MPerBlock, - NPerBlock, - KPerBlock, - MPerXDL, - NPerXDL, - MRepeat, - NRepeat, - KPack>{}; - } -#endif else if constexpr(BlkGemmPipelineVer == BlockGemmPipelineVersion::v3) { static_assert(MRepeat >= 4, "MRepeat should at least be 4 in BlockGemmPipelineVersion::v3"); diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_moe_blockscale_b_preshuffle_selector.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_moe_blockscale_b_preshuffle_selector.hpp index 199c729f535b..96bf5e81b740 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_moe_blockscale_b_preshuffle_selector.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_moe_blockscale_b_preshuffle_selector.hpp @@ -93,32 +93,6 @@ constexpr auto BlockGemmBlockMoeScaleBPreshufflePipeline_Selector() KPack>{}; } } -#if 0 - else if constexpr(BlkGemmPipelineVer == BlockGemmPipelineVersion::v2) - { - return BlockwiseGemmXdlops_pipeline_moe_blockscale_bpreshuffle_v2< - BlkGemmPipeSche, - BlockSize, - ADataType, - BDataType, - ComputeDataType, - AccDataType, - ATileDesc, - BTileDesc, - AMmaTileDesc, - BMmaTileDesc, - ABlockTransferSrcScalarPerVector, - BBlockTransferSrcScalarPerVector, - MPerBlock, - NPerBlock, - KPerBlock, - MPerXDL, - NPerXDL, - MRepeat, - NRepeat, - KPack>{}; - } -#endif else if constexpr(BlkGemmPipelineVer == BlockGemmPipelineVersion::v3) { static_assert(MRepeat >= 4, "MRepeat should at least be 4 in BlockGemmPipelineVersion::v3"); From 9aaf85dbb0a01ec1d7c35c64819540e8c746b5bd Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:28:56 -0400 Subject: [PATCH 06/49] Remove dead #if 0 blocks in ck/tensor_operation/gpu/device/impl Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- ...emm_softmax_gemm_permute_wmma_cshuffle.hpp | 141 ------------------ ...e_grouped_query_attention_forward_wmma.hpp | 141 ------------------ ...ice_multi_query_attention_forward_wmma.hpp | 141 ------------------ 3 files changed, 423 deletions(-) diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_batched_gemm_softmax_gemm_permute_wmma_cshuffle.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_batched_gemm_softmax_gemm_permute_wmma_cshuffle.hpp index 6b595c4dce91..043adf5fc0d3 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_batched_gemm_softmax_gemm_permute_wmma_cshuffle.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_batched_gemm_softmax_gemm_permute_wmma_cshuffle.hpp @@ -1433,147 +1433,6 @@ struct DeviceBatchedGemmSoftmaxGemmPermute_Wmma_CShuffle // TODO: properly implement this check return true; } -#if 0 - static bool IsSupportedArgument(const Argument& arg) - { - if(ck::is_gfx11_supported()) - { - if constexpr(!(is_same_v || is_same_v)) - { - printf("DeviceOp: Acc0 Type err"); - return false; - } - - if constexpr(!(is_same_v || is_same_v)) - { - printf("DeviceOp: Acc1 Type err"); - return false; - } - } - else - { - printf("DeviceOp: Arch err"); - return false; - } - - if(!GridwiseOp::CheckValidity(arg.a_grid_desc, - arg.b0_grid_desc, - arg.b1_grid_desc, - arg.c_grid_desc_m_n_, - arg.block_2_ctile_map_)) - { - return false; - } - - // Check if C permute dimension matches GEMM + GEMM shape - const index_t c_g = arg.c_grid_desc_g_m_n_.GetLength(I0); // unpadded - - if(!(c_g == arg.batch_count_)) - { - printf("DeviceOp: BatchCount err"); - return false; - } - - // Note: we need raw lengths since threadwise copy can not handle vector load when part of - // vector is out of bounds - // Note: need lowest dim in Ms/Ns/Ks/Os, not merged M/N/K/O - const auto MzRaw = arg.raw_lengths_mz_lz_kz_nz_[0]; - const auto LzRaw = arg.raw_lengths_mz_lz_kz_nz_[1]; - const auto KzRaw = arg.raw_lengths_mz_lz_kz_nz_[2]; - const auto NzRaw = arg.raw_lengths_mz_lz_kz_nz_[3]; - - // Check scalar per vector requirement - const auto a_extent_lowest = ABlockTransferSrcVectorDim == 2 ? KzRaw : MzRaw; - const auto b0_extent_lowest = B0BlockTransferSrcVectorDim == 2 ? KzRaw : LzRaw; - const auto b1_extent_lowest = B1BlockTransferSrcVectorDim == 2 ? LzRaw : NzRaw; - const auto c_extent_lowest = NzRaw; - - if(!(a_extent_lowest % ABlockTransferSrcScalarPerVector == 0 && - b0_extent_lowest % B0BlockTransferSrcScalarPerVector == 0 && - b1_extent_lowest % B1BlockTransferSrcScalarPerVector == 0 && - c_extent_lowest % CShuffleBlockTransferScalarPerVector_NPerBlock == 0)) - { - printf("DeviceOp: Data Transfer Vector scalar err"); - return false; - } - - // Check vector load/store requirement - const auto a_stride_lowest = - ABlockTransferSrcVectorDim == 2 ? arg.a_mz_kz_strides_[1] : arg.a_mz_kz_strides_[0]; - const auto b0_stride_lowest = - B0BlockTransferSrcVectorDim == 2 ? arg.b0_lz_kz_strides_[1] : arg.b0_lz_kz_strides_[0]; - const auto b1_stride_lowest = - B1BlockTransferSrcVectorDim == 2 ? arg.b1_nz_lz_strides_[1] : arg.b1_nz_lz_strides_[0]; - const auto c_stride_lowest = arg.c_mz_nz_strides_[1]; - - if(!(a_stride_lowest == 1 || b0_stride_lowest == 1 || b1_stride_lowest == 1 || - c_stride_lowest == 1)) - { - printf("DeviceOp: Data Vectorize transfer err"); - return false; - } - - return true; - } - - // polymorphic - bool IsSupportedArgument(const BaseArgument* p_arg) override - { - return IsSupportedArgument(*dynamic_cast(p_arg)); - } - - static auto MakeArgument( - const ADataType* p_a, - const B0DataType* p_b0, - const B1DataType* p_b1, - CDataType* p_c, - const std::array p_acc0_biases, - const std::array p_acc1_biases, - const std::array& a_gs_ms_ks_lengths, - const std::array& a_gs_ms_ks_strides, - const std::array& b0_gs_ls_ks_lengths, - const std::array& b0_gs_ls_ks_strides, - const std::array& b1_gs_ns_ls_lengths, - const std::array& b1_gs_ns_ls_strides, - const std::array& c_gs_ms_ns_lengths, - const std::array& c_gs_ms_ns_strides, - const std::array, NumAcc0Bias> acc0_biases_gs_ms_ls_lengths, - const std::array, NumAcc0Bias> acc0_biases_gs_ms_ls_strides, - const std::array, NumAcc1Bias> acc1_biases_gs_ms_ns_lengths, - const std::array, NumAcc1Bias> acc1_biases_gs_ms_ns_strides, - AElementwiseOperation a_element_op, - B0ElementwiseOperation b0_element_op, - AccElementwiseOperation acc_element_op, - B1ElementwiseOperation b1_element_op, - CElementwiseOperation c_element_op) - { - return Argument{p_a, - p_b0, - p_b1, - p_c, - p_acc0_biases, - p_acc1_biases, - a_gs_ms_ks_lengths, - a_gs_ms_ks_strides, - b0_gs_ls_ks_lengths, - b0_gs_ls_ks_strides, - b1_gs_ns_ls_lengths, - b1_gs_ns_ls_strides, - c_gs_ms_ns_lengths, - c_gs_ms_ns_strides, - acc0_biases_gs_ms_ls_lengths, - acc0_biases_gs_ms_ls_strides, - acc1_biases_gs_ms_ns_lengths, - acc1_biases_gs_ms_ns_strides, - 1, - 1, - a_element_op, - b0_element_op, - acc_element_op, - b1_element_op, - c_element_op}; - } -#endif // polymorphic std::unique_ptr MakeArgumentPointer( diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_grouped_query_attention_forward_wmma.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_grouped_query_attention_forward_wmma.hpp index 6aa766ab5ce6..d1269c6d9ae4 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_grouped_query_attention_forward_wmma.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_grouped_query_attention_forward_wmma.hpp @@ -956,147 +956,6 @@ struct DeviceGroupedQueryAttentionForward_Wmma // TODO: properly implement this check return true; } -#if 0 - static bool IsSupportedArgument(const Argument& arg) - { - if(ck::is_gfx11_supported()) - { - if constexpr(!(is_same_v || is_same_v)) - { - printf("DeviceOp: Acc0 Type err"); - return false; - } - - if constexpr(!(is_same_v || is_same_v)) - { - printf("DeviceOp: Acc1 Type err"); - return false; - } - } - else - { - printf("DeviceOp: Arch err"); - return false; - } - - if(!GridwiseOp::CheckValidity(arg.a_grid_desc, - arg.b0_grid_desc, - arg.b1_grid_desc, - arg.c_grid_desc_m_n_, - arg.block_2_ctile_map_)) - { - return false; - } - - // Check if C permute dimension matches GEMM + GEMM shape - const index_t c_g = arg.c_grid_desc_g_m_n_.GetLength(I0); // unpadded - - if(!(c_g == arg.batch_count_)) - { - printf("DeviceOp: BatchCount err"); - return false; - } - - // Note: we need raw lengths since threadwise copy can not handle vector load when part of - // vector is out of bounds - // Note: need lowest dim in Ms/Ns/Ks/Os, not merged M/N/K/O - const auto MzRaw = arg.raw_lengths_mz_lz_kz_nz_[0]; - const auto LzRaw = arg.raw_lengths_mz_lz_kz_nz_[1]; - const auto KzRaw = arg.raw_lengths_mz_lz_kz_nz_[2]; - const auto NzRaw = arg.raw_lengths_mz_lz_kz_nz_[3]; - - // Check scalar per vector requirement - const auto a_extent_lowest = ABlockTransferSrcVectorDim == 2 ? KzRaw : MzRaw; - const auto b0_extent_lowest = B0BlockTransferSrcVectorDim == 2 ? KzRaw : LzRaw; - const auto b1_extent_lowest = B1BlockTransferSrcVectorDim == 2 ? LzRaw : NzRaw; - const auto c_extent_lowest = NzRaw; - - if(!(a_extent_lowest % ABlockTransferSrcScalarPerVector == 0 && - b0_extent_lowest % B0BlockTransferSrcScalarPerVector == 0 && - b1_extent_lowest % B1BlockTransferSrcScalarPerVector == 0 && - c_extent_lowest % CShuffleBlockTransferScalarPerVector_NPerBlock == 0)) - { - printf("DeviceOp: Data Transfer Vector scalar err"); - return false; - } - - // Check vector load/store requirement - const auto a_stride_lowest = - ABlockTransferSrcVectorDim == 2 ? arg.a_mz_kz_strides_[1] : arg.a_mz_kz_strides_[0]; - const auto b0_stride_lowest = - B0BlockTransferSrcVectorDim == 2 ? arg.b0_lz_kz_strides_[1] : arg.b0_lz_kz_strides_[0]; - const auto b1_stride_lowest = - B1BlockTransferSrcVectorDim == 2 ? arg.b1_nz_lz_strides_[1] : arg.b1_nz_lz_strides_[0]; - const auto c_stride_lowest = arg.c_mz_nz_strides_[1]; - - if(!(a_stride_lowest == 1 || b0_stride_lowest == 1 || b1_stride_lowest == 1 || - c_stride_lowest == 1)) - { - printf("DeviceOp: Data Vectorize transfer err"); - return false; - } - - return true; - } - - // polymorphic - bool IsSupportedArgument(const BaseArgument* p_arg) override - { - return IsSupportedArgument(*dynamic_cast(p_arg)); - } - - static auto MakeArgument( - const ADataType* p_a, - const B0DataType* p_b0, - const B1DataType* p_b1, - CDataType* p_c, - const std::array p_acc0_biases, - const std::array p_acc1_biases, - const std::array& a_gs_ms_ks_lengths, - const std::array& a_gs_ms_ks_strides, - const std::array& b0_gs_ls_ks_lengths, - const std::array& b0_gs_ls_ks_strides, - const std::array& b1_gs_ns_ls_lengths, - const std::array& b1_gs_ns_ls_strides, - const std::array& c_gs_ms_ns_lengths, - const std::array& c_gs_ms_ns_strides, - const std::array, NumAcc0Bias> acc0_biases_gs_ms_ls_lengths, - const std::array, NumAcc0Bias> acc0_biases_gs_ms_ls_strides, - const std::array, NumAcc1Bias> acc1_biases_gs_ms_ns_lengths, - const std::array, NumAcc1Bias> acc1_biases_gs_ms_ns_strides, - AElementwiseOperation a_element_op, - B0ElementwiseOperation b0_element_op, - AccElementwiseOperation acc_element_op, - B1ElementwiseOperation b1_element_op, - CElementwiseOperation c_element_op) - { - return Argument{p_a, - p_b0, - p_b1, - p_c, - p_acc0_biases, - p_acc1_biases, - a_gs_ms_ks_lengths, - a_gs_ms_ks_strides, - b0_gs_ls_ks_lengths, - b0_gs_ls_ks_strides, - b1_gs_ns_ls_lengths, - b1_gs_ns_ls_strides, - c_gs_ms_ns_lengths, - c_gs_ms_ns_strides, - acc0_biases_gs_ms_ls_lengths, - acc0_biases_gs_ms_ls_strides, - acc1_biases_gs_ms_ns_lengths, - acc1_biases_gs_ms_ns_strides, - 1, - 1, - a_element_op, - b0_element_op, - acc_element_op, - b1_element_op, - c_element_op}; - } -#endif // polymorphic std::unique_ptr MakeArgumentPointer( diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_multi_query_attention_forward_wmma.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_multi_query_attention_forward_wmma.hpp index a303b6f80831..a9d916c6a031 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_multi_query_attention_forward_wmma.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_multi_query_attention_forward_wmma.hpp @@ -948,147 +948,6 @@ struct DeviceMultiQueryAttentionForward_Wmma // TODO: properly implement this check return true; } -#if 0 - static bool IsSupportedArgument(const Argument& arg) - { - if(ck::is_gfx11_supported()) - { - if constexpr(!(is_same_v || is_same_v)) - { - printf("DeviceOp: Acc0 Type err"); - return false; - } - - if constexpr(!(is_same_v || is_same_v)) - { - printf("DeviceOp: Acc1 Type err"); - return false; - } - } - else - { - printf("DeviceOp: Arch err"); - return false; - } - - if(!GridwiseOp::CheckValidity(arg.a_grid_desc, - arg.b0_grid_desc, - arg.b1_grid_desc, - arg.c_grid_desc_m_n_, - arg.block_2_ctile_map_)) - { - return false; - } - - // Check if C permute dimension matches GEMM + GEMM shape - const index_t c_g = arg.c_grid_desc_g_m_n_.GetLength(I0); // unpadded - - if(!(c_g == arg.batch_count_)) - { - printf("DeviceOp: BatchCount err"); - return false; - } - - // Note: we need raw lengths since threadwise copy can not handle vector load when part of - // vector is out of bounds - // Note: need lowest dim in Ms/Ns/Ks/Os, not merged M/N/K/O - const auto MzRaw = arg.raw_lengths_mz_lz_kz_nz_[0]; - const auto LzRaw = arg.raw_lengths_mz_lz_kz_nz_[1]; - const auto KzRaw = arg.raw_lengths_mz_lz_kz_nz_[2]; - const auto NzRaw = arg.raw_lengths_mz_lz_kz_nz_[3]; - - // Check scalar per vector requirement - const auto a_extent_lowest = ABlockTransferSrcVectorDim == 2 ? KzRaw : MzRaw; - const auto b0_extent_lowest = B0BlockTransferSrcVectorDim == 2 ? KzRaw : LzRaw; - const auto b1_extent_lowest = B1BlockTransferSrcVectorDim == 2 ? LzRaw : NzRaw; - const auto c_extent_lowest = NzRaw; - - if(!(a_extent_lowest % ABlockTransferSrcScalarPerVector == 0 && - b0_extent_lowest % B0BlockTransferSrcScalarPerVector == 0 && - b1_extent_lowest % B1BlockTransferSrcScalarPerVector == 0 && - c_extent_lowest % CShuffleBlockTransferScalarPerVector_NPerBlock == 0)) - { - printf("DeviceOp: Data Transfer Vector scalar err"); - return false; - } - - // Check vector load/store requirement - const auto a_stride_lowest = - ABlockTransferSrcVectorDim == 2 ? arg.a_mz_kz_strides_[1] : arg.a_mz_kz_strides_[0]; - const auto b0_stride_lowest = - B0BlockTransferSrcVectorDim == 2 ? arg.b0_lz_kz_strides_[1] : arg.b0_lz_kz_strides_[0]; - const auto b1_stride_lowest = - B1BlockTransferSrcVectorDim == 2 ? arg.b1_nz_lz_strides_[1] : arg.b1_nz_lz_strides_[0]; - const auto c_stride_lowest = arg.c_mz_nz_strides_[1]; - - if(!(a_stride_lowest == 1 || b0_stride_lowest == 1 || b1_stride_lowest == 1 || - c_stride_lowest == 1)) - { - printf("DeviceOp: Data Vectorize transfer err"); - return false; - } - - return true; - } - - // polymorphic - bool IsSupportedArgument(const BaseArgument* p_arg) override - { - return IsSupportedArgument(*dynamic_cast(p_arg)); - } - - static auto MakeArgument( - const ADataType* p_a, - const B0DataType* p_b0, - const B1DataType* p_b1, - CDataType* p_c, - const std::array p_acc0_biases, - const std::array p_acc1_biases, - const std::array& a_gs_ms_ks_lengths, - const std::array& a_gs_ms_ks_strides, - const std::array& b0_gs_ls_ks_lengths, - const std::array& b0_gs_ls_ks_strides, - const std::array& b1_gs_ns_ls_lengths, - const std::array& b1_gs_ns_ls_strides, - const std::array& c_gs_ms_ns_lengths, - const std::array& c_gs_ms_ns_strides, - const std::array, NumAcc0Bias> acc0_biases_gs_ms_ls_lengths, - const std::array, NumAcc0Bias> acc0_biases_gs_ms_ls_strides, - const std::array, NumAcc1Bias> acc1_biases_gs_ms_ns_lengths, - const std::array, NumAcc1Bias> acc1_biases_gs_ms_ns_strides, - AElementwiseOperation a_element_op, - B0ElementwiseOperation b0_element_op, - AccElementwiseOperation acc_element_op, - B1ElementwiseOperation b1_element_op, - CElementwiseOperation c_element_op) - { - return Argument{p_a, - p_b0, - p_b1, - p_c, - p_acc0_biases, - p_acc1_biases, - a_gs_ms_ks_lengths, - a_gs_ms_ks_strides, - b0_gs_ls_ks_lengths, - b0_gs_ls_ks_strides, - b1_gs_ns_ls_lengths, - b1_gs_ns_ls_strides, - c_gs_ms_ns_lengths, - c_gs_ms_ns_strides, - acc0_biases_gs_ms_ls_lengths, - acc0_biases_gs_ms_ls_strides, - acc1_biases_gs_ms_ns_lengths, - acc1_biases_gs_ms_ns_strides, - 1, - 1, - a_element_op, - b0_element_op, - acc_element_op, - b1_element_op, - c_element_op}; - } -#endif // polymorphic std::unique_ptr MakeArgumentPointer( From d7cf680e394f236145c18977bc7f09ccf334dabd Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:28:58 -0400 Subject: [PATCH 07/49] Remove dead #if 0 blocks in ck/tensor_operation/gpu/grid Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../gridwise_gemm_xdl_cshuffle_streamk_v3.hpp | 175 ---- .../grid/gridwise_gemm_xdl_cshuffle_v3.hpp | 39 - ...wise_gemm_xdl_cshuffle_v3_b_preshuffle.hpp | 39 - .../gridwise_gemm_xdl_cshuffle_v3_b_scale.hpp | 39 - ...ridwise_gemm_xdl_cshuffle_v3_multi_abd.hpp | 59 -- .../gridwise_gemm_xdl_cshuffle_v3_multi_d.hpp | 39 - ..._gemm_xdl_cshuffle_v3_multi_d_ab_scale.hpp | 39 - ...m_xdl_cshuffle_v3_multi_d_b_preshuffle.hpp | 39 - .../grid/gridwise_gemm_xdl_cshuffle_v3_mx.hpp | 39 - ...se_gemm_xdl_cshuffle_v3_mx_bpreshuffle.hpp | 39 - .../gpu/grid/gridwise_moe_mx_gemm.hpp | 806 ------------------ .../gpu/grid/gridwise_moe_mx_gemm_bns.hpp | 44 - 12 files changed, 1396 deletions(-) diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_streamk_v3.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_streamk_v3.hpp index 4b679adc8dba..2252ebf98008 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_streamk_v3.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_streamk_v3.hpp @@ -351,74 +351,6 @@ struct GridwiseGemm_xdl_cshuffle_streamk_v3 make_tuple(Sequence<0, 2>{}, Sequence<1>{})); return a_grid_desc_ak0_m_ak1; -#if 0 - using GemmSpecialization = tensor_operation::device::GemmSpecialization; - - if constexpr(GemmSpec == GemmSpecialization::MKPadding || - GemmSpec == GemmSpecialization::MNKPadding) - { - // pad both M and K - const auto a_grid_desc_m_k = - transform_tensor_descriptor(a_grid_desc_mraw_kraw, - make_tuple(make_right_pad_transform(M, MPad - M), - make_right_pad_transform(K, KPad - K)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - - const auto a_grid_desc_ak0_m_ak1 = transform_tensor_descriptor( - a_grid_desc_m_k, - make_tuple(make_unmerge_transform(make_tuple(AK0, AK1Value)), - make_pass_through_transform(MPad)), - make_tuple(Sequence<1>{}, Sequence<0>{}), - make_tuple(Sequence<0, 2>{}, Sequence<1>{})); - - return a_grid_desc_ak0_m_ak1; - } - else if constexpr(GemmSpec == GemmSpecialization::MPadding || - GemmSpec == GemmSpecialization::MNPadding) - { - // pad M, but not K - const auto a_grid_desc_ak0_m_ak1 = transform_tensor_descriptor( - a_grid_desc_mraw_kraw, - make_tuple(make_unmerge_transform(make_tuple(AK0, AK1Value)), - make_right_pad_transform(M, MPad - M)), - make_tuple(Sequence<1>{}, Sequence<0>{}), - make_tuple(Sequence<0, 2>{}, Sequence<1>{})); - - return a_grid_desc_ak0_m_ak1; - } - else if constexpr(GemmSpec == GemmSpecialization::KPadding || - GemmSpec == GemmSpecialization::NKPadding) - { - // pad K, but not M - const auto a_grid_desc_m_k = transform_tensor_descriptor( - a_grid_desc_mraw_kraw, - make_tuple(make_pass_through_transform(M), make_right_pad_transform(K, KPad - K)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - - const auto a_grid_desc_ak0_m_ak1 = transform_tensor_descriptor( - a_grid_desc_m_k, - make_tuple(make_unmerge_transform(make_tuple(AK0, AK1Value)), - make_pass_through_transform(M)), - make_tuple(Sequence<1>{}, Sequence<0>{}), - make_tuple(Sequence<0, 2>{}, Sequence<1>{})); - - return a_grid_desc_ak0_m_ak1; - } - else - { - // not pad M or K - const auto a_grid_desc_ak0_m_ak1 = transform_tensor_descriptor( - a_grid_desc_mraw_kraw, - make_tuple(make_unmerge_transform(make_tuple(AK0, AK1Value)), - make_pass_through_transform(M)), - make_tuple(Sequence<1>{}, Sequence<0>{}), - make_tuple(Sequence<0, 2>{}, Sequence<1>{})); - - return a_grid_desc_ak0_m_ak1; - } -#endif } __device__ static auto MakeBGridDescriptor_BK0_N_BK1( @@ -451,74 +383,6 @@ struct GridwiseGemm_xdl_cshuffle_streamk_v3 make_tuple(Sequence<0, 2>{}, Sequence<1>{})); return b_grid_desc_bk0_n_bk1; -#if 0 - using GemmSpecialization = tensor_operation::device::GemmSpecialization; - - if constexpr(GemmSpec == GemmSpecialization::NKPadding || - GemmSpec == GemmSpecialization::MNKPadding) - { - // pad both N and K - const auto b_grid_desc_n_k = - transform_tensor_descriptor(b_grid_desc_nraw_kraw, - make_tuple(make_right_pad_transform(N, NPad - N), - make_right_pad_transform(K, KPad - K)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - - const auto b_grid_desc_bk0_n_bk1 = transform_tensor_descriptor( - b_grid_desc_n_k, - make_tuple(make_unmerge_transform(make_tuple(BK0, BK1Value)), - make_pass_through_transform(NPad)), - make_tuple(Sequence<1>{}, Sequence<0>{}), - make_tuple(Sequence<0, 2>{}, Sequence<1>{})); - - return b_grid_desc_bk0_n_bk1; - } - else if constexpr(GemmSpec == GemmSpecialization::NPadding || - GemmSpec == GemmSpecialization::MNPadding) - { - // pad N, but not K - const auto b_grid_desc_bk0_n_bk1 = transform_tensor_descriptor( - b_grid_desc_nraw_kraw, - make_tuple(make_unmerge_transform(make_tuple(BK0, BK1Value)), - make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<1>{}, Sequence<0>{}), - make_tuple(Sequence<0, 2>{}, Sequence<1>{})); - - return b_grid_desc_bk0_n_bk1; - } - else if constexpr(GemmSpec == GemmSpecialization::KPadding || - GemmSpec == GemmSpecialization::MKPadding) - { - // pad K, but not N - const auto b_grid_desc_n_k = transform_tensor_descriptor( - b_grid_desc_nraw_kraw, - make_tuple(make_pass_through_transform(N), make_right_pad_transform(K, KPad - K)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - - const auto b_grid_desc_bk0_n_bk1 = transform_tensor_descriptor( - b_grid_desc_n_k, - make_tuple(make_unmerge_transform(make_tuple(BK0, BK1Value)), - make_pass_through_transform(N)), - make_tuple(Sequence<1>{}, Sequence<0>{}), - make_tuple(Sequence<0, 2>{}, Sequence<1>{})); - - return b_grid_desc_bk0_n_bk1; - } - else - { - // not pad N or K - const auto b_grid_desc_bk0_n_bk1 = transform_tensor_descriptor( - b_grid_desc_nraw_kraw, - make_tuple(make_unmerge_transform(make_tuple(BK0, BK1Value)), - make_pass_through_transform(N)), - make_tuple(Sequence<1>{}, Sequence<0>{}), - make_tuple(Sequence<0, 2>{}, Sequence<1>{})); - - return b_grid_desc_bk0_n_bk1; - } -#endif } template @@ -559,45 +423,6 @@ struct GridwiseGemm_xdl_cshuffle_streamk_v3 make_right_pad_transform(N, NPad - N)), make_tuple(Sequence<0>{}, Sequence<1>{}), make_tuple(Sequence<0>{}, Sequence<1>{})); -#if 0 - using GemmSpecialization = tensor_operation::device::GemmSpecialization; - - if constexpr(GemmSpec == GemmSpecialization::MNPadding || - GemmSpec == GemmSpecialization::MNKPadding) - { - // pad M and N - return transform_tensor_descriptor(c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), - make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::MPadding || - GemmSpec == GemmSpecialization::MKPadding) - { - // pad M, but not N - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), make_pass_through_transform(N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::NPadding || - GemmSpec == GemmSpecialization::NKPadding) - { - // pad N, but not M - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_pass_through_transform(M), make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else - { - // not pad M or N - return c_grid_desc_mraw_nraw; - } -#endif } struct Problem diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3.hpp index 5c5eb9405f50..d926efab843a 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3.hpp @@ -682,45 +682,6 @@ struct GridwiseGemm_xdl_cshuffle_v3 make_right_pad_transform(N, NPad - N)), make_tuple(Sequence<0>{}, Sequence<1>{}), make_tuple(Sequence<0>{}, Sequence<1>{})); -#if 0 - using GemmSpecialization = tensor_operation::device::GemmSpecialization; - - if constexpr(GemmSpec == GemmSpecialization::MNPadding || - GemmSpec == GemmSpecialization::MNKPadding) - { - // pad M and N - return transform_tensor_descriptor(c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), - make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::MPadding || - GemmSpec == GemmSpecialization::MKPadding) - { - // pad M, but not N - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), make_pass_through_transform(N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::NPadding || - GemmSpec == GemmSpecialization::NKPadding) - { - // pad N, but not M - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_pass_through_transform(M), make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else - { - // not pad M or N - return c_grid_desc_mraw_nraw; - } -#endif } struct Problem diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_b_preshuffle.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_b_preshuffle.hpp index 7f1a42fb2637..a81679ea78cd 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_b_preshuffle.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_b_preshuffle.hpp @@ -613,45 +613,6 @@ struct GridwiseGemm_xdl_cshuffle_v3_b_preshuffle make_right_pad_transform(N, NPad - N)), make_tuple(Sequence<0>{}, Sequence<1>{}), make_tuple(Sequence<0>{}, Sequence<1>{})); -#if 0 - using GemmSpecialization = tensor_operation::device::GemmSpecialization; - - if constexpr(GemmSpec == GemmSpecialization::MNPadding || - GemmSpec == GemmSpecialization::MNKPadding) - { - // pad M and N - return transform_tensor_descriptor(c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), - make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::MPadding || - GemmSpec == GemmSpecialization::MKPadding) - { - // pad M, but not N - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), make_pass_through_transform(N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::NPadding || - GemmSpec == GemmSpecialization::NKPadding) - { - // pad N, but not M - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_pass_through_transform(M), make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else - { - // not pad M or N - return c_grid_desc_mraw_nraw; - } -#endif } struct Problem diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_b_scale.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_b_scale.hpp index daa4fd2e8a2a..f9be9e494ba8 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_b_scale.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_b_scale.hpp @@ -568,45 +568,6 @@ struct GridwiseGemm_xdl_cshuffle_v3 make_right_pad_transform(N, NPad - N)), make_tuple(Sequence<0>{}, Sequence<1>{}), make_tuple(Sequence<0>{}, Sequence<1>{})); -#if 0 - using GemmSpecialization = tensor_operation::device::GemmSpecialization; - - if constexpr(GemmSpec == GemmSpecialization::MNPadding || - GemmSpec == GemmSpecialization::MNKPadding) - { - // pad M and N - return transform_tensor_descriptor(c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), - make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::MPadding || - GemmSpec == GemmSpecialization::MKPadding) - { - // pad M, but not N - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), make_pass_through_transform(N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::NPadding || - GemmSpec == GemmSpecialization::NKPadding) - { - // pad N, but not M - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_pass_through_transform(M), make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else - { - // not pad M or N - return c_grid_desc_mraw_nraw; - } -#endif } struct Problem diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_abd.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_abd.hpp index f018730300eb..58eed43be94f 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_abd.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_abd.hpp @@ -806,58 +806,6 @@ struct GridwiseGemm_xdl_cshuffle_v3 index_t b_k_split_offset; }; -#if 0 - struct SplitKBatchOffsetMultiABD - { - __device__ SplitKBatchOffsetMultiABD(AsGridPointer& p_as_grid, - BsGridPointer& p_bs_grid, - Argument& karg) - { - static_for<0, NumATensor, 1>{}([&](auto i) { - using ALayout_ = remove_cvref_t>; - if constexpr(is_same_v) - { - as_k_split_offset[i] = blockIdx.z * karg.KRead; - } - else if constexpr(is_same_v) - { - as_k_split_offset[i] = blockIdx.z * karg.KRead * karg.StrideAs[i]; - } - - p_as_grid_(i) = p_as_grid[i] + as_k_split_offset[i]; - }); - - static_for<0, NumBTensor, 1>{}([&](auto i) { - using BLayout_ = remove_cvref_t>; - if constexpr(is_same_v) - { - bs_k_split_offset[i] = blockIdx.z * karg.KRead * karg.StrideBs[i]; - } - else if constexpr(is_same_v) - { - bs_k_split_offset[i] = blockIdx.z * karg.KRead; - } - - p_bs_grid_(i) = p_bs_grid[i] + bs_k_split_offset[i]; - }); - - if(blockIdx.z < static_cast(karg.KBatch - 1)) - { - karg.K = karg.KRead; - } - else - { - karg.K = karg.K - karg.KRead * (karg.KBatch - 1); - } - } - - AsGridPointer p_as_grid_; - BsGridPointer p_bs_grid_; - std::array as_k_split_offset; - std::array bs_k_split_offset; - }; -#endif - using BlockwiseGemmPipe = remove_cvref_t< decltype(BlockGemmPipeline_Selector< BlkGemmPipelineVer, @@ -1147,13 +1095,6 @@ struct GridwiseGemm_xdl_cshuffle_v3 const auto ds_grid_desc_m_n = MakeDsGridDescriptor_M_N( problem.M, problem.MPadded, problem.N, problem.NPadded, problem.StrideDs); -#if 0 - static_for<0, NumDTensor, 1>{}([&](auto j) { - ds_grid_desc_m_n(j) = MakeCGridDescriptor_M_N( - problem.M, problem.MPadded, problem.N, problem.NPadded, problem.StrideDs[j]); - }); -#endif - const auto ds_grid_desc_mblock_mperblock_nblock_nperblock = MakeDsGridDescriptor_MBlock_MPerBlock_NBlock_NPerBlock( ds_grid_desc_m_n, problem.MBlock, problem.NBlock); diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_d.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_d.hpp index a3dffed09d31..671cfe4967f7 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_d.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_d.hpp @@ -642,45 +642,6 @@ struct GridwiseGemmMultiD_xdl_cshuffle_v3 make_right_pad_transform(N, NPad - N)), make_tuple(Sequence<0>{}, Sequence<1>{}), make_tuple(Sequence<0>{}, Sequence<1>{})); -#if 0 - using GemmSpecialization = tensor_operation::device::GemmSpecialization; - - if constexpr(GemmSpec == GemmSpecialization::MNPadding || - GemmSpec == GemmSpecialization::MNKPadding) - { - // pad M and N - return transform_tensor_descriptor(c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), - make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::MPadding || - GemmSpec == GemmSpecialization::MKPadding) - { - // pad M, but not N - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), make_pass_through_transform(N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::NPadding || - GemmSpec == GemmSpecialization::NKPadding) - { - // pad N, but not M - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_pass_through_transform(M), make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else - { - // not pad M or N - return c_grid_desc_mraw_nraw; - } -#endif } __host__ __device__ static auto MakeDsGridDescriptor_M_N( diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_d_ab_scale.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_d_ab_scale.hpp index 36895f55eab7..54260d4386af 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_d_ab_scale.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_d_ab_scale.hpp @@ -558,45 +558,6 @@ struct GridwiseGemmMultiD_ABScale_xdl_cshuffle_v3 make_right_pad_transform(N, NPad - N)), make_tuple(Sequence<0>{}, Sequence<1>{}), make_tuple(Sequence<0>{}, Sequence<1>{})); -#if 0 - using GemmSpecialization = tensor_operation::device::GemmSpecialization; - - if constexpr(GemmSpec == GemmSpecialization::MNPadding || - GemmSpec == GemmSpecialization::MNKPadding) - { - // pad M and N - return transform_tensor_descriptor(c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), - make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::MPadding || - GemmSpec == GemmSpecialization::MKPadding) - { - // pad M, but not N - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), make_pass_through_transform(N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::NPadding || - GemmSpec == GemmSpecialization::NKPadding) - { - // pad N, but not M - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_pass_through_transform(M), make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else - { - // not pad M or N - return c_grid_desc_mraw_nraw; - } -#endif } __host__ __device__ static auto MakeDsGridDescriptor_M_N( diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_d_b_preshuffle.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_d_b_preshuffle.hpp index e810a467e762..28bcf14cd04e 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_d_b_preshuffle.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_d_b_preshuffle.hpp @@ -609,45 +609,6 @@ struct GridwiseGemmMultiD_xdl_cshuffle_v3_b_preshuffle make_right_pad_transform(N, NPad - N)), make_tuple(Sequence<0>{}, Sequence<1>{}), make_tuple(Sequence<0>{}, Sequence<1>{})); -#if 0 - using GemmSpecialization = tensor_operation::device::GemmSpecialization; - - if constexpr(GemmSpec == GemmSpecialization::MNPadding || - GemmSpec == GemmSpecialization::MNKPadding) - { - // pad M and N - return transform_tensor_descriptor(c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), - make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::MPadding || - GemmSpec == GemmSpecialization::MKPadding) - { - // pad M, but not N - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), make_pass_through_transform(N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::NPadding || - GemmSpec == GemmSpecialization::NKPadding) - { - // pad N, but not M - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_pass_through_transform(M), make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else - { - // not pad M or N - return c_grid_desc_mraw_nraw; - } -#endif } __host__ __device__ static auto MakeDsGridDescriptor_M_N( diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_mx.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_mx.hpp index d2dd1d243cb0..fa231c9b02e8 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_mx.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_mx.hpp @@ -669,45 +669,6 @@ struct GridwiseGemmMX_xdl_cshuffle_v3 make_right_pad_transform(N, NPad - N)), make_tuple(Sequence<0>{}, Sequence<1>{}), make_tuple(Sequence<0>{}, Sequence<1>{})); -#if 0 - using GemmSpecialization = tensor_operation::device::GemmSpecialization; - - if constexpr(GemmSpec == GemmSpecialization::MNPadding || - GemmSpec == GemmSpecialization::MNKPadding) - { - // pad M and N - return transform_tensor_descriptor(c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), - make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::MPadding || - GemmSpec == GemmSpecialization::MKPadding) - { - // pad M, but not N - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), make_pass_through_transform(N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::NPadding || - GemmSpec == GemmSpecialization::NKPadding) - { - // pad N, but not M - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_pass_through_transform(M), make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else - { - // not pad M or N - return c_grid_desc_mraw_nraw; - } -#endif } struct Problem diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_mx_bpreshuffle.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_mx_bpreshuffle.hpp index 88f5dd44f3f9..43a46d6ff458 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_mx_bpreshuffle.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_mx_bpreshuffle.hpp @@ -696,45 +696,6 @@ struct GridwiseGemmMX_xdl_cshuffle_v3_bpreshuffle make_right_pad_transform(N, NPad - N)), make_tuple(Sequence<0>{}, Sequence<1>{}), make_tuple(Sequence<0>{}, Sequence<1>{})); -#if 0 - using GemmSpecialization = tensor_operation::device::GemmSpecialization; - - if constexpr(GemmSpec == GemmSpecialization::MNPadding || - GemmSpec == GemmSpecialization::MNKPadding) - { - // pad M and N - return transform_tensor_descriptor(c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), - make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::MPadding || - GemmSpec == GemmSpecialization::MKPadding) - { - // pad M, but not N - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_right_pad_transform(M, MPad - M), make_pass_through_transform(N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else if constexpr(GemmSpec == GemmSpecialization::NPadding || - GemmSpec == GemmSpecialization::NKPadding) - { - // pad N, but not M - return transform_tensor_descriptor( - c_grid_desc_mraw_nraw, - make_tuple(make_pass_through_transform(M), make_right_pad_transform(N, NPad - N)), - make_tuple(Sequence<0>{}, Sequence<1>{}), - make_tuple(Sequence<0>{}, Sequence<1>{})); - } - else - { - // not pad M or N - return c_grid_desc_mraw_nraw; - } -#endif } struct Problem diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_moe_mx_gemm.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_moe_mx_gemm.hpp index 79e3a4466024..d1d136bcc87a 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_moe_mx_gemm.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_moe_mx_gemm.hpp @@ -30,48 +30,6 @@ namespace ck { // 2. Occupied __shared__ won't release until whole shader end, a.k.a AB and C may not use same lds // buffer when we declare __shared__ inside blkgemmpipe -#if 0 -template -__global__ void -#if CK_USE_LAUNCH_BOUNDS -__launch_bounds__(GridwiseGemm::MaxBlockSize, MinimumOccupancy) -#endif - // __attribute__((amdgpu_waves_per_eu(1, 1))) - kernel_moe_mxgemm(typename GridwiseGemm::Argument karg) -{ -#if defined(__gfx9__) - if constexpr(GridwiseGemm::template IsValidCompilationParameter()) - { - __shared__ char p_shared[GridwiseGemm::GetSharedMemoryNumberOfByte()]; - - auto splitk_batch_offset = typename GridwiseGemm::SplitKBatchOffset(karg, blockIdx.z); - - GridwiseGemm::template Run( - karg.p_sorted_token_ids, - karg.p_sorted_expert_ids, - karg.p_max_token_id, - karg.p_a_grid + splitk_batch_offset.a_k_split_offset, - karg.p_a_scale_grid + splitk_batch_offset.a_k_split_offset, - karg.p_b_grid + splitk_batch_offset.b_k_split_offset, - karg.p_b_scale_grid + splitk_batch_offset.b_k_split_offset, - karg.p_ds_grid, - karg.p_c_grid, - p_shared, - karg, - karg.a_element_op, - karg.b_element_op, - karg.c_element_op); - } -#else - ignore = karg; -#endif // end of if (defined(__gfx9__)) -} -#endif - template , "A/B ElementwiseOperation should be PassThrough as load_to_lds is used!"); -#if 0 - template - __device__ static void Run(const index_t* p_sorted_token_ids, - const index_t* p_sorted_expert_ids, - const index_t* p_max_token_id, - const ADataType* p_a_grid, - const AScaleDataType* p_a_scale_grid, - const BDataType* p_b_grid, - const BScaleDataType* p_b_scale_grid, - DsGridPointer& p_ds_grid, - CDataType* p_c_grid, - void* p_shared, - const Problem& problem, - AElementwiseOperation a_element_op, - BElementwiseOperation b_element_op, - CElementwiseOperation c_element_op) - { - ignore = a_element_op; - ignore = b_element_op; - const auto a_grid_desc_ak0_m_ak1 = MakeAGridDescriptor_AK0_M_AK1( - IsInputGemm ? problem.NumTokens : problem.NumTokens * problem.TopK, - problem.MPadded, - problem.K, - problem.KPadded, - problem.StrideA, - problem.AK0); - const auto b_grid_desc_bk0_n_bk1 = MakeBGridDescriptor_BK0_N_BK1( - problem.K, problem.KPadded, problem.N, problem.NPadded, problem.StrideB, problem.BK0); - const auto c_grid_desc_m_n = MakeCGridDescriptor_M_N( - IsInputGemm ? problem.NumTokens * problem.TopK : problem.NumTokens, - problem.MPadded, - problem.N, - problem.NPadded, - problem.StrideC); - - const auto a_scale_grid_desc_am_ak = make_naive_tensor_descriptor_packed( - make_tuple(problem.M / (MXdlPack * MPerXdl), - math::integer_divide_ceil(problem.K, (ScaleBlockSize / APackedSize)) / - (KXdlPack * 64 / MPerXdl), - 64 * KXdlPack * MXdlPack / scale_pack_size_a)); - - const auto b_scale_grid_desc_bn_ak = make_naive_tensor_descriptor_packed( - make_tuple(problem.N / (NXdlPack * NPerXdl), - math::integer_divide_ceil(problem.K, (ScaleBlockSize / BPackedSize)) / - (KXdlPack * 64 / NPerXdl), - 64 * KXdlPack * NXdlPack / scale_pack_size_b)); - - const auto c_grid_desc_mblock_mperblock_nblock_nperblock = - MakeCGridDescriptor_MBlock_MPerBlock_NBlock_NPerBlock( - c_grid_desc_m_n, problem.MBlock, problem.NBlock); - - const index_t max_token_id = __builtin_amdgcn_readfirstlane(p_max_token_id[0]); - const index_t expert_block_id = NSwizzle ? blockIdx.x / problem.NBlock : blockIdx.y; - if(expert_block_id * MPerBlock >= max_token_id) - return; - const index_t expert_id = - __builtin_amdgcn_readfirstlane(p_sorted_expert_ids[expert_block_id]); - - const auto block_mn = [&]() -> std::pair { - if constexpr(NSwizzle) - { - const index_t ecnt_prefix = p_max_token_id[1 + expert_id]; - const index_t prefix_block = ecnt_prefix * problem.NBlock; - const index_t ecnt = p_max_token_id[2 + expert_id] - ecnt_prefix; - const index_t expert_swizzle = - ecnt > 0 ? ecnt : 1; // p_max_token_id[expert_id + 1]; // 2 - const index_t bid_new = blockIdx.x - prefix_block; - const index_t nid = __builtin_amdgcn_readfirstlane( - bid_new % 8 + bid_new / (8 * expert_swizzle) * 8); - const index_t mid = - __builtin_amdgcn_readfirstlane(ecnt_prefix + bid_new / 8 % expert_swizzle); - return {nid, mid}; - } - else - { - return {blockIdx.x, blockIdx.y}; - } - }(); - - const index_t block_n_id = block_mn.first; - const index_t block_m_id = block_mn.second; - const index_t token0 = - __builtin_amdgcn_readfirstlane(p_sorted_token_ids[block_m_id * MPerBlock] & 0xffffff); - - // constexpr auto M0 = ABlockTransferThreadClusterLengths_AK0_M_AK1{}.At(I1); - constexpr auto AMThreads = ABlockTransferThreadClusterLengths_AK0_M_AK1{}.At(I1); - constexpr auto AK0Threads = ABlockTransferThreadClusterLengths_AK0_M_AK1{}.At(I0); - constexpr auto AK1Threads = ABlockTransferThreadClusterLengths_AK0_M_AK1{}.At(I2); - constexpr auto AKThreads = AK0Threads * AK1Threads; - constexpr auto AMRepeats = MPerBlock / AMThreads; - const index_t token_pos = block_m_id * MPerBlock + threadIdx.x / AKThreads * AMRepeats; - - if(token_pos >= max_token_id || token0 >= problem.NumTokens) - return; - StaticallyIndexedArray gather_offsets; - static_for<0, AMRepeats, 1>{}([&](auto m0) { - const index_t fused_token = p_sorted_token_ids[token_pos + m0]; - index_t token_offset = fused_token & 0xffffff; - if constexpr(!IsInputGemm) - { - token_offset = token_offset * problem.TopK + (fused_token >> 24); - } - gather_offsets(m0) = static_cast(token_offset); - }); - - const long_index_t expert_stride = - __builtin_amdgcn_readfirstlane(static_cast(problem.N) * problem.K * (IsInputGemm ? 2 : 1)); - const long_index_t expert_scale_stride = __builtin_amdgcn_readfirstlane( - static_cast(problem.N) * (IsInputGemm ? 2 : 1) * - math::integer_divide_ceil(problem.K, ScaleBlockSize / BPackedSize)); - - // N0, K0, Blocksize*KPack - const index_t n_block_data_idx_on_grid = - __builtin_amdgcn_readfirstlane(block_n_id * NPerBlock); - - // Gride buffer creation - const auto a_grid_buf = make_dynamic_buffer( - p_a_grid, a_grid_desc_ak0_m_ak1.GetElementSpaceSize()); - const auto b_grid_buf = make_dynamic_buffer( - p_b_grid + static_cast(expert_id) * expert_stride, b_grid_desc_bk0_n_bk1.GetElementSpaceSize()); - - // A, B scale buffer - const auto a_scale_grid_buf = make_dynamic_buffer( - p_a_scale_grid, a_scale_grid_desc_am_ak.GetElementSpaceSize()); - const auto b_scale_grid_buf = make_dynamic_buffer( - p_b_scale_grid + (static_cast(expert_id) * expert_scale_stride) / sizeof(BScaleDataType), - b_scale_grid_desc_bn_ak.GetElementSpaceSize()); - - // lds max alignment - constexpr auto max_lds_align = math::lcm(AK1Number, BK1Number); - - // A matrix in LDS memory, dst of blockwise copy - constexpr auto a_block_desc_ak0_m_ak1 = GetABlockDescriptor_AK0PerBlock_MPerBlock_AK1(); - - // B matrix in LDS memory, dst of blockwise copy - constexpr auto b_block_desc_bk0_n_bk1 = GetBBlockDescriptor_BK0PerBlock_NPerBlock_BK1(); - - // A matrix blockwise direct to LDS copy - auto a_blockwise_copy = ThreadGroupTensorSliceTransfer_Gather_DirectLoad< - ThisThreadBlock, - Sequence, - ABlockTransferThreadClusterLengths_AK0_M_AK1, - ABlockTransferThreadClusterArrangeOrder, - ADataType, - ADataType, - decltype(a_grid_desc_ak0_m_ak1), - decltype(a_block_desc_ak0_m_ak1), - ABlockTransferSrcAccessOrder, - ABlockTransferSrcVectorDim, - 2, - ABlockTransferSrcScalarPerVector, - IndexType, - 1>(a_grid_desc_ak0_m_ak1, - make_multi_index(0, 0, 0), - a_block_desc_ak0_m_ak1, - make_multi_index(0, 0, 0), - gather_offsets); - - // B matrix blockwise copy - auto b_blockwise_copy = - ThreadGroupTensorSliceTransfer_DirectLoad, - BBlockTransferThreadClusterLengths_BK0_N_BK1, - BBlockTransferThreadClusterArrangeOrder, - BDataType, - BDataType, - decltype(b_grid_desc_bk0_n_bk1), - decltype(b_block_desc_bk0_n_bk1), - BBlockTransferSrcAccessOrder, - BBlockTransferSrcVectorDim, - 2, - BBlockTransferSrcScalarPerVector>( - b_grid_desc_bk0_n_bk1, - make_multi_index(0, n_block_data_idx_on_grid, 0), - b_block_desc_bk0_n_bk1, - make_multi_index(0, 0, 0)); - - // LDS allocation for A and B: be careful of alignment - constexpr auto a_block_space_size_aligned = math::integer_least_multiple( - a_block_desc_ak0_m_ak1.GetElementSpaceSize(), max_lds_align); - - // Cast after lds - auto a_block_buf = make_dynamic_buffer( - static_cast(p_shared), a_block_desc_ak0_m_ak1.GetElementSpaceSize()); - - auto b_block_buf = make_dynamic_buffer( - reinterpret_cast(static_cast(p_shared) + - a_block_space_size_aligned * sizeof(ADataType)), - b_block_desc_bk0_n_bk1.GetElementSpaceSize()); - - constexpr auto a_block_slice_copy_step = make_multi_index(KPerBlock / AK1Number, 0, 0); - constexpr auto b_block_slice_copy_step = make_multi_index(KPerBlock / BK1Number, 0, 0); - - // Blockwise GEMM pipeline - static_assert(std::is_default_constructible_v); - auto blockwise_gemm_pipeline = BlockwiseGemmPipe{}; - auto c_thread_buf = blockwise_gemm_pipeline.GetCThreadBuffer(); - decltype(c_thread_buf) c_thread_buf_up; - - StaticBufferTupleOfVector - c_thread_buf_fp32; - - const index_t num_k_block_main_loop = __builtin_amdgcn_readfirstlane( - (a_grid_desc_ak0_m_ak1.GetLength(I0) * a_grid_desc_ak0_m_ak1.GetLength(I2)) / - KPerBlock); - - // a and b scale processing - const auto wave_idx = BlockwiseGemmPipe::GetWaveIdx(); - const auto waveId_m = wave_idx[I0]; - const auto waveId_n = wave_idx[I1]; - - auto thread_offset_shuffled = - get_thread_local_1d_id() % BlockwiseGemmPipe::WaveSize * KXdlPack * MXdlPack; - - auto a_thread_offset_m = waveId_m; - - auto a_scale_thread_copy = ThreadwiseTensorSliceTransfer_v2< - AScaleDataType, - AScaleDataType, - decltype(a_scale_grid_desc_am_ak), - decltype(BlockwiseGemmPipe::a_scale_thread_desc), - Sequence<1, 1, KXdlPack * MXdlPack / scale_pack_size_a>, // SliceLengths - Sequence<0, 1, 2>, // DimAccessOrder - 2, // SrcVectorDim - KXdlPack * MXdlPack / scale_pack_size_a, // SrcScalarPerVector - 1, // SrcScalarStrideInVector - true>(a_scale_grid_desc_am_ak, - make_multi_index(block_m_id * MPerBlock / MPerXdl / MXdlPack + a_thread_offset_m, - 0, - thread_offset_shuffled / scale_pack_size_a)); - - // B scale load - auto b_thread_offset_n = waveId_n; - - auto b_scale_thread_copy = ThreadwiseTensorSliceTransfer_v2< - BScaleDataType, - BScaleDataType, - decltype(b_scale_grid_desc_bn_ak), - decltype(BlockwiseGemmPipe::b_scale_thread_desc), - Sequence<1, 1, KXdlPack * NXdlPack / scale_pack_size_b>, // SliceLengths - Sequence<0, 1, 2>, // DimAccessOrder - 2, // SrcVectorDim - KXdlPack * NXdlPack / scale_pack_size_b, // SrcScalarPerVector - 1, // SrcScalarStrideInVector - true>(b_scale_grid_desc_bn_ak, - make_multi_index(block_n_id * NPerBlock / NPerXdl / NXdlPack + b_thread_offset_n, - 0, - thread_offset_shuffled / scale_pack_size_b)); - - if constexpr(IsInputGemm) - { - constexpr auto b_block_space_size_aligned = math::integer_least_multiple( - b_block_desc_bk0_n_bk1.GetElementSpaceSize(), max_lds_align); - auto b_block_buf_up = make_dynamic_buffer( - reinterpret_cast(static_cast(p_shared) + - a_block_space_size_aligned * sizeof(ADataType) + - b_block_space_size_aligned * sizeof(BDataType)), - b_block_desc_bk0_n_bk1.GetElementSpaceSize()); - - const BDataType* p_b_grid_up = p_b_grid + expert_stride / 2; - const auto b_grid_buf_up = make_dynamic_buffer( - p_b_grid_up + static_cast(expert_id) * expert_stride, - b_grid_desc_bk0_n_bk1.GetElementSpaceSize()); - - auto b_blockwise_copy_up = ThreadGroupTensorSliceTransfer_DirectLoad< - ThisThreadBlock, - Sequence, - BBlockTransferThreadClusterLengths_BK0_N_BK1, - BBlockTransferThreadClusterArrangeOrder, - BDataType, - BDataType, - decltype(b_grid_desc_bk0_n_bk1), - decltype(b_block_desc_bk0_n_bk1), - BBlockTransferSrcAccessOrder, - BBlockTransferSrcVectorDim, - 2, - BBlockTransferSrcScalarPerVector>(b_grid_desc_bk0_n_bk1, - make_multi_index(0, n_block_data_idx_on_grid, 0), - b_block_desc_bk0_n_bk1, - make_multi_index(0, 0, 0)); - - const BScaleDataType* p_b_scale_grid_up = - p_b_scale_grid + expert_scale_stride / 2 / sizeof(BScaleDataType); - const auto b_scale_grid_buf_up = make_dynamic_buffer( - p_b_scale_grid_up + static_cast(expert_id) * expert_scale_stride / sizeof(BScaleDataType), - b_scale_grid_desc_bn_ak.GetElementSpaceSize()); - - auto b_scale_thread_copy_up = ThreadwiseTensorSliceTransfer_v2< - BScaleDataType, - BScaleDataType, - decltype(b_scale_grid_desc_bn_ak), - decltype(BlockwiseGemmPipe::b_scale_thread_desc), - Sequence<1, 1, KXdlPack * NXdlPack / scale_pack_size_b>, // SliceLengths - Sequence<0, 1, 2>, // DimAccessOrder - 2, // SrcVectorDim - KXdlPack * MXdlPack / scale_pack_size_b, // SrcScalarPerVector - 1, // SrcScalarStrideInVector - true>( - b_scale_grid_desc_bn_ak, - make_multi_index(block_n_id * NPerBlock / NPerXdl / NXdlPack + b_thread_offset_n, - 0, - thread_offset_shuffled / scale_pack_size_b)); - - blockwise_gemm_pipeline.template Run( - // A - a_grid_desc_ak0_m_ak1, - a_block_desc_ak0_m_ak1, - a_blockwise_copy, - a_grid_buf, - a_block_buf, - a_block_slice_copy_step, - // Gate and Up - b_grid_desc_bk0_n_bk1, - b_block_desc_bk0_n_bk1, - b_blockwise_copy, - b_blockwise_copy_up, - b_grid_buf, - b_grid_buf_up, - b_block_buf, - b_block_buf_up, - b_block_slice_copy_step, - // C - c_thread_buf, - c_thread_buf_up, - // A scale - a_scale_grid_desc_am_ak, - a_scale_thread_copy, - a_scale_grid_buf, - // Gate and Up scale - b_scale_grid_desc_bn_ak, - b_scale_thread_copy, - b_scale_thread_copy_up, - b_scale_grid_buf, - b_scale_grid_buf_up, - num_k_block_main_loop); - } - else - { - blockwise_gemm_pipeline.template Run( - a_grid_desc_ak0_m_ak1, // A - a_block_desc_ak0_m_ak1, - a_blockwise_copy, - a_grid_buf, - a_block_buf, - a_block_slice_copy_step, - b_grid_desc_bk0_n_bk1, // B - b_block_desc_bk0_n_bk1, - b_blockwise_copy, - b_grid_buf, - b_block_buf, - b_block_slice_copy_step, - c_thread_buf, // C - a_scale_grid_desc_am_ak, // A scale - a_scale_thread_copy, - a_scale_grid_buf, - b_scale_grid_desc_bn_ak, // B scale - b_scale_thread_copy, - b_scale_grid_buf, - num_k_block_main_loop); - } - - // shuffle C and write out - { - static_assert(MXdlPerWave % CShuffleMXdlPerWavePerShuffle == 0 && - NXdlPerWave % CShuffleNXdlPerWavePerShuffle == 0, - "wrong!"); - static_assert(CShuffleMXdlPerWavePerShuffle % MXdlPack == 0 && - CShuffleNXdlPerWavePerShuffle % NXdlPack == 0, - "wrong!"); - - constexpr index_t MWave = MPerBlock / (MXdlPerWave * MPerXdl); - constexpr index_t NWave = NPerBlock / (NXdlPerWave * NPerXdl); - - // TODO: hacky, fix it! - constexpr auto c_thread_desc_m0_n0_m1_n1_m2_m3_m4_n2 = - blockwise_gemm_pipeline.GetCThreadDescriptor_M0_N0_M1_N1_M2_N2_M3_M4_M5_N3(); - - // TODO: hacky, fix it! - // c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2_tmp is only used to get lengths - constexpr auto c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2_tmp = - blockwise_gemm_pipeline.GetCBlockDescriptor_M0_N0_M1_N1_M2_N2_M3_M4_M5_N3(); - - constexpr auto M0 = c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2_tmp.GetLength(I0); - constexpr auto N0 = c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2_tmp.GetLength(I1); - constexpr auto M1 = c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2_tmp.GetLength(I2); - constexpr auto N1 = c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2_tmp.GetLength(I3); - constexpr auto M2 = c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2_tmp.GetLength(I4); - constexpr auto N2 = c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2_tmp.GetLength(I5); - constexpr auto M3 = c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2_tmp.GetLength(I6); - constexpr auto M4 = c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2_tmp.GetLength(I7); - constexpr auto M5 = c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2_tmp.GetLength(I8); - constexpr auto N3 = c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2_tmp.GetLength(I9); - - // mul scales - static_assert(M0 * M1 * M2 * M3 * M4 * M5 == MPerBlock); - static_assert(M5 == 4); - const index_t m1 = get_warp_local_1d_id() / NWave; // Mwave id - const index_t m4 = threadIdx.x % get_warp_size() / MPerXdl; - - vector_type topk_weights; // for gemm2 only - static_for<0, NXdlPerWave / NXdlPack, 1>{}([&](auto n0) { - static_for<0, NXdlPack, 1>{}([&](auto inxdl) { // NXdlPack - static_for<0, MXdlPerWave / MXdlPack, 1>{}([&](auto m0) { // MXDLPerWave - static_for<0, MXdlPack, 1>{}([&](auto imxdl) { // MXdlPack - static_for<0, M3, 1>{}([&](auto m3) { // m_inst_num_groups_per_blk - const index_t m_pos = block_m_id * MPerBlock + - m0 * M2 * M1 * M3 * M4 * M5 + - m1 * M2 * M3 * M4 * M5 + - imxdl * M3 * M4 * M5 + m3 * M4 * M5 + m4 * M5; - if constexpr(MulRoutedWeight) - { - topk_weights = - *c_style_pointer_cast*>( - p_ds_grid[I2] + m_pos); - } - static_for<0, M5, 1>{}([&](auto m5) { // m_inst_group_size - constexpr index_t c_offset = - blockwise_gemm_pipeline.GetCThreadDesc().CalculateOffset( - make_tuple(m0, n0, imxdl, inxdl, m3 * M5 + m5)); - constexpr auto cidx = Number{}; - - if constexpr(IsInputGemm) // gu fusion - { - if constexpr(ActivationOperation == - Activation::silu_and_mul) - { - float gate = c_thread_buf[cidx]; - float up = c_thread_buf_up[cidx]; - if constexpr(MulRoutedWeight) - { - gate = gate * topk_weights.AsType()[m5]; - up = up * topk_weights.AsType()[m5]; - } - tensor_operation::element_wise::Silu{}(gate, gate); - c_thread_buf_fp32(cidx) = gate * up; - } - else if(ActivationOperation == Activation::gelu_and_mul) - { - float gate = c_thread_buf[cidx]; - float up = c_thread_buf_up[cidx]; - if constexpr(MulRoutedWeight) - { - gate = gate * topk_weights.AsType()[m5]; - up = up * topk_weights.AsType()[m5]; - } - tensor_operation::element_wise::Gelu{}(gate, gate); - c_thread_buf_fp32(cidx) = gate * up; - - /*float gate = c_thread_buf[cidx]; - float up = c_thread_buf_up[cidx]; - if constexpr(MulRoutedWeight) - { - gate = gate * topk_weights.AsType()[m5]; - //up = up * topk_weights.AsType()[m5]; - } - tensor_operation::element_wise::Gelu{}(gate, gate); - c_thread_buf_fp32(cidx) = up;*/ - } - } - else - { - c_thread_buf_fp32(cidx) = c_thread_buf[cidx]; - if constexpr(MulRoutedWeight) - { - c_thread_buf_fp32(cidx) = - topk_weights.AsType()[m5] * - c_thread_buf_fp32[cidx]; - } - } - }); - }); - }); - }); - }); - }); - - constexpr auto c_shuffle_block_desc_mblock_mperblock_nblock_nperblock = - GetCShuffleBlockDescriptor_MBlock_MPerBlock_NBlock_NPerBlock(); - - auto c_shuffle_block_buf = make_dynamic_buffer( - static_cast(p_shared), - c_shuffle_block_desc_mblock_mperblock_nblock_nperblock.GetElementSpaceSize()); - - constexpr auto c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2 = transform_tensor_descriptor( - c_shuffle_block_desc_mblock_mperblock_nblock_nperblock, - make_tuple( - make_freeze_transform(I0), - make_unmerge_transform(make_tuple( - Number{}, // M0 (MXdlPerWave) - // per shuffle - M1, // M1 = MWave - M2, // M2 = MXdlPack - M3, // M3 * M4 * M5 = MPerXdl - M4, - M5)), - make_freeze_transform(I0), - make_unmerge_transform(make_tuple( - Number{}, // N0 (NXdlPerWave) - // per shuffle - N1, // N1 = NWave - N2, // N2 = NXdlPack - N3))), // N3 = NPerXdl - make_tuple(Sequence<0>{}, Sequence<1>{}, Sequence<2>{}, Sequence<3>{}), - make_tuple(Sequence<>{}, - Sequence<0, 2, 4, 6, 7, 8>{}, - Sequence<>{}, - Sequence<1, 3, 5, 9>{})); - - // calculate origin of thread output tensor on global memory - // blockwise GEMM c matrix starting index - const auto c_thread_mtx_on_block = - blockwise_gemm_pipeline.CalculateCThreadOriginDataIndex(I0, I0, I0, I0); - - const index_t m_thread_data_on_block = c_thread_mtx_on_block[I0]; - const index_t n_thread_data_on_block = c_thread_mtx_on_block[I1]; - - const auto m_thread_data_on_block_to_m0_m1_m2_m3_m4_adaptor = - make_single_stage_tensor_adaptor( - make_tuple(make_merge_transform(make_tuple(M0, M1, M2, M3, M4, M5))), - make_tuple(Sequence<0, 1, 2, 3, 4, 5>{}), - make_tuple(Sequence<0>{})); - - const auto m_thread_data_on_block_idx = - m_thread_data_on_block_to_m0_m1_m2_m3_m4_adaptor.CalculateBottomIndex( - make_multi_index(m_thread_data_on_block)); - - const auto n_thread_data_on_block_to_n0_n1_n2_adaptor = - make_single_stage_tensor_adaptor( - make_tuple(make_merge_transform(make_tuple(N0, N1, N2, N3))), - make_tuple(Sequence<0, 1, 2, 3>{}), - make_tuple(Sequence<0>{})); - - const auto n_thread_data_on_block_idx = - n_thread_data_on_block_to_n0_n1_n2_adaptor.CalculateBottomIndex( - make_multi_index(n_thread_data_on_block)); - - // shuffle: threadwise copy C from VGPR to LDS - auto c_thread_copy_vgpr_to_lds = ThreadwiseTensorSliceTransfer_v1r3< - AccDataType, - CShuffleDataType, - decltype(c_thread_desc_m0_n0_m1_n1_m2_m3_m4_n2), - decltype(c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2), - ck::tensor_operation::element_wise::PassThrough, - Sequence, - Sequence<0, 1, 2, 3, 4, 5, 6, 7, 8, 9>, - 9, - 1, - InMemoryDataOperationEnum::Set, - 1, - true>{c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2, - make_multi_index(0, - 0, - m_thread_data_on_block_idx[I1], - n_thread_data_on_block_idx[I1], - m_thread_data_on_block_idx[I2], - n_thread_data_on_block_idx[I2], - m_thread_data_on_block_idx[I3], - m_thread_data_on_block_idx[I4], - m_thread_data_on_block_idx[I5], - n_thread_data_on_block_idx[I3]), - ck::tensor_operation::element_wise::PassThrough{}}; - - using EDataType = CDataType; - - const auto ds_grid_desc_m_n = MakeDsGridDescriptor_M_N( - problem.M, problem.MPadded, problem.N, problem.NPadded, problem.StrideDs); - - const auto ds_grid_desc_mblock_mperblock_nblock_nperblock = - MakeDsGridDescriptor_MBlock_MPerBlock_NBlock_NPerBlock( - ds_grid_desc_m_n, problem.MBlock, problem.NBlock); - - const auto ds_grid_buf = generate_tuple( - [&](auto i) { - return make_dynamic_buffer( - p_ds_grid[i], ds_grid_desc_m_n[i].GetElementSpaceSize()); - }, - Number{}); - - // tuple of reference to C/Ds tensor descriptors - const auto c_ds_desc_refs = concat_tuple_of_reference( - tie(c_shuffle_block_desc_mblock_mperblock_nblock_nperblock), - generate_tie([&](auto i) -> const auto& // return type should be reference - { return ds_grid_desc_mblock_mperblock_nblock_nperblock[i]; }, - Number{})); - - // tuple of reference to C/Ds tensor descriptors - const auto c_ds_buf_refs = concat_tuple_of_reference( - tie(c_shuffle_block_buf), - generate_tie([&](auto i) -> const auto& // return type should be reference - { return ds_grid_buf[i]; }, - Number{})); - - // tuple of starting index of C/Ds blockwise copy - const auto idx_c_ds_block_begin = - container_concat(make_tuple(make_multi_index(0, 0, 0, 0)), - generate_tuple( - [&](auto) { - return make_multi_index(block_m_id, 0, block_n_id, 0); - // return make_multi_index(block_work_idx[I0], 0, - // block_work_idx[I1], 0); - }, - Number{})); - - const auto e_grid_desc_mblock_mperblock_nblock_nperblock = - c_grid_desc_mblock_mperblock_nblock_nperblock; - - using CDEBlockTransferCluster = - CShuffleBlockTransferClusterLengths_MBlock_MPerBlock_NBlock_NPerBlock; - const auto EGlobalMemoryDataOperation = CGlobalMemoryDataOperation; - constexpr index_t scatter_weight_idx = 3; // hack fix felix - auto cde_block_copy_lds_and_global = ThreadGroupTensorSliceTransfer_v7r3_scatter< - ThisThreadBlock, - decltype(container_concat(make_tuple(CShuffleDataType{}), DsDataType{})), - Tuple, - decltype(c_ds_desc_refs), - decltype(tie(e_grid_desc_mblock_mperblock_nblock_nperblock)), - CElementwiseOperation, - Sequence(EGlobalMemoryDataOperation)>, // FIXME: make - // Sequence support - // arbitray type - Sequence<1, - CShuffleMXdlPerWavePerShuffle * MWave * MPerXdl, - 1, - CShuffleNXdlPerWavePerShuffle * NWave * NPerXdl>, // BlockSliceLengths, - CDEBlockTransferCluster, - Sequence<0, 1, 2, 3>, // typename ThreadClusterArrangeOrder, - Sequence<0, 1, 2, 3>, // typename SrcDimAccessOrder, - Sequence<0, 1, 2, 3>, // typename DstDimAccessOrder, - 3, // index_t SrcVectorDim, - 3, // index_t DstVectorDim, - CDEShuffleBlockTransferScalarPerVectors, - CShuffleBlockTransferScalarPerVector_NPerBlock, - sequence_merge_t< - Sequence, - uniform_sequence_gen_t>, // ThreadTransferSrcResetCoordinateAfterRunFlags - Sequence, // ThreadTransferDstResetCoordinateAfterRunFlags - IndexType, - 1, // ScatterDim - true, // OutputScatter: false, only use scatter weights - scatter_weight_idx // ScatterWeightIdx: ascale - >{c_ds_desc_refs, - idx_c_ds_block_begin, - tie(e_grid_desc_mblock_mperblock_nblock_nperblock), - make_tuple(make_multi_index(0, 0, block_n_id, 0)), - c_element_op}; - - auto c_grid_buf = make_dynamic_buffer( - p_c_grid, c_grid_desc_mblock_mperblock_nblock_nperblock.GetElementSpaceSize()); - - constexpr auto sfc_c_vgpr = - SpaceFillingCurve, - Sequence<0, 1, 2, 3, 4, 5, 6, 7, 8, 9>, - Sequence>{}; - - constexpr index_t num_access = sfc_c_vgpr.GetNumOfAccess(); - - // space filling curve for shuffled blockwise C/D/E - constexpr auto sfc_cde_block = - SpaceFillingCurve, - Sequence<0, 2, 1, 3>, - Sequence<1, - CShuffleMXdlPerWavePerShuffle * MWave * MPerXdl, - 1, - CShuffleNXdlPerWavePerShuffle * NWave * NPerXdl>>{}; - - static_assert(num_access == sfc_cde_block.GetNumOfAccess(), "wrong!"); - constexpr auto EMThreads = - CDEBlockTransferCluster{}.At(I0) * CDEBlockTransferCluster{}.At(I1); - constexpr auto EMRepeats = CShuffleMXdlPerWavePerShuffle * MWave * MPerXdl / EMThreads; - constexpr auto ENThreads = - CDEBlockTransferCluster{}.At(I2) * CDEBlockTransferCluster{}.At(I3); - static_for<0, num_access, 1>{}([&](auto access_id) { - // make sure it's safe to write to LDS - StaticallyIndexedArray scatter_offsets; - - auto dstidx = sfc_cde_block.GetIndex(access_id); - const index_t c_token_pos = - block_m_id * MPerBlock + threadIdx.x / ENThreads * EMRepeats + dstidx(I1); - static_for<0, EMRepeats, 1>{}([&](auto m0) { - const index_t fused_token = p_sorted_token_ids[c_token_pos + m0]; - IndexType token_offset = fused_token & 0xffffff; - if constexpr(IsInputGemm) - { - token_offset = token_offset * problem.TopK + (fused_token >> 24); - } - scatter_offsets(m0) = static_cast(token_offset) * problem.N; - }); - - block_sync_lds(); - - // each thread write its data from VGPR to LDS - c_thread_copy_vgpr_to_lds.Run(c_thread_desc_m0_n0_m1_n1_m2_m3_m4_n2, - sfc_c_vgpr.GetIndexTupleOfNumber(access_id), - c_thread_buf_fp32, - c_block_desc_m0_n0_m1_n1_m2_m3_m4_n2, - c_shuffle_block_buf); - - // make sure it's safe to read from LDS - block_sync_lds(); - - // each block copy its data from LDS to global - cde_block_copy_lds_and_global.Run( - c_ds_desc_refs, - c_ds_buf_refs, - tie(e_grid_desc_mblock_mperblock_nblock_nperblock), - tie(c_grid_buf), - scatter_offsets); - - if constexpr(access_id < num_access - 1) - { - constexpr auto cde_lds_and_global_step = - sfc_cde_block.GetForwardStep(access_id); - - // move on Ds - static_for<0, NumDTensor, 1>{}([&](auto i) { - cde_block_copy_lds_and_global.MoveSrcSliceWindow( - c_ds_desc_refs, i + I1, cde_lds_and_global_step); - }); - - // move on E - cde_block_copy_lds_and_global.MoveDstSliceWindow( - tie(e_grid_desc_mblock_mperblock_nblock_nperblock), - I0, - cde_lds_and_global_step); - } - }); - } - } -#endif - template diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_moe_mx_gemm_bns.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_moe_mx_gemm_bns.hpp index 8559b78fe019..d428cb5e998c 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_moe_mx_gemm_bns.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_moe_mx_gemm_bns.hpp @@ -70,50 +70,6 @@ __launch_bounds__(GridwiseGemm::MaxBlockSize, MinimumOccupancy) #endif // end of if (defined(__gfx9__)) } -#if 0 -template -__global__ void -#if CK_USE_LAUNCH_BOUNDS -__launch_bounds__(GridwiseGemm::MaxBlockSize, MinimumOccupancy) -#endif - // __attribute__((amdgpu_waves_per_eu(1, 1))) - kernel_moe_mxgemm_2lds(typename GridwiseGemm::Argument karg) -{ -#if defined(__gfx9__) - if constexpr(GridwiseGemm::template IsValidCompilationParameter()) - { - __shared__ char p_shared[GridwiseGemm::GetSharedMemoryNumberOfByte()]; - __shared__ char p_shared1[GridwiseGemm::GetSharedMemoryNumberOfByte()]; - - // auto splitk_batch_offset = typename GridwiseGemm::SplitKBatchOffset(karg, blockIdx.z); - - GridwiseGemm::template Run_2Lds( - karg.p_sorted_token_ids, - karg.p_sorted_expert_ids, - karg.p_max_token_id, - karg.p_a_grid, - karg.p_a_scale_grid, - karg.p_b_grid, - karg.p_b_scale_grid, - karg.p_ds_grid, - karg.p_c_grid, - p_shared, - p_shared1, - karg, - karg.a_element_op, - karg.b_element_op, - karg.c_element_op); - } -#else - ignore = karg; -#endif // end of if (defined(__gfx9__)) -} -#endif - template Date: Thu, 9 Apr 2026 00:29:04 -0400 Subject: [PATCH 08/49] Remove dead #if 0 blocks in ck/utility Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../ck/utility/container_element_picker.hpp | 6 ------ .../include/ck/utility/dynamic_buffer.hpp | 6 ------ .../include/ck/utility/transpose_vectors.hpp | 17 ----------------- .../include/ck/utility/workgroup_barrier.hpp | 14 -------------- 4 files changed, 43 deletions(-) diff --git a/projects/composablekernel/include/ck/utility/container_element_picker.hpp b/projects/composablekernel/include/ck/utility/container_element_picker.hpp index 9de2466e7145..cec6c85298e1 100644 --- a/projects/composablekernel/include/ck/utility/container_element_picker.hpp +++ b/projects/composablekernel/include/ck/utility/container_element_picker.hpp @@ -15,9 +15,6 @@ template struct ContainerElementPicker { using type = ContainerElementPicker; -#if 0 - using data_type = typename Arr::data_type; -#endif __host__ __device__ constexpr ContainerElementPicker() = delete; @@ -81,9 +78,6 @@ template struct ConstantContainerElementPicker { using type = ConstantContainerElementPicker; -#if 0 - using data_type = typename Arr::data_type; -#endif __host__ __device__ constexpr ConstantContainerElementPicker() = delete; diff --git a/projects/composablekernel/include/ck/utility/dynamic_buffer.hpp b/projects/composablekernel/include/ck/utility/dynamic_buffer.hpp index 00fab270e818..ce4c92425e9a 100644 --- a/projects/composablekernel/include/ck/utility/dynamic_buffer.hpp +++ b/projects/composablekernel/include/ck/utility/dynamic_buffer.hpp @@ -361,14 +361,8 @@ struct DynamicBuffer { if(is_valid_element) { -#if 0 - X tmp = x; - - __builtin_memcpy(&(p_data_[i]), &tmp, sizeof(X)); -#else // if(i >= 2169041600) *c_style_pointer_cast(&p_data_[i]) = x; -#endif } } } diff --git a/projects/composablekernel/include/ck/utility/transpose_vectors.hpp b/projects/composablekernel/include/ck/utility/transpose_vectors.hpp index de20674ef2f8..11b503da699f 100644 --- a/projects/composablekernel/include/ck/utility/transpose_vectors.hpp +++ b/projects/composablekernel/include/ck/utility/transpose_vectors.hpp @@ -18,22 +18,6 @@ struct transpose_vectors; // transpose fp16 2x2 __device__ void transpose_fp16_2x2(const half2_t& x0, const half2_t& x1, half2_t& y0, half2_t& y1) { -#if 0 - static constexpr auto I0 = Number<0>{}; - static constexpr auto I1 = Number<1>{}; - - const vector_type vx0{x0}, vx1{x1}; - vector_type vy0, vy1; - - vy0.template AsType()(I0) = vx0.template AsType()[I0]; - vy0.template AsType()(I1) = vx1.template AsType()[I0]; - - vy1.template AsType()(I0) = vx0.template AsType()[I1]; - vy1.template AsType()(I1) = vx1.template AsType()[I1]; - - y0 = vy0.template AsType()[I0]; - y1 = vy1.template AsType()[I0]; -#else constexpr int32_t m0 = 0x05040100; constexpr int32_t m1 = 0x07060302; @@ -43,7 +27,6 @@ __device__ void transpose_fp16_2x2(const half2_t& x0, const half2_t& x1, half2_t // index is reversed because of little endianness (least significant bits first) y0 = bit_cast(__builtin_amdgcn_perm(bit_cast(x1), bit_cast(x0), m0)); y1 = bit_cast(__builtin_amdgcn_perm(bit_cast(x1), bit_cast(x0), m1)); -#endif } template diff --git a/projects/composablekernel/include/ck/utility/workgroup_barrier.hpp b/projects/composablekernel/include/ck/utility/workgroup_barrier.hpp index 0e440799be17..0be341da88de 100644 --- a/projects/composablekernel/include/ck/utility/workgroup_barrier.hpp +++ b/projects/composablekernel/include/ck/utility/workgroup_barrier.hpp @@ -12,20 +12,6 @@ struct workgroup_barrier __device__ uint32_t ld(uint32_t offset) { -#if 0 - float d = llvm_amdgcn_raw_buffer_load_fp32( - amdgcn_make_buffer_resource(base_ptr), - 0, - offset, - AMDGCN_BUFFER_GLC); - union cvt { - float f32; - uint32_t u32; - }; - cvt x; - x.f32 = d; - return x.u32; -#endif return __atomic_load_n(base_ptr + offset, __ATOMIC_RELAXED); } From 9ab192f3d7697bac102978f132452180525979c3 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:29:05 -0400 Subject: [PATCH 09/49] Remove dead #if 0 blocks in ck_tile/core/arch Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../ck_tile/core/arch/amd_buffer_addressing.hpp | 16 ---------------- .../core/arch/amd_buffer_addressing_builtins.hpp | 16 ---------------- 2 files changed, 32 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/core/arch/amd_buffer_addressing.hpp b/projects/composablekernel/include/ck_tile/core/arch/amd_buffer_addressing.hpp index a32f26dadf7f..6a9c9e3fafbe 100644 --- a/projects/composablekernel/include/ck_tile/core/arch/amd_buffer_addressing.hpp +++ b/projects/composablekernel/include/ck_tile/core/arch/amd_buffer_addressing.hpp @@ -2166,27 +2166,11 @@ CK_TILE_DEVICE void amd_buffer_store_impl(const thread_buffer src_thread_d } else if constexpr(N == 8) { -#if 0 - thread_buffer tmp{src_thread_data}; - - llvm_amdgcn_raw_buffer_store_fp16x4(tmp.template get_as()[number<0>{}], - dst_wave_buffer_resource, - dst_thread_addr_offset, - dst_wave_addr_offset, - static_cast(coherence)); - - llvm_amdgcn_raw_buffer_store_fp16x4(tmp.template get_as()[number<1>{}], - dst_wave_buffer_resource, - dst_thread_addr_offset, - dst_wave_addr_offset + 4 * sizeof(fp16_t), - static_cast(coherence)); -#else llvm_amdgcn_raw_buffer_store_fp32x4(bit_cast(src_thread_data), dst_wave_buffer_resource, dst_thread_addr_offset, dst_wave_addr_offset, static_cast(coherence)); -#endif } } else if constexpr(std::is_same::value) // bf16 diff --git a/projects/composablekernel/include/ck_tile/core/arch/amd_buffer_addressing_builtins.hpp b/projects/composablekernel/include/ck_tile/core/arch/amd_buffer_addressing_builtins.hpp index 7d57858f266d..8056b76af7a2 100644 --- a/projects/composablekernel/include/ck_tile/core/arch/amd_buffer_addressing_builtins.hpp +++ b/projects/composablekernel/include/ck_tile/core/arch/amd_buffer_addressing_builtins.hpp @@ -1992,27 +1992,11 @@ CK_TILE_DEVICE void amd_buffer_store_impl(const thread_buffer src_thread_d } else if constexpr(N == 8) { -#if 0 - thread_buffer tmp{src_thread_data}; - - llvm_amdgcn_raw_buffer_store_fp16x4(tmp.template get_as()[number<0>{}], - dst_wave_buffer_resource, - dst_thread_addr_offset, - dst_wave_addr_offset, - static_cast(coherence)); - - llvm_amdgcn_raw_buffer_store_fp16x4(tmp.template get_as()[number<1>{}], - dst_wave_buffer_resource, - dst_thread_addr_offset, - dst_wave_addr_offset + 4 * sizeof(fp16_t), - static_cast(coherence)); -#else llvm_amdgcn_raw_buffer_store_fp32x4(bit_cast(src_thread_data), dst_wave_buffer_resource, dst_thread_addr_offset, dst_wave_addr_offset, static_cast(coherence)); -#endif } } else if constexpr(std::is_same::value) // bf16 From 8dc68440db7ad7b7dd9d804900b7c1e1599b099d Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:29:37 -0400 Subject: [PATCH 10/49] Remove dead #if 0 blocks in ck_tile/core/container Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../ck_tile/core/container/sequence.hpp | 26 ---------------- .../container/statically_indexed_array.hpp | 14 --------- .../ck_tile/core/container/thread_buffer.hpp | 31 ------------------- 3 files changed, 71 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/core/container/sequence.hpp b/projects/composablekernel/include/ck_tile/core/container/sequence.hpp index 73ce09b20e84..4e94d6e9023b 100644 --- a/projects/composablekernel/include/ck_tile/core/container/sequence.hpp +++ b/projects/composablekernel/include/ck_tile/core/container/sequence.hpp @@ -480,32 +480,6 @@ struct sequence_split using right_type = decltype(Seq::extract(range1{})); }; -#if 0 -// reverse sequence -template -struct sequence_reverse -{ - static constexpr index_t NSize = Seq{}.size(); - - using seq_split = sequence_split; - using type = typename sequence_merge< - typename sequence_reverse::type, - typename sequence_reverse::type>::type; -}; - -template -struct sequence_reverse> -{ - using type = sequence; -}; - -template -struct sequence_reverse> -{ - using type = sequence; -}; -#endif - namespace detail { template struct seq_reverse; diff --git a/projects/composablekernel/include/ck_tile/core/container/statically_indexed_array.hpp b/projects/composablekernel/include/ck_tile/core/container/statically_indexed_array.hpp index d35934ab045e..111b8a8c58c7 100644 --- a/projects/composablekernel/include/ck_tile/core/container/statically_indexed_array.hpp +++ b/projects/composablekernel/include/ck_tile/core/container/statically_indexed_array.hpp @@ -24,18 +24,4 @@ using statically_indexed_array = array; #endif // consider always use ck_tile::array for this purpose -#if 0 -template -CK_TILE_HOST_DEVICE constexpr auto make_statically_indexed_array(const X& x, const Xs&... xs) -{ - return statically_indexed_array(x, static_cast(xs)...); -} - -// make empty statically_indexed_array -template -CK_TILE_HOST_DEVICE constexpr auto make_statically_indexed_array() -{ - return statically_indexed_array(); -} -#endif } // namespace ck_tile diff --git a/projects/composablekernel/include/ck_tile/core/container/thread_buffer.hpp b/projects/composablekernel/include/ck_tile/core/container/thread_buffer.hpp index a955b7f84f1d..58e417a612d1 100644 --- a/projects/composablekernel/include/ck_tile/core/container/thread_buffer.hpp +++ b/projects/composablekernel/include/ck_tile/core/container/thread_buffer.hpp @@ -23,18 +23,6 @@ CK_TILE_HOST_DEVICE constexpr auto make_thread_buffer(Ts&&... ts) } #else -#if 0 -template -using thread_buffer = array; - -template -CK_TILE_HOST_DEVICE constexpr auto make_thread_buffer(Ts&&... ts) -{ - return make_array(ts...); -} - -#endif - // clang-format off template struct thread_buffer { @@ -103,25 +91,6 @@ struct thread_buffer { return vx.data; } -#if 0 - template ::value, bool>::type = false> - CK_TILE_HOST_DEVICE constexpr void _set_as(number is, X_ x) - { - using X = remove_cvref_t; - - constexpr index_t kSPerX = vector_traits::vector_size; - - union { - X_ data; - tuple_array sub_data; - } vx {x}; - - static_for<0, kSPerX, 1>{}( - [&](auto j) { operator()((is * number{}) + j) = vx.sub_data[j]; }); - } -#endif #define TB_COMMON_AS() \ From 7e6d7a24775ed7878637f783017b17f9ae28b2a4 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:30:12 -0400 Subject: [PATCH 11/49] Remove dead #if 0 blocks in ck_tile/core/numeric Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../include/ck_tile/core/numeric/half.hpp | 87 ------------------- .../include/ck_tile/core/numeric/int8.hpp | 21 ----- 2 files changed, 108 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/core/numeric/half.hpp b/projects/composablekernel/include/ck_tile/core/numeric/half.hpp index b6a7e86d3caf..c172f48cadba 100644 --- a/projects/composablekernel/include/ck_tile/core/numeric/half.hpp +++ b/projects/composablekernel/include/ck_tile/core/numeric/half.hpp @@ -264,93 +264,6 @@ bool operator>(const half_t& x, const half_t& y) { return __hgt(x.to_fp16(), y.t CK_TILE_DEVICE bool operator>=(const half_t& x, const half_t& y) { return __hge(x.to_fp16(), y.to_fp16()); } -#if 0 -CK_TILE_DEVICE -half_t operator+(const half_t& x, const half_t& y) -{ - return half_t(__hadd(x.to_fp16(), y.to_fp16())); -} - -CK_TILE_DEVICE -half_t operator-(const half_t& x) { return half_t(__hneg(x.to_fp16())); } - -CK_TILE_DEVICE -half_t operator-(const half_t& x, const half_t& y) -{ - return half_t(__hsub(x.to_fp16(), y.to_fp16())); -} - -CK_TILE_DEVICE -half_t operator*(const half_t& x, const half_t& y) -{ - return half_t(__hmul(x.to_fp16(), y.to_fp16())); -} - -CK_TILE_DEVICE -half_t operator/(const half_t& x, const half_t& y) -{ - return half_t(__hdiv(x.to_fp16(), y.to_fp16())); -} - -CK_TILE_DEVICE -half_t& operator+=(half_t& x, const half_t& y) -{ - x = half_t(__hadd(x.to_fp16(), y.to_fp16())); - return x; -} - -CK_TILE_DEVICE -half_t& operator-=(half_t& x, const half_t& y) -{ - x = half_t(__hsub(x.to_fp16(), y.to_fp16())); - return x; -} - -CK_TILE_DEVICE -half_t& operator*=(half_t& x, const half_t& y) -{ - x = half_t(__hmul(x.to_fp16(), y.to_fp16())); - return x; -} - -CK_TILE_DEVICE -half_t& operator/=(half_t& x, const half_t& y) -{ - x = half_t(__hdiv(x.to_fp16(), y.to_fp16())); - return x; -} - -CK_TILE_DEVICE -half_t& operator++(half_t& x) -{ - x = half_t(__hadd(x.to_fp16(), half_t(1.0f).to_fp16())); - return x; -} - -CK_TILE_DEVICE -half_t& operator--(half_t& x) -{ - x = half_t(__hsub(x.to_fp16(), half_t(1.0f).to_fp16())); - return x; -} - -CK_TILE_DEVICE -half_t operator++(half_t& x, int) -{ - half_t y(x); - x = half_t(__hadd(x.to_fp16(), half_t(1.0f).to_fp16())); - return y; -} - -CK_TILE_DEVICE -half_t operator--(half_t& x, int) -{ - half_t y(x); - x = half_t(__hsub(x.to_fp16(), half_t(1.0f).to_fp16())); - return y; -} -#endif - #if CK_TILE_USE_CUSTOM_DATA_TYPE CK_TILE_ARITHMETIC_USING_FLOAT(CK_TILE_HOST, half_t) #endif diff --git a/projects/composablekernel/include/ck_tile/core/numeric/int8.hpp b/projects/composablekernel/include/ck_tile/core/numeric/int8.hpp index aa9f820c17b1..7b0f102f2bf6 100644 --- a/projects/composablekernel/include/ck_tile/core/numeric/int8.hpp +++ b/projects/composablekernel/include/ck_tile/core/numeric/int8.hpp @@ -73,27 +73,6 @@ struct numeric CK_TILE_HOST_DEVICE static constexpr int8_t zero() { return 0; } }; -#if 0 - -template <> -struct numeric_traits -{ - static constexpr int exp = 5; - static constexpr int mant = 10; - static constexpr int bias = 15; - static constexpr uint16_t nan_mask = 0x7C00; - static constexpr uint16_t head_mask = 0xFC00; - static constexpr uint16_t mant_mask = 0x3FF; - static constexpr uint16_t exp_mask = 0x1F; - static constexpr uint32_t Inf = 0x7C00; - static constexpr uint32_t NegInf = 0xFC00; - static constexpr uint32_t NaN = 0x7C01; - static constexpr uint32_t Neg0 = 0x8000; - static constexpr int PackedSize = 1; - using bitwise_type = uint16_t; -}; -#endif - CK_TILE_HOST_DEVICE constexpr float int8_to_float(const int8_t& x) { return static_cast(x); } From 72170f7435d0e3b0cfd66dabe3299a4f2df5a7c5 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:30:55 -0400 Subject: [PATCH 12/49] Remove dead #if 0 blocks in ck_tile/core/tensor Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../ck_tile/core/tensor/tile_distribution.hpp | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/core/tensor/tile_distribution.hpp b/projects/composablekernel/include/ck_tile/core/tensor/tile_distribution.hpp index aa5714e5c24d..beafc0311575 100644 --- a/projects/composablekernel/include/ck_tile/core/tensor/tile_distribution.hpp +++ b/projects/composablekernel/include/ck_tile/core/tensor/tile_distribution.hpp @@ -454,45 +454,6 @@ struct tile_distribution_detail } // namespace detail -#if 0 -// this returns a constexpr tile_distribution -template -CK_TILE_HOST_DEVICE constexpr auto make_tile_distribution(StaticTileDistributionEncoding_) -{ - using DstrEncode = remove_cvref_t; - - constexpr auto adaptor_impl = - detail::make_adaptor_encoding_for_tile_distribution(StaticTileDistributionEncoding_{}); - - constexpr auto ps_ys_to_xs_adaptor_impl = adaptor_impl.template at<0>(); - constexpr auto ys_to_d_adaptor_impl = adaptor_impl.template at<1>(); - constexpr index_t d_length = adaptor_impl.template at<2>(); - constexpr auto rh_major_minor_to_hidden_ids_impl = adaptor_impl.template at<3>(); - - constexpr auto ps_ys_to_xs_adaptor = - CONSTRUCT_TENSOR_ADAPTOR_FROM_ENCODING(ps_ys_to_xs_adaptor_impl); - - constexpr auto ys_to_d_adaptor = CONSTRUCT_TENSOR_ADAPTOR_FROM_ENCODING(ys_to_d_adaptor_impl); - - constexpr auto ys_to_d_descriptor = - make_tensor_descriptor_from_adaptor(ys_to_d_adaptor, d_length); - - // - constexpr index_t ndim_rh_major = DstrEncode::detail::ndim_rh_major_; - constexpr auto ndims_rhs_minor = DstrEncode::detail::ndims_rhs_minor_; - - constexpr auto rh_major_minor_to_hidden_ids = - TO_TUPLE_OF_SEQUENCE(rh_major_minor_to_hidden_ids_impl, ndim_rh_major, ndims_rhs_minor); - - return tile_distribution< - remove_cvref_t, - remove_cvref_t, - remove_cvref_t, - detail::tile_distribution_detail>>{ - ps_ys_to_xs_adaptor, ys_to_d_descriptor}; -} -#endif - // this returns a static tile_distribution template CK_TILE_HOST_DEVICE constexpr auto make_static_tile_distribution(StaticTileDistributionEncoding_) From b723106f7ce09c8d7a247268e4708ae2a33327e9 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:31:43 -0400 Subject: [PATCH 13/49] Remove dead #if 0 blocks in ck_tile/ops/elementwise Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../unary_element_wise_operation.hpp | 63 ------------------- 1 file changed, 63 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/ops/elementwise/unary_element_wise_operation.hpp b/projects/composablekernel/include/ck_tile/ops/elementwise/unary_element_wise_operation.hpp index 4ad699629c02..4e971649d00c 100644 --- a/projects/composablekernel/include/ck_tile/ops/elementwise/unary_element_wise_operation.hpp +++ b/projects/composablekernel/include/ck_tile/ops/elementwise/unary_element_wise_operation.hpp @@ -745,14 +745,6 @@ struct PassThroughPack2 template CK_TILE_HOST_DEVICE void operator()(Y& y, const X& x) const; -#if 0 - CK_TILE_HOST_DEVICE constexpr void operator()(ck_tile::fp16x2_t& y, const ck_tile::f8x2_t& x) const - { - auto t = type_convert(x); - y = type_convert(t); - } -#endif - CK_TILE_HOST_DEVICE constexpr void operator()(fp16x2_t& y, const pk_int4_t& x) const { uint8_t x_u8 = bit_cast(x); @@ -871,61 +863,6 @@ struct UnaryConvert } }; -#if 0 -struct ConvertBF16RTN -{ - // convert to bf16 using round to nearest (rtn) - template - CK_TILE_HOST_DEVICE void operator()(Y& y, const X& x) const - { - // check Y datatype - static_assert(std::is_same_v, "Data type is not supported by this operation!"); - - // check X datatype - static_assert(std::is_same_v || std::is_same_v, - "Data type is not supported by this operation!"); - - y = bf16_convert_rtn(x); - } -}; - -struct ConvertF8SR -{ - // convert to fp8 using stochastic rounding (SR) - template - CK_TILE_HOST_DEVICE void operator()(Y& y, const X& x) const - { - // check Y datatype - static_assert(std::is_same_v || std::is_same_v, - "Data type is not supported by this operation!"); - - // check X datatype - static_assert(std::is_same_v || std::is_same_v, - "Data type is not supported by this operation!"); - - y = f8_convert_sr(x); - } -}; - -struct ConvertF8RNE -{ - // convert to fp8 using rounding to nearest even - template - CK_TILE_HOST_DEVICE void operator()(Y& y, const X& x) const - { - // check Y datatype - static_assert(std::is_same_v || std::is_same_v, - "Data type is not supported by this operation!"); - - // check X datatype - static_assert(std::is_same_v || std::is_same_v, - "Data type is not supported by this operation!"); - - y = f8_convert_rne(x); - } -}; -#endif - struct Scale { static constexpr const char* name = "Scale"; From ec903a12e813a804ac16d65229c03167ab20127f Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:33:07 -0400 Subject: [PATCH 14/49] Remove dead #if 0 blocks in ck_tile/ops/fused_moe/kernel Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../fused_moe/kernel/moe_sorting_kernel.hpp | 109 ------------------ 1 file changed, 109 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/ops/fused_moe/kernel/moe_sorting_kernel.hpp b/projects/composablekernel/include/ck_tile/ops/fused_moe/kernel/moe_sorting_kernel.hpp index 52b2b8657435..313372775657 100644 --- a/projects/composablekernel/include/ck_tile/ops/fused_moe/kernel/moe_sorting_kernel.hpp +++ b/projects/composablekernel/include/ck_tile/ops/fused_moe/kernel/moe_sorting_kernel.hpp @@ -3047,53 +3047,6 @@ struct MoeSortingMultiPhaseKernel_P23 x_r = x_v; #endif { -#if 0 -#pragma unroll - for(int j = 0; j < index_pack / 2; j++) - { - int i_token = i * kBlockSize * index_pack + threadIdx.x + j * kBlockSize; - index_t x = x_d[j]; - int i_topk = x - 1; // topk of this token - int i_show = x != 0 ? 1 : 0; // has this token or not - int cumsum = i_show; - impl::moe_sorting_wave_cumsum(cumsum); - - __syncthreads(); - if(lane_id == get_warp_size() - 1) - { - s[4 + wave_id] = cumsum; - } - __syncthreads(); - - // reduce cross wave - static_for<0, kBlockSize / get_warp_size() - 1, 1>{}([&](auto i_w) { - IndexType prev = s[4 + i_w]; - prev = wave_id > i_w ? prev : 0; // mask out - cumsum += prev; - }); - cumsum += prev_cumsum; // add previous round cumsum - if(threadIdx.x == kBlockSize - 1) - { - s[0] = cumsum; - } - __syncthreads(); - - int position = cumsum - i_show; - prev_cumsum = s[0]; // update the last cumsum - - if(i_show) - { -#if CK_TILE_REFERENCE_MOE_SORTING_MOCK_ID - p_sorted_token_ids[e_start + position] = - MOE_SORTING_MOCK_ID(i_token, i_topk); -#else - p_sorted_token_ids[e_start + position] = i_token; -#endif - p_sorted_weights[e_start + position] = - p_weights[i_token * kargs.topk_mdiv.divisor + i_topk]; - } - } -#endif { d_t i_topk; d_t i_show; @@ -3151,68 +3104,6 @@ struct MoeSortingMultiPhaseKernel_P23 } position += i_show[j]; }); - -#if 0 - int i_token = i * kBlockSize * index_pack + threadIdx.x * 2 + j * kBlockSize * 2; - index_t x = x_d[j]; - index_t x0 = static_cast(x & 0xffff); - index_t x1 = static_cast(x >> 16); - int i_topk_0 = x0 - 1; // topk of this token - int i_show_0 = x0 != 0 ? 1 : 0; // has this token or not - int i_topk_1 = x1 - 1; // topk of this token - int i_show_1 = x1 != 0 ? 1 : 0; // has this token or not - int cumsum = i_show_0 + i_show_1; - impl::moe_sorting_wave_cumsum(cumsum); - - __syncthreads(); - if(lane_id == get_warp_size() - 1) - { - s[4 + wave_id] = cumsum; - } - __syncthreads(); - - // reduce cross wave - static_for<0, kBlockSize / get_warp_size() - 1, 1>{}([&](auto i_w) { - IndexType prev = s[4 + i_w]; - prev = wave_id > i_w ? prev : 0; // mask out - cumsum += prev; - }); - cumsum += prev_cumsum; // add previous round cumsum - if(threadIdx.x == kBlockSize - 1) - { - s[0] = cumsum; - } - __syncthreads(); - - int position_0 = cumsum - i_show_0 - i_show_1; - prev_cumsum = s[0]; // update the last cumsum - - if(i_show_0) - { -#if CK_TILE_REFERENCE_MOE_SORTING_MOCK_ID - p_sorted_token_ids[e_start + position_0] = - MOE_SORTING_MOCK_ID(i_token, i_topk_0); -#else - p_sorted_token_ids[e_start + position_0] = i_token; -#endif - p_sorted_weights[e_start + position_0] = - p_weights[i_token * kargs.topk_mdiv.divisor + i_topk_0]; - } - - int position_1 = cumsum - i_show_1; - - if(i_show_1) - { -#if CK_TILE_REFERENCE_MOE_SORTING_MOCK_ID - p_sorted_token_ids[e_start + position_1] = - MOE_SORTING_MOCK_ID(i_token + 1, i_topk_1); -#else - p_sorted_token_ids[e_start + position_1] = i_token + 1; -#endif - p_sorted_weights[e_start + position_1] = - p_weights[(i_token + 1) * kargs.topk_mdiv.divisor + i_topk_1]; - } -#endif } } } From a7825101e9f6452961d52bcc9ebf3c107ce6223f Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:33:53 -0400 Subject: [PATCH 15/49] Remove dead #if 0 blocks in ck_tile/ops/gemm/block Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- ...gemm_areg_bsmem_creg_v1_default_policy.hpp | 23 ------------------- ...gemm_areg_bsmem_creg_v2_default_policy.hpp | 23 ------------------- ...gemm_asmem_breg_creg_v1_default_policy.hpp | 23 ------------------- 3 files changed, 69 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_v1_default_policy.hpp b/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_v1_default_policy.hpp index 2280f6f875c1..3a7c0362f757 100644 --- a/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_v1_default_policy.hpp +++ b/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_v1_default_policy.hpp @@ -19,30 +19,7 @@ struct BlockGemmARegBSmemCRegV1DefaultPolicy std::is_same_v && std::is_same_v) { -#if 0 - constexpr index_t kBlockSize = Problem::kBlockSize; - - constexpr index_t kMPerBlock = Problem::BlockGemmShape::kM; - constexpr index_t kNPerBlock = Problem::BlockGemmShape::kN; - constexpr index_t kKPerBlock = Problem::BlockGemmShape::kK; - - static_assert(kBlockSize % get_warp_size() == 0, "wrong!"); - - constexpr index_t NumWarp = kBlockSize / get_warp_size(); - - // FIXME - if constexpr(NumWarp == 4 && kMPerBlock % 128 == 0 && - kNPerBlock % 128 == 0 % kKPerBlock % 16 == 0) - { - return make_tuple(WarpGemmMfmaF16F16F32M32N32K8{}, 4, 1); - } - else - { - return make_tuple(WarpGemmMfmaF16F16F32M32N32K8{}, 4, 1); - } -#else return make_tuple(WarpGemmMfmaF16F16F32M32N32K8TransposedCDistribution{}, 4, 1); -#endif } else if constexpr(std::is_same_v && std::is_same_v && diff --git a/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_v2_default_policy.hpp b/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_v2_default_policy.hpp index b8290c95d895..0b1cea942587 100644 --- a/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_v2_default_policy.hpp +++ b/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_v2_default_policy.hpp @@ -16,30 +16,7 @@ struct BlockGemmARegBSmemCRegV2DefaultPolicy CK_TILE_HOST_DEVICE static constexpr auto GetWarpGemmMWarpNWarp() { -#if 0 - constexpr index_t kBlockSize = Problem::kBlockSize; - - constexpr index_t kMPerBlock = Problem::BlockGemmShape::kM; - constexpr index_t kNPerBlock = Problem::BlockGemmShape::kN; - constexpr index_t kKPerBlock = Problem::BlockGemmShape::kK; - - static_assert(kBlockSize % get_warp_size() == 0, "wrong!"); - - constexpr index_t NumWarp = kBlockSize / get_warp_size(); - - // FIXME - if constexpr(NumWarp == 4 && kMPerBlock % 128 == 0 && - kNPerBlock % 128 == 0 % kKPerBlock % 16 == 0) - { - return make_tuple(WarpGemmMfmaF16F16F32M32N32K8{}, 4, 1); - } - else - { - return make_tuple(WarpGemmMfmaF16F16F32M32N32K8{}, 4, 1); - } -#else return make_tuple(WarpGemmMfmaF16F16F32M32N32K8TransposedCDistribution{}, 4, 1); -#endif } }; diff --git a/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_asmem_breg_creg_v1_default_policy.hpp b/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_asmem_breg_creg_v1_default_policy.hpp index 29022e764f65..0622cc624ff4 100644 --- a/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_asmem_breg_creg_v1_default_policy.hpp +++ b/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_asmem_breg_creg_v1_default_policy.hpp @@ -19,30 +19,7 @@ struct BlockGemmASmemBRegCRegV1DefaultPolicy std::is_same_v && std::is_same_v) { -#if 0 - constexpr index_t kBlockSize = Problem::kBlockSize; - - constexpr index_t kMPerBlock = Problem::BlockGemmShape::kM; - constexpr index_t kNPerBlock = Problem::BlockGemmShape::kN; - constexpr index_t kKPerBlock = Problem::BlockGemmShape::kK; - - static_assert(kBlockSize % get_warp_size() == 0, "wrong!"); - - constexpr index_t NumWarp = kBlockSize / get_warp_size(); - - // FIXME - if constexpr(NumWarp == 4 && kMPerBlock % 128 == 0 && - kNPerBlock % 128 == 0 % kKPerBlock % 16 == 0) - { - return make_tuple(WarpGemmMfmaF16F16F32M32N32K8{}, 4, 1); - } - else - { - return make_tuple(WarpGemmMfmaF16F16F32M32N32K8{}, 4, 1); - } -#else return make_tuple(WarpGemmMfmaF16F16F32M32N32K8TransposedCDistribution{}, 4, 1); -#endif } else if constexpr(std::is_same_v && std::is_same_v && From 91ae169748a3dc2bc136d7f7f33dd81cfee013a7 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:34:31 -0400 Subject: [PATCH 16/49] Remove dead #if 0 blocks in ck_tile/ops/norm_reduce/block Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../ops/norm_reduce/block/block_norm_reduce.hpp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/ops/norm_reduce/block/block_norm_reduce.hpp b/projects/composablekernel/include/ck_tile/ops/norm_reduce/block/block_norm_reduce.hpp index da9c5c4d57b0..a7f17155a861 100644 --- a/projects/composablekernel/include/ck_tile/ops/norm_reduce/block/block_norm_reduce.hpp +++ b/projects/composablekernel/include/ck_tile/ops/norm_reduce/block/block_norm_reduce.hpp @@ -360,17 +360,6 @@ struct BlockNormReduceCrossWarpSync template CK_TILE_DEVICE constexpr index_t block_tile_welford_calculate_max_count(int row_size) { -#if 0 - using S = BlockShape; - index_t LastloopN = row_size % S::Block_N == 0 ? S::Block_N : row_size % S::Block_N; - constexpr index_t NThread = S::WarpPerBlock_N * S::ThreadPerWarp_N; - index_t iNLane = get_thread_id() % NThread; - index_t iN0 = LastloopN / (S::Vector_N * S::ThreadPerWarp_N); - index_t iN1 = (LastloopN % (S::Vector_N * S::ThreadPerWarp_N)) / S::Vector_N; - index_t N2 = (LastloopN % (S::Vector_N * S::ThreadPerWarp_N)) % S::Vector_N; - index_t iN3 = iNLane < iN1 ? S::Vector_N : iNLane == iN1 ? N2 : 0; - return iN0 * S::Vector_N + iN3; -#endif using S_ = BlockShape; constexpr index_t ThreadsPerBlock_N = S_::WarpPerBlock_N * S_::ThreadPerWarp_N; From 03ac154d02ab4f31d2054c7cad9f61962a893ea8 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:35:25 -0400 Subject: [PATCH 17/49] Remove dead #if 0 blocks in ck_tile/ops/reduce/block Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../ops/reduce/block/block_reduce2d.hpp | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/ops/reduce/block/block_reduce2d.hpp b/projects/composablekernel/include/ck_tile/ops/reduce/block/block_reduce2d.hpp index abad5ed03179..8cf207c3901c 100644 --- a/projects/composablekernel/include/ck_tile/ops/reduce/block/block_reduce2d.hpp +++ b/projects/composablekernel/include/ck_tile/ops/reduce/block/block_reduce2d.hpp @@ -140,28 +140,6 @@ struct BlockReduce2d ReducePacksPerXDim{}); } -#if 0 - constexpr auto I0 = number<0>{}; - constexpr auto I1 = number<1>{}; - constexpr auto spans = XDistributedTensor_::get_distributed_spans(); - - // FIXME: hard coded to reduce 2nd axis - sweep_tile_span(spans[I0], [&](auto dstr_idx_i0) { - constexpr auto y_dstr_idx = make_tuple(dstr_idx_i0); - - auto y = y_tensor[y_dstr_idx]; - - sweep_tile_span(spans[I1], [&](auto dstr_idx_i1) { - constexpr auto in_dstr_idx = make_tuple(dstr_idx_i0, dstr_idx_i1); - const auto x = ck_tile::type_convert(x_tensor[in_dstr_idx]); - - y = reduce_func(y, x); - }); - - y_tensor(y_dstr_idx) = y; - }); -#endif - template CK_TILE_DEVICE static auto MakeYBlockTile() { From 854d33b5d7bcd5f27927f7c92c39ab7384afd303 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:36:11 -0400 Subject: [PATCH 18/49] Remove dead #if 0 blocks in lib/ck/lib/tensor_operation_instance/gpu Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../gpu/gemm_streamk.hpp | 60 ------------------- 1 file changed, 60 deletions(-) diff --git a/projects/composablekernel/library/include/ck/library/tensor_operation_instance/gpu/gemm_streamk.hpp b/projects/composablekernel/library/include/ck/library/tensor_operation_instance/gpu/gemm_streamk.hpp index f46ebbacf715..348216129ffb 100644 --- a/projects/composablekernel/library/include/ck/library/tensor_operation_instance/gpu/gemm_streamk.hpp +++ b/projects/composablekernel/library/include/ck/library/tensor_operation_instance/gpu/gemm_streamk.hpp @@ -52,66 +52,6 @@ struct DeviceOperationInstanceFactory> op_ptrs; -#if 0 - if constexpr(is_same_v && is_same_v && - is_same_v) - { - if constexpr(is_same_v && is_same_v && - is_same_v) - { - add_device_gemm_xdl_splitk_f32_f32_f32_mk_kn_mn_instances(op_ptrs); - } - else if constexpr(is_same_v && is_same_v && - is_same_v) - { - add_device_gemm_xdl_splitk_f32_f32_f32_mk_nk_mn_instances(op_ptrs); - } - else if constexpr(is_same_v && is_same_v && - is_same_v) - { - add_device_gemm_xdl_splitk_f32_f32_f32_km_kn_mn_instances(op_ptrs); - } - else if constexpr(is_same_v && is_same_v && - is_same_v) - { - add_device_gemm_xdl_splitk_f32_f32_f32_km_nk_mn_instances(op_ptrs); - } - } - else if constexpr(is_same_v && is_same_v && - is_same_v) - { - if constexpr(is_same_v && is_same_v && - is_same_v) - { - add_device_gemm_xdl_splitk_f16_f16_f16_mk_kn_mn_v1_instances(op_ptrs); - add_device_gemm_xdl_splitk_f16_f16_f16_mk_kn_mn_v1_irregular_instances(op_ptrs); - add_device_gemm_xdl_splitk_f16_f16_f16_mk_kn_mn_v1_interwave_instances(op_ptrs); - add_device_gemm_xdl_splitk_f16_f16_f16_mk_kn_mn_v1_interwave_irregular_instances(op_ptrs); - add_device_gemm_xdl_splitk_f16_f16_f16_mk_kn_mn_v2_instances(op_ptrs); - add_device_gemm_xdl_splitk_f16_f16_f16_mk_kn_mn_v2_irregular_instances(op_ptrs); - } - else if constexpr(is_same_v && is_same_v && - is_same_v) - { - add_device_gemm_xdl_splitk_f16_f16_f16_mk_nk_mn_v1_instances(op_ptrs); - add_device_gemm_xdl_splitk_f16_f16_f16_mk_nk_mn_v1_irregular__instances(op_ptrs); - add_device_gemm_xdl_splitk_f16_f16_f16_mk_nk_mn_v1_interwave_instances(op_ptrs); - add_device_gemm_xdl_splitk_f16_f16_f16_mk_nk_mn_v1_interwave_irregular_instances(op_ptrs); - add_device_gemm_xdl_splitk_f16_f16_f16_mk_nk_mn_v2_instances(op_ptrs); - add_device_gemm_xdl_splitk_f16_f16_f16_mk_nk_mn_v2_irregular_instances(op_ptrs); - } - else if constexpr(is_same_v && is_same_v && - is_same_v) - { - add_device_gemm_xdl_splitk_f16_f16_f16_km_kn_mn_instances(op_ptrs); - } - else if constexpr(is_same_v && is_same_v && - is_same_v) - { - add_device_gemm_xdl_splitk_f16_f16_f16_km_nk_mn_instances(op_ptrs); - } - } -#endif if constexpr(is_same_v && is_same_v && is_same_v) { From 090a9ad9828256e3aa4a3aa3600f623384122896 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:36:12 -0400 Subject: [PATCH 19/49] Remove dead #if 0 blocks in lib/src/tensor_operation_instance/gpu/gemm_b_scale/device_gemm_b_scale_xdl_f16_i4_f16 Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../device_gemm_b_scale_xdl_f16_i4_f16_mk_nk_mn.hpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_b_scale/device_gemm_b_scale_xdl_f16_i4_f16/device_gemm_b_scale_xdl_f16_i4_f16_mk_nk_mn.hpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_b_scale/device_gemm_b_scale_xdl_f16_i4_f16/device_gemm_b_scale_xdl_f16_i4_f16_mk_nk_mn.hpp index 3d391ae931b7..a8d69afb9aad 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_b_scale/device_gemm_b_scale_xdl_f16_i4_f16/device_gemm_b_scale_xdl_f16_i4_f16_mk_nk_mn.hpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_b_scale/device_gemm_b_scale_xdl_f16_i4_f16/device_gemm_b_scale_xdl_f16_i4_f16_mk_nk_mn.hpp @@ -33,12 +33,6 @@ static constexpr auto GemmMNKPadding = GemmSpecialization::MNKPadding; static constexpr auto Intrawave = BlockGemmPipelineScheduler::Intrawave; static constexpr auto Interwave = BlockGemmPipelineScheduler::Interwave; -#if 0 -template -using device_gemm_xdl_b_scale_f16_i4_f16_mk_nk_mn_comp_instances = std::tuple< - -#endif - template using device_gemm_b_scale_xdl_f16_i4_f16_mk_nk_mn_mem_instances = std::tuple< // clang-format off From 1985d50504eb444b5def4f8f93f3e9372e3b0121 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:36:13 -0400 Subject: [PATCH 20/49] Remove dead #if 0 blocks in lib/src/tensor_operation_instance/gpu/gemm_universal/device_gemm_xdl_universal_bf16_i4_bf16 Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- ...mm_xdl_universal_bf16_i4_bf16_mk_nk_mn.hpp | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_universal/device_gemm_xdl_universal_bf16_i4_bf16/device_gemm_xdl_universal_bf16_i4_bf16_mk_nk_mn.hpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_universal/device_gemm_xdl_universal_bf16_i4_bf16/device_gemm_xdl_universal_bf16_i4_bf16_mk_nk_mn.hpp index 8ba6c485cbf7..99e809f0ecc1 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_universal/device_gemm_xdl_universal_bf16_i4_bf16/device_gemm_xdl_universal_bf16_i4_bf16_mk_nk_mn.hpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_universal/device_gemm_xdl_universal_bf16_i4_bf16/device_gemm_xdl_universal_bf16_i4_bf16_mk_nk_mn.hpp @@ -33,25 +33,6 @@ static constexpr auto GemmMNKPadding = GemmSpecialization::MNKPadding; static constexpr auto Intrawave = BlockGemmPipelineScheduler::Intrawave; static constexpr auto Interwave = BlockGemmPipelineScheduler::Interwave; -#if 0 -template -using device_gemm_xdl_universal_bf16_i4_bf16_mk_nk_mn_comp_instances = std::tuple< - // clang-format off - //#########################| ALayout| BLayout| CLayout|AData| BData| CData| AccData| Cshuffle| A| B| C| GEMM| Block| MPer| NPer| KPer| AK1| BK1|MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransferClusterLengths| CBlockTransfer| Block-wiseGemm| Block-wiseGemm| - //#########################| | | | Type| Type| Type| Type| Type| Elementwise| Elementwise| Elementwise|Specialization| Size| Block| Block| Block| | | XDL| XDL| Per| Per| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraN| MXdlPerWave| NXdlPerWave| _MBlock_MXdlPerWave_MWaveMPerXdl| ScalarPerVector| Pipeline| Pipeline| - //#########################| | | | | | | | | Operation| Operation| Operation| | | | | | | | | | Wave| Wave| Lengths_K0_M_K1| ArrangeOrder| | | PerVector| PerVector_K1| | Lengths_K0_N_K1| ArrangeOrder| | | PerVector| PerVector_K1| | PerShuffle| PerShuffle| _NBlock_NXdlPerWave_NWaveNPerXdl| _NWaveNPerXdl| Scheduler| Verision| - //#########################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | - - // Compute friendly - DeviceGemm_Xdl_CShuffleV3< Row, Col, Row, BF16, I4, BF16, F32, BF16, PassThrough, PassThrough, PassThrough, GemmSpec, 256, 128, 128, 64, 8, 16, 32, 32, 2, 2, S<8, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 0, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, 0, 1, 1, S<1, 32, 1, 8>, 8, BlockGemmPipelineScheduler::Intrawave, BlockGemmPipelineVersion::v4>, - DeviceGemm_Xdl_CShuffleV3< Row, Col, Row, BF16, I4, BF16, F32, BF16, PassThrough, PassThrough, PassThrough, GemmSpec, 256, 224, 256, 64, 8, 16, 16, 16, 7, 8, S<8, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 0, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, 0, 1, 2, S<1, 32, 1, 8>, 8, BlockGemmPipelineScheduler::Intrawave, BlockGemmPipelineVersion::v3>, - DeviceGemm_Xdl_CShuffleV3< Row, Col, Row, BF16, I4, BF16, F32, BF16, PassThrough, PassThrough, PassThrough, GemmSpec, 256, 128, 128, 64, 8, 16, 32, 32, 2, 2, S<8, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 0, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, 0, 1, 1, S<1, 32, 1, 8>, 8, BlockGemmPipelineScheduler::Intrawave, BlockGemmPipelineVersion::v3>, - DeviceGemm_Xdl_CShuffleV3< Row, Col, Row, BF16, I4, BF16, F32, BF16, PassThrough, PassThrough, PassThrough, GemmSpec, 256, 128, 128, 64, 8, 16, 32, 32, 2, 2, S<8, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 0, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, 0, 1, 1, S<1, 32, 1, 8>, 8, BlockGemmPipelineScheduler::Intrawave, BlockGemmPipelineVersion::v5>, - DeviceGemm_Xdl_CShuffleV3< Row, Col, Row, BF16, I4, BF16, F32, BF16, PassThrough, PassThrough, PassThrough, GemmSpec, 256, 128, 128, 64, 8, 16, 32, 32, 2, 2, S<8, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 0, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, 0, 1, 1, S<1, 32, 1, 8>, 8, BlockGemmPipelineScheduler::Interwave, BlockGemmPipelineVersion::v1> - // clang-format on - >; -#endif - template using device_gemm_xdl_universal_bf16_i4_bf16_mk_nk_mn_mem_instances = std::tuple< // clang-format off From 7a405acd5ebcbec590fff9d422d94edae19f864b Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:36:14 -0400 Subject: [PATCH 21/49] Remove dead #if 0 blocks in lib/src/tensor_operation_instance/gpu/gemm_universal/device_gemm_xdl_universal_f16_i4_f16 Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- ...gemm_xdl_universal_f16_i4_f16_mk_nk_mn.hpp | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_universal/device_gemm_xdl_universal_f16_i4_f16/device_gemm_xdl_universal_f16_i4_f16_mk_nk_mn.hpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_universal/device_gemm_xdl_universal_f16_i4_f16/device_gemm_xdl_universal_f16_i4_f16_mk_nk_mn.hpp index 088378b9181c..c52b9723a9a9 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_universal/device_gemm_xdl_universal_f16_i4_f16/device_gemm_xdl_universal_f16_i4_f16_mk_nk_mn.hpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_universal/device_gemm_xdl_universal_f16_i4_f16/device_gemm_xdl_universal_f16_i4_f16_mk_nk_mn.hpp @@ -33,25 +33,6 @@ static constexpr auto GemmMNKPadding = GemmSpecialization::MNKPadding; static constexpr auto Intrawave = BlockGemmPipelineScheduler::Intrawave; static constexpr auto Interwave = BlockGemmPipelineScheduler::Interwave; -#if 0 -template -using device_gemm_xdl_universal_f16_i4_f16_mk_nk_mn_comp_instances = std::tuple< - // clang-format off - //#########################| ALayout| BLayout| CLayout|AData| BData| CData| AccData| Cshuffle| A| B| C| GEMM| Block| MPer| NPer| KPer| AK1| BK1|MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransferClusterLengths| CBlockTransfer| Block-wiseGemm| Block-wiseGemm| - //#########################| | | | Type| Type| Type| Type| Type| Elementwise| Elementwise| Elementwise|Specialization| Size| Block| Block| Block| | | XDL| XDL| Per| Per| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraM| ThreadCluster| ThreadCluster| SrcAccessOrder| SrcVectorDim| SrcScalar| DstScalar| AddExtraN| MXdlPerWave| NXdlPerWave| _MBlock_MXdlPerWave_MWaveMPerXdl| ScalarPerVector| Pipeline| Pipeline| - //#########################| | | | | | | | | Operation| Operation| Operation| | | | | | | | | | Wave| Wave| Lengths_K0_M_K1| ArrangeOrder| | | PerVector| PerVector_K1| | Lengths_K0_N_K1| ArrangeOrder| | | PerVector| PerVector_K1| | PerShuffle| PerShuffle| _NBlock_NXdlPerWave_NWaveNPerXdl| _NWaveNPerXdl| Scheduler| Verision| - //#########################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | - - // Compute friendly - DeviceGemm_Xdl_CShuffleV3< Row, Col, Row, F16, I4, F16, F32, F16, PassThrough, PassThrough, PassThrough, GemmSpec, 256, 128, 128, 64, 8, 16, 32, 32, 2, 2, S<8, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 0, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, 0, 1, 1, S<1, 32, 1, 8>, 8, BlockGemmPipelineScheduler::Intrawave, BlockGemmPipelineVersion::v4>, - DeviceGemm_Xdl_CShuffleV3< Row, Col, Row, F16, I4, F16, F32, F16, PassThrough, PassThrough, PassThrough, GemmSpec, 256, 224, 256, 64, 8, 16, 16, 16, 7, 8, S<8, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 0, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, 0, 1, 2, S<1, 32, 1, 8>, 8, BlockGemmPipelineScheduler::Intrawave, BlockGemmPipelineVersion::v3>, - DeviceGemm_Xdl_CShuffleV3< Row, Col, Row, F16, I4, F16, F32, F16, PassThrough, PassThrough, PassThrough, GemmSpec, 256, 128, 128, 64, 8, 16, 32, 32, 2, 2, S<8, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 0, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, 0, 1, 1, S<1, 32, 1, 8>, 8, BlockGemmPipelineScheduler::Intrawave, BlockGemmPipelineVersion::v3>, - DeviceGemm_Xdl_CShuffleV3< Row, Col, Row, F16, I4, F16, F32, F16, PassThrough, PassThrough, PassThrough, GemmSpec, 256, 128, 128, 64, 8, 16, 32, 32, 2, 2, S<8, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 0, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, 0, 1, 1, S<1, 32, 1, 8>, 8, BlockGemmPipelineScheduler::Intrawave, BlockGemmPipelineVersion::v5>, - DeviceGemm_Xdl_CShuffleV3< Row, Col, Row, F16, I4, F16, F32, F16, PassThrough, PassThrough, PassThrough, GemmSpec, 256, 128, 128, 64, 8, 16, 32, 32, 2, 2, S<8, 32, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 8, 8, 0, S<4, 64, 1>, S<1, 0, 2>, S<1, 0, 2>, 2, 16, 16, 0, 1, 1, S<1, 32, 1, 8>, 8, BlockGemmPipelineScheduler::Interwave, BlockGemmPipelineVersion::v1> - // clang-format on - >; -#endif - template using device_gemm_xdl_universal_f16_i4_f16_mk_nk_mn_mem_instances = std::tuple< From 760fa0e50efcdfbf771cbf5c20bd40418e17b0c9 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:36:15 -0400 Subject: [PATCH 22/49] Remove dead #if 0 blocks in test/ck_tile/add_rmsnorm2d_rdquant/instances Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../add_rmsnorm2d_rdquant_fwd_bf16_n1024_instance.cpp | 8 -------- .../add_rmsnorm2d_rdquant_fwd_fp16_n1024_instance.cpp | 8 -------- 2 files changed, 16 deletions(-) diff --git a/projects/composablekernel/test/ck_tile/add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_bf16_n1024_instance.cpp b/projects/composablekernel/test/ck_tile/add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_bf16_n1024_instance.cpp index 8f4813a47eb7..ca491148446f 100644 --- a/projects/composablekernel/test/ck_tile/add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_bf16_n1024_instance.cpp +++ b/projects/composablekernel/test/ck_tile/add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_bf16_n1024_instance.cpp @@ -5,14 +5,6 @@ // clang-format off // rm rn tm tn vn pd x 3p -#if 0 -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); - -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -#endif template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); diff --git a/projects/composablekernel/test/ck_tile/add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_fp16_n1024_instance.cpp b/projects/composablekernel/test/ck_tile/add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_fp16_n1024_instance.cpp index e357d7e3acff..f754d8e959f9 100644 --- a/projects/composablekernel/test/ck_tile/add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_fp16_n1024_instance.cpp +++ b/projects/composablekernel/test/ck_tile/add_rmsnorm2d_rdquant/instances/add_rmsnorm2d_rdquant_fwd_fp16_n1024_instance.cpp @@ -5,14 +5,6 @@ // clang-format off // rm rn tm tn vn pd x 3p -#if 0 -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); - -template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); -#endif template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); template float add_rmsnorm2d_rdquant_fwd_>(const S&, A); From 9a425fec44e559c7ab4596be7fe241157e39c1d6 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:36:16 -0400 Subject: [PATCH 23/49] Remove dead #if 0 blocks in test/ck_tile/moe_smoothquant/instances Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../instances/moe_smoothquant_bf16_n1024_instance.cpp | 8 -------- .../instances/moe_smoothquant_fp16_n1024_instance.cpp | 8 -------- 2 files changed, 16 deletions(-) diff --git a/projects/composablekernel/test/ck_tile/moe_smoothquant/instances/moe_smoothquant_bf16_n1024_instance.cpp b/projects/composablekernel/test/ck_tile/moe_smoothquant/instances/moe_smoothquant_bf16_n1024_instance.cpp index 8c72b81dc13b..56fcca3beb81 100644 --- a/projects/composablekernel/test/ck_tile/moe_smoothquant/instances/moe_smoothquant_bf16_n1024_instance.cpp +++ b/projects/composablekernel/test/ck_tile/moe_smoothquant/instances/moe_smoothquant_bf16_n1024_instance.cpp @@ -5,14 +5,6 @@ // clang-format off // rm rn tm tn vn pd 2p -#if 0 -template float moe_smoothquant_>(const S&, A); -template float moe_smoothquant_>(const S&, A); -template float moe_smoothquant_>(const S&, A); -template float moe_smoothquant_>(const S&, A); - -template float moe_smoothquant_>(const S&, A); -#endif template float moe_smoothquant_>(const S&, A); template float moe_smoothquant_>(const S&, A); diff --git a/projects/composablekernel/test/ck_tile/moe_smoothquant/instances/moe_smoothquant_fp16_n1024_instance.cpp b/projects/composablekernel/test/ck_tile/moe_smoothquant/instances/moe_smoothquant_fp16_n1024_instance.cpp index 6d7a5e7c1f59..2462cd218eb5 100644 --- a/projects/composablekernel/test/ck_tile/moe_smoothquant/instances/moe_smoothquant_fp16_n1024_instance.cpp +++ b/projects/composablekernel/test/ck_tile/moe_smoothquant/instances/moe_smoothquant_fp16_n1024_instance.cpp @@ -5,14 +5,6 @@ // clang-format off // rm rn tm tn vn pd 2p -#if 0 -template float moe_smoothquant_>(const S&, A); -template float moe_smoothquant_>(const S&, A); -template float moe_smoothquant_>(const S&, A); -template float moe_smoothquant_>(const S&, A); - -template float moe_smoothquant_>(const S&, A); -#endif template float moe_smoothquant_>(const S&, A); template float moe_smoothquant_>(const S&, A); From 8e65a1935d085e6cbe6febed9809d1a71a03bbdc Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 00:36:16 -0400 Subject: [PATCH 24/49] Remove dead #if 0 blocks in test/ck_tile/smoothquant/instances Remove obsolete #if 0 disabled code blocks identified by automated dead code scan with expert triage. --- .../instances/smoothquant_bf16_n1024_instance.cpp | 8 -------- .../instances/smoothquant_fp16_n1024_instance.cpp | 8 -------- 2 files changed, 16 deletions(-) diff --git a/projects/composablekernel/test/ck_tile/smoothquant/instances/smoothquant_bf16_n1024_instance.cpp b/projects/composablekernel/test/ck_tile/smoothquant/instances/smoothquant_bf16_n1024_instance.cpp index 8a5e0c74a0f9..66f427247a12 100644 --- a/projects/composablekernel/test/ck_tile/smoothquant/instances/smoothquant_bf16_n1024_instance.cpp +++ b/projects/composablekernel/test/ck_tile/smoothquant/instances/smoothquant_bf16_n1024_instance.cpp @@ -5,14 +5,6 @@ // clang-format off // rm rn tm tn vn pd 2p -#if 0 -template float smoothquant_>(const S&, A); -template float smoothquant_>(const S&, A); -template float smoothquant_>(const S&, A); -template float smoothquant_>(const S&, A); - -template float smoothquant_>(const S&, A); -#endif template float smoothquant_>(const S&, A); template float smoothquant_>(const S&, A); diff --git a/projects/composablekernel/test/ck_tile/smoothquant/instances/smoothquant_fp16_n1024_instance.cpp b/projects/composablekernel/test/ck_tile/smoothquant/instances/smoothquant_fp16_n1024_instance.cpp index 9c08cf64f091..103f7281b02a 100644 --- a/projects/composablekernel/test/ck_tile/smoothquant/instances/smoothquant_fp16_n1024_instance.cpp +++ b/projects/composablekernel/test/ck_tile/smoothquant/instances/smoothquant_fp16_n1024_instance.cpp @@ -5,14 +5,6 @@ // clang-format off // rm rn tm tn vn pd 2p -#if 0 -template float smoothquant_>(const S&, A); -template float smoothquant_>(const S&, A); -template float smoothquant_>(const S&, A); -template float smoothquant_>(const S&, A); - -template float smoothquant_>(const S&, A); -#endif template float smoothquant_>(const S&, A); template float smoothquant_>(const S&, A); From 26040a8d9065f4590a082e1ddc51a329f57d3a94 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:30:25 -0400 Subject: [PATCH 25/49] Remove commented-out dead code in codegen/test Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../codegen/test/grouped_conv_fwd_multiple_d_v1.cpp | 4 ---- .../codegen/test/grouped_conv_fwd_multiple_d_v2.cpp | 4 ---- .../codegen/test/grouped_conv_fwd_multiple_d_v3.cpp | 4 ---- .../codegen/test/grouped_conv_fwd_multiple_d_v4.cpp | 4 ---- 4 files changed, 16 deletions(-) diff --git a/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v1.cpp b/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v1.cpp index 863501cd0aa3..9895ed7e5449 100644 --- a/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v1.cpp +++ b/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v1.cpp @@ -198,10 +198,6 @@ struct Epilogue input_left_pads, input_right_pads); - // auto res = rtc::from_gpu(out_dev); - // pass &= ck::utils::check_err(res, out_host, "Error: incorrect results!", 1e-5f, 1e-4f); - // assert(pass); - // Simple check: this checks that the output from each instance matches the output from the // first instance CHECK(report(solution, check(rtc::from_gpu(out_dev)))); diff --git a/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v2.cpp b/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v2.cpp index e748a2974340..617c2318d5eb 100644 --- a/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v2.cpp +++ b/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v2.cpp @@ -198,10 +198,6 @@ struct Epilogue input_left_pads, input_right_pads); - // auto res = rtc::from_gpu(out_dev); - // pass &= ck::utils::check_err(res, out_host, "Error: incorrect results!", 1e-5f, 1e-4f); - // assert(pass); - // Simple check: this checks that the output from each instance matches the output from the // first instance CHECK(report(solution, check(rtc::from_gpu(out_dev)))); diff --git a/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v3.cpp b/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v3.cpp index a68fb53cbaef..84516b257705 100644 --- a/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v3.cpp +++ b/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v3.cpp @@ -198,10 +198,6 @@ struct Epilogue input_left_pads, input_right_pads); - // auto res = rtc::from_gpu(out_dev); - // pass &= ck::utils::check_err(res, out_host, "Error: incorrect results!", 1e-5f, 1e-4f); - // assert(pass); - // Simple check: this checks that the output from each instance matches the output from the // first instance CHECK(report(solution, check(rtc::from_gpu(out_dev)))); diff --git a/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v4.cpp b/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v4.cpp index 0262319c3966..3490c38f6af7 100644 --- a/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v4.cpp +++ b/projects/composablekernel/codegen/test/grouped_conv_fwd_multiple_d_v4.cpp @@ -198,10 +198,6 @@ struct Epilogue input_left_pads, input_right_pads); - // auto res = rtc::from_gpu(out_dev); - // pass &= ck::utils::check_err(res, out_host, "Error: incorrect results!", 1e-5f, 1e-4f); - // assert(pass); - // Simple check: this checks that the output from each instance matches the output from the // first instance CHECK(report(solution, check(rtc::from_gpu(out_dev)))); From 1f5cd77898ed6e0d960e25412843edb2dff61854 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:17 -0400 Subject: [PATCH 26/49] Remove commented-out dead code in example/65_gemm_multiply_multiply Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../65_gemm_multiply_multiply/moe_gemm2_xdl_fp8.cpp | 3 --- .../moe_gemm2_xdl_fp8_blockscale.cpp | 10 ---------- 2 files changed, 13 deletions(-) diff --git a/projects/composablekernel/example/65_gemm_multiply_multiply/moe_gemm2_xdl_fp8.cpp b/projects/composablekernel/example/65_gemm_multiply_multiply/moe_gemm2_xdl_fp8.cpp index 9f4cd13573eb..008edb533bd7 100644 --- a/projects/composablekernel/example/65_gemm_multiply_multiply/moe_gemm2_xdl_fp8.cpp +++ b/projects/composablekernel/example/65_gemm_multiply_multiply/moe_gemm2_xdl_fp8.cpp @@ -252,9 +252,6 @@ int main(int argc, char* argv[]) Tensor expert_ids(HostTensorDescriptor({sorted_tile_num}, {1})); Tensor sorted_token_ids(HostTensorDescriptor({sorted_size}, {1})); Tensor max_token_id(HostTensorDescriptor({1})); - // max_token_id.mData[0] = valid_size; - // max_token_id.mData = {valid_size, 0, 2, 3, 4, 6, 8, 10, 12, 13}; - // int eids[] = {0, 0, 1, 2, 3, 3, 4, 4, 5, 5, 6, 7, 7, 3, 3, 3}; max_token_id.mData = {valid_size, 0, 1, 2, 3, 4, 5, 6, 7, 8}; // int eids[] = {0, 1, 2, 3, 4, 5, 6, 7, 3, 3, 3}; // {2, 1, 1, 2, 2, 2, 1, 2} for(int i = 0; i < sorted_tile_num; i++) diff --git a/projects/composablekernel/example/65_gemm_multiply_multiply/moe_gemm2_xdl_fp8_blockscale.cpp b/projects/composablekernel/example/65_gemm_multiply_multiply/moe_gemm2_xdl_fp8_blockscale.cpp index 552d3cd7b586..8ae97ef1c201 100644 --- a/projects/composablekernel/example/65_gemm_multiply_multiply/moe_gemm2_xdl_fp8_blockscale.cpp +++ b/projects/composablekernel/example/65_gemm_multiply_multiply/moe_gemm2_xdl_fp8_blockscale.cpp @@ -261,16 +261,6 @@ int main(int argc, char* argv[]) Tensor max_token_id(HostTensorDescriptor({1})); max_token_id.mData = {valid_size, 0, 1, 2, 3, 4, 5, 6, 7, 8}; - // int eids[] = {0, 1, 3, 3, 3}; - // int eids[] = {0, 1, 2, 3, 4, 5, 6, 7}; //, 3, 3, 3}; // {2, 1, 1, 2, 2, 2, 1, 2} - // int eids[] = {0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 3, 3, 3}; - // int eids[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - // 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - // 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - // 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, - // 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, - // 7, 7, - // 3, 3, 3}; for(int i = 0; i < sorted_tile_num; i++) { expert_ids.mData[i] = i / ck::math::integer_divide_ceil(valid_tile_num, experts); From b171695d63bcce75c3251a1e073855e2bdf609dd Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:18 -0400 Subject: [PATCH 27/49] Remove commented-out dead code in ck/host_utility Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../include/ck/host_utility/flush_cache.hpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/projects/composablekernel/include/ck/host_utility/flush_cache.hpp b/projects/composablekernel/include/ck/host_utility/flush_cache.hpp index 2d051233e4a9..25084bae8511 100644 --- a/projects/composablekernel/include/ck/host_utility/flush_cache.hpp +++ b/projects/composablekernel/include/ck/host_utility/flush_cache.hpp @@ -476,16 +476,6 @@ float launch_and_time_kernel_with_preprocess(const StreamConfig& stream_config, hip_check_error(hipGetLastError()); // end real kernel - // hip_check_error(hipEventRecord(stop, stream_config.stream_id_)); - // hip_check_error(hipEventSynchronize(stop)); - // float cur_time = 0; - // hip_check_error(hipEventElapsedTime(&cur_time, start, stop)); - // #if MEDIAN - // times.insert(cur_time); - // #else - // total_time += cur_time; - // #endif - #if !defined(CK_USE_WMMA) if(ck::EnvIsEnabled(CK_ENV(CK_LOGGING))) { From 82e89567b4fec1c4e884410c2a4912e16b71b1d5 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:19 -0400 Subject: [PATCH 28/49] Remove commented-out dead code in ck/problem_transform Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- ...rd_convolution3d_into_gemm_v4r4r4_ndhwc_kzyxc_ndhwk.hpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/projects/composablekernel/include/ck/problem_transform/transform_forward_convolution3d_into_gemm_v4r4r4_ndhwc_kzyxc_ndhwk.hpp b/projects/composablekernel/include/ck/problem_transform/transform_forward_convolution3d_into_gemm_v4r4r4_ndhwc_kzyxc_ndhwk.hpp index fd50e61f3284..7ccebaf35a99 100644 --- a/projects/composablekernel/include/ck/problem_transform/transform_forward_convolution3d_into_gemm_v4r4r4_ndhwc_kzyxc_ndhwk.hpp +++ b/projects/composablekernel/include/ck/problem_transform/transform_forward_convolution3d_into_gemm_v4r4r4_ndhwc_kzyxc_ndhwk.hpp @@ -137,13 +137,6 @@ transform_forward_convolution3d_into_gemm_v4r4r4_ndhwc_kzyxc_ndhwk_pad( make_tuple(Sequence<0>{}, Sequence<1>{}), make_tuple(Sequence<0>{}, Sequence<1>{})); - // const auto out_grid_desc_gemmm_gemmn = transform_tensor_descriptor( - // out_n_do_ho_wo_k_grid_desc, - // make_tuple(make_merge_transform(make_tuple(N, Do, Ho, Wo)), - // make_pass_through_transform(K)), - // make_tuple(Sequence<0, 1, 2, 3>{}, Sequence<3>{}), - // make_tuple(Sequence<0>{}, Sequence<1>{})); - return make_tuple(in_grid_desc_gemmk0_gemmm_gemmk1, wei_grid_desc_gemmk0_gemmn_gemmk1, out_grid_desc_gemmm_gemmn); From 130d3a02b068b7b7c33c360b46c96778d1e18465 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:20 -0400 Subject: [PATCH 29/49] Remove commented-out dead code in ck/tensor_operation/gpu/block Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../thread_group_tensor_slice_transfer_direct_load.hpp | 6 ------ ...hread_group_tensor_slice_transfer_gather_direct_load.hpp | 6 ------ 2 files changed, 12 deletions(-) diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_direct_load.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_direct_load.hpp index a31c9101a161..ade283995013 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_direct_load.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_direct_load.hpp @@ -144,12 +144,6 @@ struct ThreadGroupTensorSliceTransfer_DirectLoad "When loading more than one element per thread at once, the contiguous " "dimension must be the same between source and destination."); - // constexpr auto dword_bytes = 4; - // constexpr auto bytes_per_thread_load = ScalarPerVector * sizeof(SrcData); - // static_assert(bytes_per_thread_load == dword_bytes, - // "Direct load transfer requires each thread to load exactly a single " - // "DWORD of data."); - static_assert(nDim == remove_cvref_t::GetNumOfDimension() && nDim == remove_cvref_t::GetNumOfDimension() && nDim == ThreadClusterLengths::Size(), diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_gather_direct_load.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_gather_direct_load.hpp index 11043281ec76..8c6e77bccd07 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_gather_direct_load.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/block/thread_group_tensor_slice_transfer_gather_direct_load.hpp @@ -152,12 +152,6 @@ struct ThreadGroupTensorSliceTransfer_Gather_DirectLoad "When loading more than one element per thread at once, the contiguous " "dimension must be the same between source and destination."); - // constexpr auto dword_bytes = 4; - // constexpr auto bytes_per_thread_load = ScalarPerVector * sizeof(SrcData); - // static_assert(bytes_per_thread_load == dword_bytes, - // "Direct load transfer requires each thread to load exactly a single " - // "DWORD of data."); - static_assert(nDim == remove_cvref_t::GetNumOfDimension() && nDim == remove_cvref_t::GetNumOfDimension() && nDim == ThreadClusterLengths::Size(), From 51eac1ef790429f22a25a03777068d25e7335dd9 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:21 -0400 Subject: [PATCH 30/49] Remove commented-out dead code in ck/tensor_operation/gpu/device/impl Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../device_batched_contraction_multiple_d_wmma_cshuffle.hpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_batched_contraction_multiple_d_wmma_cshuffle.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_batched_contraction_multiple_d_wmma_cshuffle.hpp index c64f2c42f320..69d8eef80a60 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_batched_contraction_multiple_d_wmma_cshuffle.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/device/impl/device_batched_contraction_multiple_d_wmma_cshuffle.hpp @@ -737,11 +737,6 @@ struct DeviceBatchedContractionMultipleD_Wmma_CShuffle // Batch Offset ComputePtrOffsetOfStridedBatch compute_ptr_offset_of_batch_; - - // for checking vector load/store - // index_t MRaw_; - // index_t NRaw_; - // index_t KRaw_; }; // Invoker From c9bc0ba96f172fa86265037ecc12483368389663 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:22 -0400 Subject: [PATCH 31/49] Remove commented-out dead code in ck/tensor_operation/gpu/grid Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../gridwise_gemm_multiple_d_xdl_cshuffle.hpp | 6 ------ .../gridwise_gemm_xdl_cshuffle_v3_multi_abd.hpp | 17 ----------------- 2 files changed, 23 deletions(-) diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_multiple_d_xdl_cshuffle.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_multiple_d_xdl_cshuffle.hpp index d66679a3188f..76f0b5a8932b 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_multiple_d_xdl_cshuffle.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_multiple_d_xdl_cshuffle.hpp @@ -464,12 +464,6 @@ struct GridwiseGemmMultipleD_xdl_cshuffle return false; } - // check block-to-E-tile - // if(!block_2_etile_map.CheckValidity(e_grid_desc_m_n)) - //{ - // return false; - //} - // TODO: also check validity of all components (blockwise-copy, threadwise-copy, etc) // check tensor size: cannot be larger than 2GB each constexpr long_index_t TwoGB = (long_index_t{1} << 31); diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_abd.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_abd.hpp index 58eed43be94f..529248093b10 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_abd.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/grid/gridwise_gemm_xdl_cshuffle_v3_multi_abd.hpp @@ -1077,10 +1077,6 @@ struct GridwiseGemm_xdl_cshuffle_v3 // BsGridPointer p_bs_grid; // DsGridPointer p_ds_grid; - // const auto a_grid_desc_ak0_m_ak1 = MakeAGridDescriptor_AK0_M_AK1( - // problem.M, problem.MPadded, problem.K, problem.KPadded, problem.StrideA, problem.AK0); - // const auto b_grid_desc_bk0_n_bk1 = MakeBGridDescriptor_BK0_N_BK1( - // problem.K, problem.KPadded, problem.N, problem.NPadded, problem.StrideB, problem.BK0); const auto as_grid_desc_ak0_m_ak1 = MakeAsGridDescriptor_AK0_M_AK1( problem.M, problem.MPadded, problem.K, problem.KPadded, problem.StrideAs, problem.AK0); const auto bs_grid_desc_bk0_n_bk1 = MakeBsGridDescriptor_BK0_N_BK1( @@ -1099,11 +1095,6 @@ struct GridwiseGemm_xdl_cshuffle_v3 MakeDsGridDescriptor_MBlock_MPerBlock_NBlock_NPerBlock( ds_grid_desc_m_n, problem.MBlock, problem.NBlock); - // const auto a_grid_buf = make_dynamic_buffer( - // p_a_grid, a_grid_desc_ak0_m_ak1.GetElementSpaceSize()); - // const auto b_grid_buf = make_dynamic_buffer( - // p_bs_grid[I0], b_grid_desc_bk0_n_bk1.GetElementSpaceSize()); - const auto as_grid_buf = generate_tuple( [&](auto i) { return make_dynamic_buffer( @@ -1347,10 +1338,6 @@ struct GridwiseGemm_xdl_cshuffle_v3 const BElementwiseOperation& b_element_op, const CElementwiseOperation& c_element_op) { - // const auto a_grid_desc_ak0_m_ak1 = MakeAGridDescriptor_AK0_M_AK1( - // problem.M, problem.MPadded, problem.K, problem.KPadded, problem.StrideA, problem.AK0); - // const auto b_grid_desc_bk0_n_bk1 = MakeBGridDescriptor_BK0_N_BK1( - // problem.K, problem.KPadded, problem.N, problem.NPadded, problem.StrideB, problem.BK0); const auto as_grid_desc_ak0_m_ak1 = MakeAsGridDescriptor_AK0_M_AK1( problem.M, problem.MPadded, problem.K, problem.KPadded, problem.StrideAs, problem.AK0); const auto bs_grid_desc_bk0_n_bk1 = MakeBsGridDescriptor_BK0_N_BK1( @@ -1369,10 +1356,6 @@ struct GridwiseGemm_xdl_cshuffle_v3 MakeDsGridDescriptor_MBlock_MPerBlock_NBlock_NPerBlock( ds_grid_desc_m_n, problem.MBlock, problem.NBlock); - // const auto a_grid_buf = make_dynamic_buffer( - // p_a_grid, a_grid_desc_ak0_m_ak1.GetElementSpaceSize()); - // const auto b_grid_buf = make_dynamic_buffer( - // p_b_grid, b_grid_desc_bk0_n_bk1.GetElementSpaceSize()); const auto as_grid_buf = generate_tuple( [&](auto i) { return make_dynamic_buffer( From e732fdb2e261641f83ada1611d8b7b70365794b6 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:23 -0400 Subject: [PATCH 32/49] Remove commented-out dead code in ck/tensor_operation/gpu/warp Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../ck/tensor_operation/gpu/warp/wmma_gemm.hpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/warp/wmma_gemm.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/warp/wmma_gemm.hpp index 1b60e6cb5314..09cf0ec87ba2 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/warp/wmma_gemm.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/warp/wmma_gemm.hpp @@ -303,14 +303,10 @@ struct wmma_type Date: Thu, 9 Apr 2026 01:31:24 -0400 Subject: [PATCH 33/49] Remove commented-out dead code in ck/tensor_operation/operator_transform Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../transform_contraction_to_gemm_arraybase.hpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/projects/composablekernel/include/ck/tensor_operation/operator_transform/transform_contraction_to_gemm_arraybase.hpp b/projects/composablekernel/include/ck/tensor_operation/operator_transform/transform_contraction_to_gemm_arraybase.hpp index 9fb81012ea8a..104e51de0cbb 100644 --- a/projects/composablekernel/include/ck/tensor_operation/operator_transform/transform_contraction_to_gemm_arraybase.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/operator_transform/transform_contraction_to_gemm_arraybase.hpp @@ -20,11 +20,6 @@ __host__ __device__ static auto MakeGridDescriptorPair(const std::array& gs_ms_ns_lengths_vec, const std::array& gs_ms_ns_strides_vec) { - // if(!(gs_ms_ns_lengths_vec.size() == NumDimG + NumDimM + NumDimN && - // gs_ms_ns_strides_vec.size() == NumDimG + NumDimM + NumDimN)) - // { - // throw std::runtime_error("wrong! dimension must match input lengths"); - // } const auto to_tuple = [&](auto& vec, auto start, auto end) { return generate_tuple([&](auto i) { return vec[start + i]; }, Number{}); From 7e784c5500c4dac89ac0276c27b33a4a5e1f32c8 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:25 -0400 Subject: [PATCH 34/49] Remove commented-out dead code in ck_tile/core/container Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../include/ck_tile/core/container/array.hpp | 20 ------------------- .../include/ck_tile/core/container/tuple.hpp | 10 ---------- 2 files changed, 30 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/core/container/array.hpp b/projects/composablekernel/include/ck_tile/core/container/array.hpp index 45adbded2cc3..d6ba1efcbea6 100644 --- a/projects/composablekernel/include/ck_tile/core/container/array.hpp +++ b/projects/composablekernel/include/ck_tile/core/container/array.hpp @@ -84,19 +84,6 @@ struct array data[i] = static_cast(c); } - // template - // CK_TILE_HOST_DEVICE constexpr array(const array& o) - // { - // // static_assert(ArrayType::size() == size(), "wrong! size not the same"); - // __content = o.__content; - // } - // CK_TILE_HOST_DEVICE constexpr array& operator=(const array& o) - // { - // // static_assert(ArrayType::size() == size(), "wrong! size not the same"); - // __content = o.__content; - // return *this; - // } - CK_TILE_HOST_DEVICE static constexpr auto size() { return N; } CK_TILE_HOST_DEVICE static constexpr bool is_static() { return is_static_v; } @@ -247,13 +234,6 @@ CK_TILE_HOST_DEVICE constexpr details::return_type make_array(Ts&&... return {std::forward(ts)...}; } -// // make empty array -// template -// CK_TILE_HOST_DEVICE constexpr auto make_array() -// { -// return array{}; -// } - // compatible with old ck's initializer, make an array and fill it withe the last element from // initializer_list template diff --git a/projects/composablekernel/include/ck_tile/core/container/tuple.hpp b/projects/composablekernel/include/ck_tile/core/container/tuple.hpp index 97d5ae10df08..d7da0e14670b 100644 --- a/projects/composablekernel/include/ck_tile/core/container/tuple.hpp +++ b/projects/composablekernel/include/ck_tile/core/container/tuple.hpp @@ -292,9 +292,6 @@ struct tuple : impl::tuple_base, T...> // below function should be used under tuple_array<> type, no extra check will perform here template CK_TILE_HOST_DEVICE constexpr decltype(auto) get_as() { return reinterpret_cast&>(*this); } template CK_TILE_HOST_DEVICE constexpr decltype(auto) get_as() const { return reinterpret_cast&>(*this); } - // below index is for index *AFTER* type convert, not before - //template CK_TILE_HOST_DEVICE constexpr decltype(auto) get_as(index_t i) { TP_COM_(); return reinterpret_cast&>(*this).at(i); } - //template CK_TILE_HOST_DEVICE constexpr decltype(auto) get_as(index_t i) const { TP_COM_(); return reinterpret_cast&>(*this).at(i); } template CK_TILE_HOST_DEVICE constexpr decltype(auto) get_as(number) { TP_COM_(); return reinterpret_cast&>(*this).at(number{}); } template CK_TILE_HOST_DEVICE constexpr decltype(auto) get_as(number) const { TP_COM_(); return reinterpret_cast&>(*this).at(number{}); } @@ -333,13 +330,6 @@ struct vector_traits, void> static constexpr index_t vector_size = sizeof...(T); }; -// template -// CK_TILE_HOST_DEVICE constexpr -// tuple -// make_tuple(T const&... t) -// { -// return {t...}; -// } template CK_TILE_HOST_DEVICE constexpr bool operator==(const tuple& a, const tuple& b) { From 27c2d3fb112e8ebfc6ecd783362ab57235d555e5 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:26 -0400 Subject: [PATCH 35/49] Remove commented-out dead code in ck_tile/core/tensor Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../include/ck_tile/core/tensor/sweep_tile.hpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/core/tensor/sweep_tile.hpp b/projects/composablekernel/include/ck_tile/core/tensor/sweep_tile.hpp index 1947ce028933..35440f10f8a1 100644 --- a/projects/composablekernel/include/ck_tile/core/tensor/sweep_tile.hpp +++ b/projects/composablekernel/include/ck_tile/core/tensor/sweep_tile.hpp @@ -295,10 +295,6 @@ struct tile_sweeper F f; }; -// partial deduction is not allowed -// template -// tile_sweeper(const F&, U = {})->tile_sweeper; - // deduction guide template Date: Thu, 9 Apr 2026 01:31:27 -0400 Subject: [PATCH 36/49] Remove commented-out dead code in ck_tile/ops/flatmm/kernel Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../ops/flatmm/kernel/grouped_flatmm_kernel.hpp | 10 ---------- .../ck_tile/ops/flatmm/kernel/moe_flatmm_kernel.hpp | 7 ------- 2 files changed, 17 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/ops/flatmm/kernel/grouped_flatmm_kernel.hpp b/projects/composablekernel/include/ck_tile/ops/flatmm/kernel/grouped_flatmm_kernel.hpp index ae331374599a..ff96139f1884 100644 --- a/projects/composablekernel/include/ck_tile/ops/flatmm/kernel/grouped_flatmm_kernel.hpp +++ b/projects/composablekernel/include/ck_tile/ops/flatmm/kernel/grouped_flatmm_kernel.hpp @@ -339,16 +339,6 @@ struct GroupedFlatmmKernel : FlatmmKernel, class ScaleN = FlatmmScalePointer<-1>, diff --git a/projects/composablekernel/include/ck_tile/ops/flatmm/kernel/moe_flatmm_kernel.hpp b/projects/composablekernel/include/ck_tile/ops/flatmm/kernel/moe_flatmm_kernel.hpp index 81cf76cb07cd..6721577018f5 100644 --- a/projects/composablekernel/include/ck_tile/ops/flatmm/kernel/moe_flatmm_kernel.hpp +++ b/projects/composablekernel/include/ck_tile/ops/flatmm/kernel/moe_flatmm_kernel.hpp @@ -483,13 +483,6 @@ struct MoeFlatmmKernel if constexpr(std::is_same_v) { - // if(kargs.N % TilePartitioner::NPerBlock != 0 && FlatmmPipeline::kPadN == false) - // { - // std::cerr << "Can't support N that is not a multiple of NPerBlock" - // " without padding!" - // << std::endl; - // return false; - // } if(kargs.N % FlatmmPipeline::GetVectorSizeB() != 0) { std::cerr << "N is not a multiple of vector load size for B tensor!" << std::endl; From de9e634da74b4bde1a3925e2bab59ad51baa73d1 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:28 -0400 Subject: [PATCH 37/49] Remove commented-out dead code in ck_tile/ops/flatmm/pipeline Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../flatmm_pipeline_agmem_bgmem_creg_v1_policy.hpp | 4 ---- ...xed_prec_flatmm_pipeline_agmem_bgmem_creg_v1.hpp | 9 --------- .../moe_flatmm_pipeline_agmem_bgmem_creg.hpp | 13 ++----------- .../mx_flatmm_pipeline_agmem_bgmem_creg_v1.hpp | 4 ---- 4 files changed, 2 insertions(+), 28 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/flatmm_pipeline_agmem_bgmem_creg_v1_policy.hpp b/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/flatmm_pipeline_agmem_bgmem_creg_v1_policy.hpp index 76d191a40c9d..99c35e9f30a9 100644 --- a/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/flatmm_pipeline_agmem_bgmem_creg_v1_policy.hpp +++ b/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/flatmm_pipeline_agmem_bgmem_creg_v1_policy.hpp @@ -392,10 +392,6 @@ struct UniversalFlatmmPipelineAgBgCrPolicy constexpr index_t M1 = BlockSize / get_warp_size(); static_assert(M2 != 0, "M2 is zero, which will lead to a division by zero error."); static_assert(M1 != 0, "M1 is zero, which will lead to a division by zero error."); - // constexpr index_t M0 = MPerBlock / (M2 * M1); - // static_assert(M0 * M1 * M2 == MPerBlock, - // "Incorrect M0, M2, M1 configuration! " - // "M0, M1, M2 must cover whole MPerBlock!"); return make_static_tile_distribution( tile_distribution_encoding, diff --git a/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/mixed_prec_flatmm_pipeline_agmem_bgmem_creg_v1.hpp b/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/mixed_prec_flatmm_pipeline_agmem_bgmem_creg_v1.hpp index 0f7f742fa073..6e6547b8372b 100644 --- a/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/mixed_prec_flatmm_pipeline_agmem_bgmem_creg_v1.hpp +++ b/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/mixed_prec_flatmm_pipeline_agmem_bgmem_creg_v1.hpp @@ -1151,11 +1151,6 @@ struct F16xMXF4FlatmmPipelineAGmemBGmemCRegV1 a_warp_tensor(number{}) = load_tile(a_warp_windows_pong(number{})(number{})); } - // barrier - // if constexpr((kIter == KIterPerWarp - 1) && (mIter == MIter_2nd_last)) - // { - // block_sync_lds(); - // } }); } }); @@ -1636,10 +1631,6 @@ struct F8xMXF4FlatmmPipelineAGmemBGmemCRegV1 ? Aload_rep : 0; } - // if((kIter % KPerScaleLoad == 0) && (mIter == 0)) - // { - // load_perM = load_perM + 1; - // } SchedulerPerM(dsread_perM, dswrite_perM, load_perM); } } diff --git a/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/moe_flatmm_pipeline_agmem_bgmem_creg.hpp b/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/moe_flatmm_pipeline_agmem_bgmem_creg.hpp index 543f4dc92a41..fd1bb6da5aa6 100644 --- a/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/moe_flatmm_pipeline_agmem_bgmem_creg.hpp +++ b/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/moe_flatmm_pipeline_agmem_bgmem_creg.hpp @@ -103,13 +103,8 @@ struct MoeFlatmmPipelineAGmemBGmemCRegV1 static constexpr index_t Aload_num_perK = dswrite_num_perK; static constexpr index_t Aload_rep = dswrite_rep; static constexpr index_t Bload_num_perK = kNPerBlock * WG::kK / NWarp / BK1 / WaveSize; - // static constexpr index_t ScaleBload_K1 = ContinuousScaleNPerThread * - // ContinuousScaleKPerThread; static constexpr index_t ScaleBload_num = - // kNPerBlock * kKPerBlock / NWarp / 32 / ScaleBload_K1 / - // WaveSize; // BlockN * BlockK / NWarp / ScalePerK / ScaleB_K1 / wavesize - // static constexpr index_t KPerScaleLoad = KIterPerWarp / ScaleBload_num; - static constexpr index_t HalfMIter = (MIterPerWarp + 1) / 2; - static constexpr index_t Bload_rep = (Bload_num_perK + HalfMIter - 1) / HalfMIter; + static constexpr index_t HalfMIter = (MIterPerWarp + 1) / 2; + static constexpr index_t Bload_rep = (Bload_num_perK + HalfMIter - 1) / HalfMIter; static constexpr index_t mfma_perM_perK = NIterPerWarp * mfma_per_wg; static constexpr index_t dswrite_mIter = (DsWritePreIssue - 1) % MIterPerWarp; @@ -352,10 +347,6 @@ struct MoeFlatmmPipelineAGmemBGmemCRegV1 ? Aload_rep : 0; } - // if((kIter % KPerScaleLoad == 0) && (mIter == 0)) - // { - // load_perM = load_perM + 1; - // } SchedulerPerM(dsread_perM, dswrite_perM, load_perM); } } diff --git a/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/mx_flatmm_pipeline_agmem_bgmem_creg_v1.hpp b/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/mx_flatmm_pipeline_agmem_bgmem_creg_v1.hpp index f698541dbf36..cef66e470fb3 100644 --- a/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/mx_flatmm_pipeline_agmem_bgmem_creg_v1.hpp +++ b/projects/composablekernel/include/ck_tile/ops/flatmm/pipeline/mx_flatmm_pipeline_agmem_bgmem_creg_v1.hpp @@ -390,10 +390,6 @@ struct MXFlatmmPipelineAGmemBGmemCRegV1 : FlatmmPipelineAGmemBGmemCRegV1 Date: Thu, 9 Apr 2026 01:31:29 -0400 Subject: [PATCH 38/49] Remove commented-out dead code in ck_tile/ops/fmha/pipeline Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../pipeline/block_fmha_pipeline_qx_ks_vs_custom_policy.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/ops/fmha/pipeline/block_fmha_pipeline_qx_ks_vs_custom_policy.hpp b/projects/composablekernel/include/ck_tile/ops/fmha/pipeline/block_fmha_pipeline_qx_ks_vs_custom_policy.hpp index 581bcc19d4b4..ec62e52d0ee9 100644 --- a/projects/composablekernel/include/ck_tile/ops/fmha/pipeline/block_fmha_pipeline_qx_ks_vs_custom_policy.hpp +++ b/projects/composablekernel/include/ck_tile/ops/fmha/pipeline/block_fmha_pipeline_qx_ks_vs_custom_policy.hpp @@ -692,9 +692,6 @@ struct BlockFmhaPipelineQXKSVSCustomPolicy : BlockFmhaPipelineQXCustomPolicy().get_element_space_size(); constexpr index_t BufferSize = GetSingleSmemElementSpaceSize(); // max(SingleKSize, SingleVSize); From 9f89989f675416b54890f12bc21007a32c2e899e Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:29 -0400 Subject: [PATCH 39/49] Remove commented-out dead code in ck_tile/ops/fused_moe/kernel Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../fused_moe/kernel/moe_sorting_kernel.hpp | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/ops/fused_moe/kernel/moe_sorting_kernel.hpp b/projects/composablekernel/include/ck_tile/ops/fused_moe/kernel/moe_sorting_kernel.hpp index 313372775657..06ab134f85bd 100644 --- a/projects/composablekernel/include/ck_tile/ops/fused_moe/kernel/moe_sorting_kernel.hpp +++ b/projects/composablekernel/include/ck_tile/ops/fused_moe/kernel/moe_sorting_kernel.hpp @@ -456,9 +456,6 @@ struct MoeSortingKernel template __device__ static constexpr T wave_reduce(T local, F reduce_f, number = {}) { - // constexpr int wave_size = 64; - // constexpr int reduce_stage = 6; // 1<<6=64 - // clang-format off constexpr int reduce_stage = [](){ if constexpr(wave_size_ == 2) return 1; else if constexpr(wave_size_ == 4) return 2; @@ -1206,17 +1203,21 @@ CK_TILE_HOST_DEVICE index_t moe_sorting_mp_sem_smem_size() template CK_TILE_DEVICE constexpr T moe_sorting_wave_reduce(T local, F reduce_f, number = {}) { - // constexpr int wave_size = 64; - // constexpr int reduce_stage = 6; // 1<<6=64 - // clang-format off - constexpr int reduce_stage = [](){ - if constexpr(wave_size_ == 2) return 1; - else if constexpr(wave_size_ == 4) return 2; - else if constexpr(wave_size_ == 8) return 3; - else if constexpr(wave_size_ == 16) return 4; - else if constexpr(wave_size_ == 32) return 5; - else if constexpr(wave_size_ == 64) return 6; - else return 0; + constexpr int reduce_stage = []() { + if constexpr(wave_size_ == 2) + return 1; + else if constexpr(wave_size_ == 4) + return 2; + else if constexpr(wave_size_ == 8) + return 3; + else if constexpr(wave_size_ == 16) + return 4; + else if constexpr(wave_size_ == 32) + return 5; + else if constexpr(wave_size_ == 64) + return 6; + else + return 0; }(); // clang-format on T v_local = local; From b47421cb4d469d438b79077398cd7220d3dcf004 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:30 -0400 Subject: [PATCH 40/49] Remove commented-out dead code in ck_tile/ops/fused_moe/pipeline Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../ops/fused_moe/pipeline/moe_sorting_pipeline.hpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/ops/fused_moe/pipeline/moe_sorting_pipeline.hpp b/projects/composablekernel/include/ck_tile/ops/fused_moe/pipeline/moe_sorting_pipeline.hpp index f70f4ddacc9b..828847091aca 100644 --- a/projects/composablekernel/include/ck_tile/ops/fused_moe/pipeline/moe_sorting_pipeline.hpp +++ b/projects/composablekernel/include/ck_tile/ops/fused_moe/pipeline/moe_sorting_pipeline.hpp @@ -14,14 +14,6 @@ namespace ck_tile { -// template -// struct MoeSortingPipeline -// { -// // TODO: this kernel only support warp per row -// using Problem = remove_cvref_t; -// using Policy = remove_cvref_t; -// using WeightType = typename Problem::WeightType; - // template // CK_TILE_DEVICE auto operator()(const TopkIdWindow& topk_id_window, // const WeightWindow& weight_window, From 6eda2b755825e30ed7b75bd0926b32526648cbab Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:31 -0400 Subject: [PATCH 41/49] Remove commented-out dead code in ck_tile/ops/gemm/block Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../ops/gemm/block/block_gemm_areg_bsmem_creg_one_warp_v1.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_one_warp_v1.hpp b/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_one_warp_v1.hpp index a7f1cef51942..1a61b69b3426 100644 --- a/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_one_warp_v1.hpp +++ b/projects/composablekernel/include/ck_tile/ops/gemm/block/block_gemm_areg_bsmem_creg_one_warp_v1.hpp @@ -36,9 +36,6 @@ struct BlockGemmARegBSmemCRegOneWarpV1 std::is_same_v>, "wrong!"); - // constexpr index_t MPerBlock = ABlockTensorTmp{}.get_lengths()[number<0>{}]; - // constexpr index_t NPerBlock = BBlockWindowTmp{}.get_window_lengths()[number<0>{}]; - // constexpr index_t KPerBlock = ABlockTensorTmp{}.get_lengths()[number<1>{}]; constexpr index_t MPerBlock = BlockGemmShape::kM; constexpr index_t NPerBlock = BlockGemmShape::kN; constexpr index_t KPerBlock = BlockGemmShape::kK; From 95c29f4e6927e7848b85838aec4708b170801226 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:32 -0400 Subject: [PATCH 42/49] Remove commented-out dead code in ck_tile/ops/norm_reduce/block Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../ck_tile/ops/norm_reduce/block/block_norm_reduce.hpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/ops/norm_reduce/block/block_norm_reduce.hpp b/projects/composablekernel/include/ck_tile/ops/norm_reduce/block/block_norm_reduce.hpp index a7f17155a861..717fb4678ccc 100644 --- a/projects/composablekernel/include/ck_tile/ops/norm_reduce/block/block_norm_reduce.hpp +++ b/projects/composablekernel/include/ck_tile/ops/norm_reduce/block/block_norm_reduce.hpp @@ -120,10 +120,6 @@ struct BlockNormReduceSync constexpr index_t idim_p_lane = NDimP - 1; - // const auto ps_idx = make_array(get_warp_id(), get_lane_id()); - // const auto rs_idx = - // mean_tensor.get_tile_distribution().calculate_rs_index_from_ps_index(ps_idx); - constexpr index_t thread_buf_size = MeanDistributedTensor_::get_thread_buffer_size(); static_assert(thread_buf_size == VarDistributedTensor_::get_thread_buffer_size()); From 0a963d8ff99d75fbf0b887d118a642ca7630c99a Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:32 -0400 Subject: [PATCH 43/49] Remove commented-out dead code in ck_tile/ops/reduce/block Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../include/ck_tile/ops/reduce/block/block_reduce2d.hpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/projects/composablekernel/include/ck_tile/ops/reduce/block/block_reduce2d.hpp b/projects/composablekernel/include/ck_tile/ops/reduce/block/block_reduce2d.hpp index 8cf207c3901c..a14f103eb687 100644 --- a/projects/composablekernel/include/ck_tile/ops/reduce/block/block_reduce2d.hpp +++ b/projects/composablekernel/include/ck_tile/ops/reduce/block/block_reduce2d.hpp @@ -218,10 +218,6 @@ struct BlockReduce2dSync constexpr index_t idim_p_lane = NDimP - 1; - // const auto ps_idx = make_array(get_warp_id(), get_lane_id()); - // const auto rs_idx = - // y_tensor.get_tile_distribution().calculate_rs_index_from_ps_index(ps_idx); - constexpr index_t thread_buf_size = YDistributedTensor_::get_thread_buffer_size(); // loop over thread data From 6d1dc424f714347f668178d77a036f4a255786fc Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:33 -0400 Subject: [PATCH 44/49] Remove commented-out dead code in lib/src/tensor_operation_instance/gpu/gemm_streamk Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../device_gemm_xdl_streamk_f16_f16_f16_mk_kn_mn_instance.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_streamk/device_gemm_xdl_streamk_f16_f16_f16_mk_kn_mn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_streamk/device_gemm_xdl_streamk_f16_f16_f16_mk_kn_mn_instance.cpp index 600154a9fde2..919236deeecc 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_streamk/device_gemm_xdl_streamk_f16_f16_f16_mk_kn_mn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_streamk/device_gemm_xdl_streamk_f16_f16_f16_mk_kn_mn_instance.cpp @@ -26,9 +26,6 @@ using S = ck::Sequence; using PassThrough = ck::tensor_operation::element_wise::PassThrough; -// static constexpr auto GemmDefault = ck::tensor_operation::device::GemmSpecialization::Default; -// static constexpr auto GemmMNPadding = -// ck::tensor_operation::device::GemmSpecialization::MNPadding; using device_gemm_xdl_streamk_f16_f16_f16_mk_kn_mn_generic_instances = std::tuple< // clang-format off //##################|AData| BData| CData| AccData| ALayout| BLayout| CLayout| A| B| C| Block| MPer| NPer| K0Per| K1| MPer| NPer| MXdl| NXdl| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockTransfer| ABlockLds| BBlockTransfer| BBlockTransfer| BBlockTransfer| BlockTransfer| BBlockTransfer| BBlockTransfer| BBlockLds| CShuffle| CShuffle| CBlockTransferClusterLengths| CBlockTransfer| From 385da91512c2c53702decd7a39b540ecadb74d19 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:34 -0400 Subject: [PATCH 45/49] Remove commented-out dead code in profiler/src Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- projects/composablekernel/profiler/src/profile_softmax.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/projects/composablekernel/profiler/src/profile_softmax.cpp b/projects/composablekernel/profiler/src/profile_softmax.cpp index 096a2d4eb441..31cc0fd23a39 100644 --- a/projects/composablekernel/profiler/src/profile_softmax.cpp +++ b/projects/composablekernel/profiler/src/profile_softmax.cpp @@ -278,11 +278,4 @@ int profile_softmax(int argc, char* argv[]) return 0; } -// hijack main() for quick debugging -// int main(int argc, char* argv[]) -// { -// profile_normalization(argc, argv); -// return 0; -// } - REGISTER_PROFILER_OPERATION("softmax", "Softmax", profile_softmax); From 95a5e572b0a2cf77e43f2117f8dc00a1aa66a85d Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 01:31:35 -0400 Subject: [PATCH 46/49] Remove commented-out dead code in test/block_swizzle_test Remove obsolete commented-out code blocks identified by automated dead code scan with expert triage. --- .../test/block_swizzle_test/block_swizzle_test.cpp | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/projects/composablekernel/test/block_swizzle_test/block_swizzle_test.cpp b/projects/composablekernel/test/block_swizzle_test/block_swizzle_test.cpp index 36a26492cfa9..af1bc0658ec1 100644 --- a/projects/composablekernel/test/block_swizzle_test/block_swizzle_test.cpp +++ b/projects/composablekernel/test/block_swizzle_test/block_swizzle_test.cpp @@ -120,17 +120,7 @@ struct block_dispatcher_t uint32_t get_grid_dims_x() { return dp_start_block_idx + dp_num_blocks; } - uint32_t get_block_idx(uint32_t bid) - { - // block id is linearily allocated along sk blocks (dp blocks are fine) - // this function will compute blockIdx.x and the linear sk block mapping - // uint32_t block_idx = 0; - // if(bid < sk_num_big_blocks) { - // uint32_t current_k_iter = bid * k_iters_per_big_block; - // tile_idx = current_k_iter / k_iters_per_tile; - // } - return bid; - } + uint32_t get_block_idx(uint32_t bid) { return bid; } uint32_t get_current_itr(uint32_t block_idx) { From 604081c4d660bd35ce00316d092a5ed88353c347 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 02:12:35 -0400 Subject: [PATCH 47/49] Remove 4 orphaned files with verified replacements - test_gemm_pipeline_compiler.cpp: split into 13 smaller test files - test_grouped_gemm_quant.cpp: split into 5 smaller test files - 2 unsplit f8_f8_f16 instance files: superseded by _part1/_part2 splits Each file verified to have clear replacements already in the build. --- ..._f8_f16_mk_nk_mn_comp_default_instance.cpp | 32 - ...f8_f16_mk_nk_mn_comp_kpadding_instance.cpp | 32 - .../gemm/test_gemm_pipeline_compiler.cpp | 900 ------------------ .../test_grouped_gemm_quant.cpp | 61 -- 4 files changed, 1025 deletions(-) delete mode 100644 projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_multiply_multiply/device_gemm_multiply_multiply_xdl_f8_f8_f16/device_gemm_multiply_multiply_xdl_f8_f8_f16_mk_nk_mn_comp_default_instance.cpp delete mode 100644 projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_multiply_multiply/device_gemm_multiply_multiply_xdl_f8_f8_f16/device_gemm_multiply_multiply_xdl_f8_f8_f16_mk_nk_mn_comp_kpadding_instance.cpp delete mode 100644 projects/composablekernel/test/ck_tile/gemm/test_gemm_pipeline_compiler.cpp delete mode 100644 projects/composablekernel/test/ck_tile/grouped_gemm_quant/test_grouped_gemm_quant.cpp diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_multiply_multiply/device_gemm_multiply_multiply_xdl_f8_f8_f16/device_gemm_multiply_multiply_xdl_f8_f8_f16_mk_nk_mn_comp_default_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_multiply_multiply/device_gemm_multiply_multiply_xdl_f8_f8_f16/device_gemm_multiply_multiply_xdl_f8_f8_f16_mk_nk_mn_comp_default_instance.cpp deleted file mode 100644 index 447681a29483..000000000000 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_multiply_multiply/device_gemm_multiply_multiply_xdl_f8_f8_f16/device_gemm_multiply_multiply_xdl_f8_f8_f16_mk_nk_mn_comp_default_instance.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -// SPDX-License-Identifier: MIT - -#include "device_gemm_multiply_multiply_xdl_f8_f8_f16_mk_nk_mn.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -void add_device_gemm_multiply_multiply_xdl_f8_f8_f16_mk_nk_mn_comp_default_instances( - std::vector, - Row, - F8, - F8, - Tuple, - F16, - PassThrough, - PassThrough, - MultiplyMultiply>>>& instances) -{ - add_device_operation_instances( - instances, - device_gemm_multiply_multiply_xdl_f8_f8_f16_mk_nk_mn_comp_instances{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_multiply_multiply/device_gemm_multiply_multiply_xdl_f8_f8_f16/device_gemm_multiply_multiply_xdl_f8_f8_f16_mk_nk_mn_comp_kpadding_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_multiply_multiply/device_gemm_multiply_multiply_xdl_f8_f8_f16/device_gemm_multiply_multiply_xdl_f8_f8_f16_mk_nk_mn_comp_kpadding_instance.cpp deleted file mode 100644 index 7a377210c2e5..000000000000 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/gemm_multiply_multiply/device_gemm_multiply_multiply_xdl_f8_f8_f16/device_gemm_multiply_multiply_xdl_f8_f8_f16_mk_nk_mn_comp_kpadding_instance.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -// SPDX-License-Identifier: MIT - -#include "device_gemm_multiply_multiply_xdl_f8_f8_f16_mk_nk_mn.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -void add_device_gemm_multiply_multiply_xdl_f8_f8_f16_mk_nk_mn_comp_kpadding_instances( - std::vector, - Row, - F8, - F8, - Tuple, - F16, - PassThrough, - PassThrough, - MultiplyMultiply>>>& instances) -{ - add_device_operation_instances( - instances, - device_gemm_multiply_multiply_xdl_f8_f8_f16_mk_nk_mn_comp_instances{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck diff --git a/projects/composablekernel/test/ck_tile/gemm/test_gemm_pipeline_compiler.cpp b/projects/composablekernel/test/ck_tile/gemm/test_gemm_pipeline_compiler.cpp deleted file mode 100644 index bda1f55b6a10..000000000000 --- a/projects/composablekernel/test/ck_tile/gemm/test_gemm_pipeline_compiler.cpp +++ /dev/null @@ -1,900 +0,0 @@ -// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -// SPDX-License-Identifier: MIT - -#include "test_gemm_pipeline_kernel_types.hpp" -#include "test_gemm_pipeline_util.hpp" -#include "gtest/gtest.h" - -// ============================================================================ -// Comprehensive GEMM Compiler Validation Test Suite -// This file consolidates all GEMM pipeline tests for compiler validation -// Covers essential combinations of data types, layouts, and pipeline types -// ============================================================================ - -// ---------------------------------------------------------------------------- -// Test Class Definitions for Different Pipeline Types -// ---------------------------------------------------------------------------- - -template -class TestGemmMem : public TestCkTileGemmPipeline> -{ -}; - -#if defined(CK_TILE_USE_WMMA) -template -class TestGemmMemWmma : public TestCkTileGemmPipeline> -{ -}; -#endif - -template -class TestGemmCompV3 : public TestCkTileGemmPipeline> -{ -}; - -#if defined(CK_TILE_USE_WMMA) -template -class TestGemmCompV3Wmma : public TestCkTileGemmPipeline> -{ -}; -#endif - -template -class TestGemmCompV4 : public TestCkTileGemmPipeline> -{ -}; - -#if defined(CK_TILE_USE_WMMA) -template -class TestGemmCompV4Wmma : public TestCkTileGemmPipeline> -{ -}; -#endif - -template -class TestGemmCompV6 : public TestCkTileGemmPipeline> -{ -}; - -template -class TestGemmPersistent : public TestCkTileGemmPipeline> -{ -}; - -#if defined(CK_TILE_USE_WMMA) -template -class TestGemmPersistentWmma : public TestCkTileGemmPipeline> -{ -}; -#endif - -// ---------------------------------------------------------------------------- -// Type Definitions for Each Pipeline Configuration -// ---------------------------------------------------------------------------- - -// Memory Pipeline Types -using MemTestTypes = ::testing::Types< - // Parameters: ALayout, BLayout, CLayout, ADataType, BDataType, AccDataType, CDataType, - // M_BlockSize, N_BlockSize, K_BlockSize, M_TileSize, N_TileSize, K_TileSize, Scheduler, - // PipelineType - - std::tuple, - std::tuple>; - -#if defined(CK_TILE_USE_WMMA) -// Memory Pipeline WMMA Types -using MemWmmaTestTypes = ::testing::Types< - std::tuple, - std::tuple>; -#endif - -// CompV3 Pipeline Types -using CompV3TestTypes = ::testing::Types< - std::tuple, - std::tuple>; - -#if defined(CK_TILE_USE_WMMA) -// CompV3 Pipeline WMMA Types -using CompV3WmmaTestTypes = ::testing::Types< - std::tuple, - std::tuple>; -#endif - -// CompV4 Pipeline Types -using CompV4TestTypes = ::testing::Types< - std::tuple, - std::tuple>; - -#if defined(CK_TILE_USE_WMMA) -// CompV4 Pipeline WMMA Types -using CompV4WmmaTestTypes = ::testing::Types< - std::tuple, - std::tuple>; -#endif - -// CompV6 Pipeline Types -using CompV6TestTypes = ::testing::Types< - std::tuple, - std::tuple>; - -// Persistent CompV3 Pipeline Types -using PersistentTestTypes = ::testing::Types, - std::tuple>; - -#if defined(CK_TILE_USE_WMMA) -// Persistent CompV3 Pipeline WMMA Types -using PersistentWmmaTestTypes = ::testing::Types, - std::tuple>; -#endif - -// ---------------------------------------------------------------------------- -// Test Suite Registrations -// ---------------------------------------------------------------------------- - -TYPED_TEST_SUITE(TestGemmMem, MemTestTypes); -#if defined(CK_TILE_USE_WMMA) -TYPED_TEST_SUITE(TestGemmMemWmma, MemWmmaTestTypes); -#endif -TYPED_TEST_SUITE(TestGemmCompV3, CompV3TestTypes); -#if defined(CK_TILE_USE_WMMA) -TYPED_TEST_SUITE(TestGemmCompV3Wmma, CompV3WmmaTestTypes); -#endif -TYPED_TEST_SUITE(TestGemmCompV4, CompV4TestTypes); -#if defined(CK_TILE_USE_WMMA) -TYPED_TEST_SUITE(TestGemmCompV4Wmma, CompV4WmmaTestTypes); -#endif -TYPED_TEST_SUITE(TestGemmCompV6, CompV6TestTypes); -TYPED_TEST_SUITE(TestGemmPersistent, PersistentTestTypes); -#if defined(CK_TILE_USE_WMMA) -TYPED_TEST_SUITE(TestGemmPersistentWmma, PersistentWmmaTestTypes); -#endif - -// ============================================================================ -// Memory Pipeline Tests (Mem) -// ============================================================================ - -#define TEST_SUITE_NAME TestGemmMem - -TYPED_TEST(TEST_SUITE_NAME, SmallM_SingleRow) -{ - std::vector Ms{1}; - constexpr int N = 1024; - constexpr int K = TestFixture::K_Tile * 2; - - for(int M : Ms) - { - if constexpr(std::is_same_v) - { - EXPECT_THROW((this->Run(M, N, K)), std::runtime_error); - } - else - { - this->Run(M, N, K); - } - } -} - -TYPED_TEST(TEST_SUITE_NAME, SingleTile) -{ - this->Run(TestFixture::M_Tile, TestFixture::N_Tile, TestFixture::K_Tile); -} - -TYPED_TEST(TEST_SUITE_NAME, ExactlyTwoTiles_M) -{ - this->Run(TestFixture::M_Tile * 2, TestFixture::N_Tile, TestFixture::K_Tile * 2); -} - -TYPED_TEST(TEST_SUITE_NAME, ExactlyTwoTiles_N) -{ - this->Run(TestFixture::M_Tile, TestFixture::N_Tile * 2, TestFixture::K_Tile * 2); -} - -TYPED_TEST(TEST_SUITE_NAME, ExactlyTwoTiles_K) -{ - this->Run(TestFixture::M_Tile, TestFixture::N_Tile, TestFixture::K_Tile * 2); -} - -TYPED_TEST(TEST_SUITE_NAME, Regular_512x1024x512) -{ - constexpr int M = 512; - constexpr int N = 1024; - constexpr int K = 512; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, Square_1024x1024x1024) -{ - constexpr int M = 1024; - constexpr int N = 1024; - constexpr int K = 1024; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, LargeMatrix_2048x2048x2048) -{ - constexpr int M = 2048; - constexpr int N = 2048; - constexpr int K = 2048; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, VeryLargeMatrix_4096x4096x4096) -{ - constexpr int M = 4096; - constexpr int N = 4096; - constexpr int K = 4096; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, TallSkinny_4096x128x1024) -{ - constexpr int M = 4096; - constexpr int N = 128; - constexpr int K = 1024; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, ShortWide_128x4096x1024) -{ - constexpr int M = 128; - constexpr int N = 4096; - constexpr int K = 1024; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, DeepNarrow_2048x2048x8192) -{ - constexpr int M = 2048; - constexpr int N = 2048; - constexpr int K = 8192; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, StressTest_ExtremelyTallMatrix) -{ - constexpr int M = 16384; - constexpr int N = 64; - constexpr int K = 512; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, StressTest_ExtremelyWideMatrix) -{ - constexpr int M = 64; - constexpr int N = 16384; - constexpr int K = 512; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, StressTest_VeryDeepK) -{ - constexpr int M = 1024; - constexpr int N = 1024; - constexpr int K = 16384; - this->Run(M, N, K); -} - -#undef TEST_SUITE_NAME - -#if defined(CK_TILE_USE_WMMA) -// ============================================================================ -// Memory Pipeline Tests with WMMA -// ============================================================================ - -#define TEST_SUITE_NAME TestGemmMemWmma - -TYPED_TEST(TEST_SUITE_NAME, SingleTile_WMMA) -{ - this->Run(TestFixture::M_Tile, TestFixture::N_Tile, TestFixture::K_Tile); -} - -TYPED_TEST(TEST_SUITE_NAME, Regular_WMMA) -{ - constexpr int M = 512; - constexpr int N = 1024; - constexpr int K = 512; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, LargeMatrix_WMMA) -{ - constexpr int M = 2048; - constexpr int N = 2048; - constexpr int K = 2048; - this->Run(M, N, K); -} - -#undef TEST_SUITE_NAME -#endif // CK_TILE_USE_WMMA - -// ============================================================================ -// Compute V3 Pipeline Tests -// ============================================================================ - -#define TEST_SUITE_NAME TestGemmCompV3 - -TYPED_TEST(TEST_SUITE_NAME, SmallM_CompV3) -{ - std::vector Ms{1, 2}; - constexpr int N = 1024; - std::vector Ks; - for(auto K_count : {2, 4}) - { - Ks.push_back(K_count * TestFixture::K_Tile); - } - - for(int M : Ms) - { - for(int K : Ks) - { - if constexpr(std::is_same_v) - { - EXPECT_THROW((this->Run(M, N, K)), std::runtime_error); - } - else - { - this->Run(M, N, K); - } - } - } -} - -TYPED_TEST(TEST_SUITE_NAME, SingleTile_CompV3) -{ - this->Run(TestFixture::M_Tile, TestFixture::N_Tile, TestFixture::K_Tile); -} - -TYPED_TEST(TEST_SUITE_NAME, MidLargeM_CompV3) -{ - std::vector Ms{127, 255}; - constexpr int N = 1024; - - std::vector Ks; - for(auto K_count : {2, 4}) - { - Ks.push_back(K_count * TestFixture::K_Tile); - } - - constexpr int VecLoadSize = (std::is_same_v || - std::is_same_v || - std::is_same_v) - ? 16 - : 8; - - for(int M : Ms) - { - for(int K : Ks) - { - if constexpr(std::is_same_v) - { - if(M % VecLoadSize == 0) - { - this->Run(M, N, K); - } - else - { - EXPECT_THROW((this->Run(M, N, K)), std::runtime_error); - } - } - else - { - this->Run(M, N, K); - } - } - } -} - -TYPED_TEST(TEST_SUITE_NAME, Regular_CompV3) -{ - constexpr int M = 512; - constexpr int N = 1024; - constexpr int K = 512; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, LargeMatrix_CompV3) -{ - constexpr int M = 2048; - constexpr int N = 2048; - constexpr int K = 2048; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, BatchedSmall_CompV3) -{ - constexpr int M = 256; - constexpr int N = 256; - constexpr int K = 256; - this->Run(M, N, K); -} - -#undef TEST_SUITE_NAME - -#if defined(CK_TILE_USE_WMMA) -// ============================================================================ -// Compute V3 Pipeline Tests with WMMA -// ============================================================================ - -#define TEST_SUITE_NAME TestGemmCompV3Wmma - -TYPED_TEST(TEST_SUITE_NAME, SmallM_CompV3Wmma) -{ - std::vector Ms{1, 2}; - constexpr int N = 1024; - std::vector Ks; - for(auto K_count : {2, 4}) - { - Ks.push_back(K_count * TestFixture::K_Tile); - } - - for(int M : Ms) - { - for(int K : Ks) - { - if constexpr(std::is_same_v) - { - EXPECT_THROW((this->Run(M, N, K)), std::runtime_error); - } - else - { - this->Run(M, N, K); - } - } - } -} - -TYPED_TEST(TEST_SUITE_NAME, SingleTile_CompV3Wmma) -{ - this->Run(TestFixture::M_Tile, TestFixture::N_Tile, TestFixture::K_Tile); -} - -TYPED_TEST(TEST_SUITE_NAME, Regular_CompV3Wmma) -{ - constexpr int M = 512; - constexpr int N = 1024; - constexpr int K = 512; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, LargeMatrix_CompV3Wmma) -{ - constexpr int M = 2048; - constexpr int N = 2048; - constexpr int K = 2048; - this->Run(M, N, K); -} - -#undef TEST_SUITE_NAME -#endif // CK_TILE_USE_WMMA - -// ============================================================================ -// Compute V4 Pipeline Tests -// ============================================================================ - -#define TEST_SUITE_NAME TestGemmCompV4 - -TYPED_TEST(TEST_SUITE_NAME, SmallM_CompV4) -{ - std::vector Ms{1, 2}; - constexpr int N = 1024; - std::vector Ks; - for(auto K_count : {2, 4}) - { - Ks.push_back(K_count * TestFixture::K_Tile); - } - - for(int M : Ms) - { - for(int K : Ks) - { - if constexpr(std::is_same_v) - { - EXPECT_THROW((this->Run(M, N, K)), std::runtime_error); - } - else - { - this->Run(M, N, K); - } - } - } -} - -TYPED_TEST(TEST_SUITE_NAME, SingleTile_CompV4) -{ - this->Run(TestFixture::M_Tile, TestFixture::N_Tile, TestFixture::K_Tile); -} - -TYPED_TEST(TEST_SUITE_NAME, Regular_CompV4) -{ - constexpr int M = 512; - constexpr int N = 1024; - constexpr int K = 512; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, LargeMatrix_CompV4) -{ - constexpr int M = 2048; - constexpr int N = 2048; - constexpr int K = 2048; - this->Run(M, N, K); -} - -#undef TEST_SUITE_NAME - -#if defined(CK_TILE_USE_WMMA) -// ============================================================================ -// Compute V4 Pipeline Tests with WMMA -// ============================================================================ - -#define TEST_SUITE_NAME TestGemmCompV4Wmma - -TYPED_TEST(TEST_SUITE_NAME, SingleTile_CompV4Wmma) -{ - this->Run(TestFixture::M_Tile, TestFixture::N_Tile, TestFixture::K_Tile); -} - -TYPED_TEST(TEST_SUITE_NAME, Regular_CompV4Wmma) -{ - constexpr int M = 512; - constexpr int N = 1024; - constexpr int K = 512; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, LargeMatrix_CompV4Wmma) -{ - constexpr int M = 2048; - constexpr int N = 2048; - constexpr int K = 2048; - this->Run(M, N, K); -} - -#undef TEST_SUITE_NAME -#endif // CK_TILE_USE_WMMA - -// ============================================================================ -// Compute V6 Pipeline Tests -// ============================================================================ - -#define TEST_SUITE_NAME TestGemmCompV6 - -TYPED_TEST(TEST_SUITE_NAME, SmallM_CompV6) -{ - std::vector Ms{1, 2}; - constexpr int N = 1024; - std::vector Ks; - for(auto K_count : {2, 4}) - { - Ks.push_back(K_count * TestFixture::K_Tile); - } - - for(int M : Ms) - { - for(int K : Ks) - { - if constexpr(std::is_same_v) - { - EXPECT_THROW((this->Run(M, N, K)), std::runtime_error); - } - else - { - this->Run(M, N, K); - } - } - } -} - -TYPED_TEST(TEST_SUITE_NAME, SingleTile_CompV6) -{ - this->Run(TestFixture::M_Tile, TestFixture::N_Tile, TestFixture::K_Tile); -} - -TYPED_TEST(TEST_SUITE_NAME, MidLargeM_CompV6) -{ - std::vector Ms{127, 255}; - constexpr int N = 1024; - - std::vector Ks; - for(auto K_count : {2, 4}) - { - Ks.push_back(K_count * TestFixture::K_Tile); - } - - constexpr int VecLoadSize = (std::is_same_v || - std::is_same_v || - std::is_same_v) - ? 16 - : 8; - - for(int M : Ms) - { - for(int K : Ks) - { - if constexpr(std::is_same_v) - { - if(M % VecLoadSize == 0) - { - this->Run(M, N, K); - } - else - { - EXPECT_THROW((this->Run(M, N, K)), std::runtime_error); - } - } - else - { - this->Run(M, N, K); - } - } - } -} - -TYPED_TEST(TEST_SUITE_NAME, Regular_CompV6) -{ - constexpr int M = 512; - constexpr int N = 1024; - constexpr int K = 512; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, LargeMatrix_CompV6) -{ - constexpr int M = 2048; - constexpr int N = 2048; - constexpr int K = 2048; - this->Run(M, N, K); -} - -#undef TEST_SUITE_NAME - -// ============================================================================ -// Persistent Kernel Tests -// ============================================================================ - -#define TEST_SUITE_NAME TestGemmPersistent - -TYPED_TEST(TEST_SUITE_NAME, SmallM_Persistent) -{ - std::vector Ms{1, 2}; - constexpr int N = 1024; - std::vector Ks; - for(auto K_count : {2, 4}) - { - Ks.push_back(K_count * TestFixture::K_Tile); - } - - for(int M : Ms) - { - for(int K : Ks) - { - if constexpr(std::is_same_v) - { - EXPECT_THROW((this->Run(M, N, K)), std::runtime_error); - } - else - { - this->Run(M, N, K); - } - } - } -} - -TYPED_TEST(TEST_SUITE_NAME, SingleTile_Persistent) -{ - this->Run(TestFixture::M_Tile, TestFixture::N_Tile, TestFixture::K_Tile); -} - -TYPED_TEST(TEST_SUITE_NAME, Regular_Persistent) -{ - constexpr int M = 512; - constexpr int N = 1024; - constexpr int K = 512; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, LargeMatrix_Persistent) -{ - constexpr int M = 2048; - constexpr int N = 2048; - constexpr int K = 2048; - this->Run(M, N, K); -} - -#undef TEST_SUITE_NAME - -#if defined(CK_TILE_USE_WMMA) -// ============================================================================ -// Persistent Kernel Tests with WMMA -// ============================================================================ - -#define TEST_SUITE_NAME TestGemmPersistentWmma - -TYPED_TEST(TEST_SUITE_NAME, SmallM_PersistentWmma) -{ - std::vector Ms{1, 2}; - constexpr int N = 1024; - std::vector Ks; - for(auto K_count : {2, 4}) - { - Ks.push_back(K_count * TestFixture::K_Tile); - } - - for(int M : Ms) - { - for(int K : Ks) - { - if constexpr(std::is_same_v) - { - EXPECT_THROW((this->Run(M, N, K)), std::runtime_error); - } - else - { - this->Run(M, N, K); - } - } - } -} - -TYPED_TEST(TEST_SUITE_NAME, SingleTile_PersistentWmma) -{ - this->Run(TestFixture::M_Tile, TestFixture::N_Tile, TestFixture::K_Tile); -} - -TYPED_TEST(TEST_SUITE_NAME, Regular_PersistentWmma) -{ - constexpr int M = 512; - constexpr int N = 1024; - constexpr int K = 512; - this->Run(M, N, K); -} - -TYPED_TEST(TEST_SUITE_NAME, LargeMatrix_PersistentWmma) -{ - constexpr int M = 2048; - constexpr int N = 2048; - constexpr int K = 2048; - this->Run(M, N, K); -} - -#undef TEST_SUITE_NAME -#endif // CK_TILE_USE_WMMA diff --git a/projects/composablekernel/test/ck_tile/grouped_gemm_quant/test_grouped_gemm_quant.cpp b/projects/composablekernel/test/ck_tile/grouped_gemm_quant/test_grouped_gemm_quant.cpp deleted file mode 100644 index 6a1a28884a6b..000000000000 --- a/projects/composablekernel/test/ck_tile/grouped_gemm_quant/test_grouped_gemm_quant.cpp +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -// SPDX-License-Identifier: MIT - -#include - -#include "gtest/gtest.h" - -#include "ck_tile/host.hpp" -#include "test_grouped_gemm_util_quant.hpp" - -using F16 = ck_tile::half_t; -using F32 = float; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Row = ck_tile::tensor_layout::gemm::RowMajor; -using Col = ck_tile::tensor_layout::gemm::ColumnMajor; -using True = ck_tile::bool_constant; -using False = ck_tile::bool_constant; -using RowColQuant = std::integral_constant; -using TensorQuant = std::integral_constant; -using AQuant = std::integral_constant; -using BQuant = std::integral_constant; - -// clang-format off -using KernelTypes = ::testing::Types< - // ALayout, BLayout, CLayout, ADataType, AQDataType, BDataType, BQDataType, AccDataType, CDataType, QuantType, PreshuffleB, Persistent, TransposeC - std::tuple< Row, Col, Row, FP8, F32, FP8, F32, F32, F16, RowColQuant, False, True, False>, - std::tuple< Col, Col, Row, FP8, F32, FP8, F32, F32, F16, RowColQuant, False, True, False>, - std::tuple< Row, Row, Row, FP8, F32, FP8, F32, F32, F16, RowColQuant, False, True, False>, - std::tuple< Col, Row, Row, FP8, F32, FP8, F32, F32, F16, RowColQuant, False, True, False>, - - std::tuple< Row, Col, Row, BF8, F32, BF8, F32, F32, F16, RowColQuant, False, True, False>, - std::tuple< Col, Col, Row, BF8, F32, BF8, F32, F32, F16, RowColQuant, False, True, False>, - std::tuple< Row, Row, Row, BF8, F32, BF8, F32, F32, F16, RowColQuant, False, True, False>, - std::tuple< Col, Row, Row, BF8, F32, BF8, F32, F32, F16, RowColQuant, False, True, False>, - - std::tuple< Row, Col, Row, FP8, F32, FP8, F32, F32, F16, TensorQuant, False, True, False>, - std::tuple< Col, Col, Row, FP8, F32, FP8, F32, F32, F16, TensorQuant, False, True, False>, - std::tuple< Row, Row, Row, FP8, F32, FP8, F32, F32, F16, TensorQuant, False, True, False>, - std::tuple< Col, Row, Row, FP8, F32, FP8, F32, F32, F16, TensorQuant, False, True, False>, - - std::tuple< Row, Col, Row, BF8, F32, BF8, F32, F32, F16, TensorQuant, False, True, False>, - std::tuple< Col, Col, Row, BF8, F32, BF8, F32, F32, F16, TensorQuant, False, True, False>, - std::tuple< Row, Row, Row, BF8, F32, BF8, F32, F32, F16, TensorQuant, False, True, False>, - std::tuple< Col, Row, Row, BF8, F32, BF8, F32, F32, F16, TensorQuant, False, True, False>, - - std::tuple< Row, Col, Row, FP8, F32, FP8, F32, F32, F16, AQuant, False, True, True>, - std::tuple< Row, Col, Row, FP8, F32, FP8, F32, F32, F16, AQuant, False, True, False>, - std::tuple< Row, Col, Row, BF8, F32, BF8, F32, F32, F16, AQuant, False, True, True>, - std::tuple< Row, Col, Row, BF8, F32, BF8, F32, F32, F16, AQuant, False, True, False>, - - std::tuple< Row, Col, Row, FP8, F32, FP8, F32, F32, F16, BQuant, False, True, False>, - std::tuple< Row, Col, Row, FP8, F32, FP8, F32, F32, F16, BQuant, True, True, False>, - std::tuple< Row, Col, Row, BF8, F32, BF8, F32, F32, F16, BQuant, False, True, False>, - std::tuple< Row, Col, Row, BF8, F32, BF8, F32, F32, F16, BQuant, True, True, False> - >; -// clang-format on - -TYPED_TEST_SUITE(TestCkTileGroupedGemmQuant, KernelTypes); - -#include "test_grouped_gemm_quant_ut_cases.inc" From 8c757119015a4217ed97ca5aacf8821dc679cc0d Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Thu, 9 Apr 2026 02:54:54 -0400 Subject: [PATCH 48/49] Extract shared boilerplate from 47 gemm_quant test files Create test_gemm_quant_common.hpp with shared includes, layout aliases, data type aliases, quant type aliases, and group size aliases that were copy-pasted across all 47 test_gemm_quant_*.cpp files. Each test file now includes the common header instead of duplicating ~20 lines of identical boilerplate. Separate .cpp files are preserved for parallel compilation. No functional changes. Net reduction: ~733 lines of duplicated code. --- .../test_gemm_quant_abquant_a4w4_base.cpp | 68 ++++------- .../test_gemm_quant_abquant_a4w4_padding.cpp | 110 +++++++----------- ...est_gemm_quant_abquant_a4w4_preshuffle.cpp | 68 ++++------- .../test_gemm_quant_abquant_base.cpp | 94 ++++++--------- .../test_gemm_quant_abquant_eightwaves.cpp | 72 +++++------- .../test_gemm_quant_abquant_padding.cpp | 61 ++++------ ...est_gemm_quant_abquant_preshuffleQuant.cpp | 68 ++++------- .../test_gemm_quant_abquant_preshuffle_2d.cpp | 76 +++++------- ...ant_abquant_preshuffle_preshuffleQuant.cpp | 68 ++++------- .../test_gemm_quant_abquant_splitk_decode.cpp | 18 +-- ...test_gemm_quant_abquant_splitk_prefill.cpp | 18 +-- .../test_gemm_quant_aquant_base_ccr.cpp | 24 +--- .../test_gemm_quant_aquant_base_rcr.cpp | 24 +--- .../test_gemm_quant_aquant_base_rrr_crr.cpp | 28 ++--- ...gemm_quant_aquant_mem_decode_interwave.cpp | 24 +--- ...gemm_quant_aquant_mem_decode_intrawave.cpp | 24 +--- ...emm_quant_aquant_mem_prefill_interwave.cpp | 24 +--- .../test_gemm_quant_aquant_prefill.cpp | 22 +--- .../test_gemm_quant_aquant_preshuffle.cpp | 32 ++--- .../test_gemm_quant_aquant_transpose_c.cpp | 20 +--- .../test_gemm_quant_bquant_1d_128.cpp | 24 +--- .../test_gemm_quant_bquant_1d_64.cpp | 18 +-- .../test_gemm_quant_bquant_2d_large_n.cpp | 16 +-- .../test_gemm_quant_bquant_2d_medium_n.cpp | 18 +-- .../test_gemm_quant_bquant_2d_small_n.cpp | 18 +-- ...emm_quant_bquant_microscale_ccr_1d_128.cpp | 18 +-- ...gemm_quant_bquant_microscale_ccr_1d_64.cpp | 22 +--- ...emm_quant_bquant_microscale_crr_1d_128.cpp | 18 +-- ...gemm_quant_bquant_microscale_crr_1d_64.cpp | 18 +-- ...emm_quant_bquant_microscale_rcr_1d_128.cpp | 20 +--- ...gemm_quant_bquant_microscale_rcr_1d_64.cpp | 20 +--- ...emm_quant_bquant_microscale_rrr_1d_128.cpp | 18 +-- ...gemm_quant_bquant_microscale_rrr_1d_64.cpp | 18 +-- ...quant_bquant_preshuffleQuant_decode_1d.cpp | 22 +--- ...quant_bquant_preshuffleQuant_decode_2d.cpp | 18 +-- ...uant_bquant_preshuffleQuant_prefill_1d.cpp | 26 +---- ...uant_bquant_preshuffleQuant_prefill_2d.cpp | 18 +-- ...gemm_quant_bquant_preshuffle_decode_1d.cpp | 22 +--- ...gemm_quant_bquant_preshuffle_decode_2d.cpp | 18 +-- ...emm_quant_bquant_preshuffle_prefill_1d.cpp | 26 +---- ...emm_quant_bquant_preshuffle_prefill_2d.cpp | 18 +-- ..._quant_bquant_preshuffle_tiled_permute.cpp | 24 +--- .../test_gemm_quant_bquant_splitk_decode.cpp | 20 +--- .../test_gemm_quant_bquant_splitk_prefill.cpp | 18 +-- .../test_gemm_quant_bquant_transpose.cpp | 18 +-- .../test_gemm_quant_common.hpp | 40 +++++++ .../test_gemm_quant_rowcol.cpp | 21 +--- .../test_gemm_quant_tensor.cpp | 21 +--- 48 files changed, 413 insertions(+), 1106 deletions(-) create mode 100644 projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_common.hpp diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_a4w4_base.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_a4w4_base.cpp index 5e2403f7d17e..78dcf1d32574 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_a4w4_base.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_a4w4_base.cpp @@ -1,44 +1,24 @@ -// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -// SPDX-License-Identifier: MIT - -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using Half = ck_tile::half_t; -using PkFP4 = ck_tile::pk_fp4_t; -using ABQuantGrouped = - std::integral_constant; - -// 1d block sizes for AQuant -using GroupSize1D = ck_tile::QuantGroupShape>; - -// 2d block sizes for BQuant -using GroupSize2D = ck_tile::QuantGroupShape>; - -// Type combinations for ABQuant tests -// Tuple format: -// clang-format off -using ABQuantTypes = ::testing::Types< - // PreshuffleQuant = false && TransposeC = false - // RCR layout with RowMajor AQ, ColumnMajor BQ - std::tuple ->; -// clang-format on - -// Test suite for ABQuant -TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantTypes); - -// AQuant tests -TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) -{ - this->run_test_with_validation(1024, 1024, 1024); -} +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_quant_common.hpp" + +// Type combinations for ABQuant tests +// Tuple format: +// clang-format off +using ABQuantTypes = ::testing::Types< + // PreshuffleQuant = false && TransposeC = false + // RCR layout with RowMajor AQ, ColumnMajor BQ + std::tuple +>; +// clang-format on + +// Test suite for ABQuant +TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantTypes); + +// AQuant tests +TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) +{ + this->run_test_with_validation(1024, 1024, 1024); +} diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_a4w4_padding.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_a4w4_padding.cpp index 1e496d5b6423..0c39d9ed2aaf 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_a4w4_padding.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_a4w4_padding.cpp @@ -1,65 +1,45 @@ -// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -// SPDX-License-Identifier: MIT - -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using Half = ck_tile::half_t; -using PkFP4 = ck_tile::pk_fp4_t; -using ABQuantGrouped = - std::integral_constant; - -// 1d block sizes for AQuant -using GroupSize1D = ck_tile::QuantGroupShape>; - -// 2d block sizes for BQuant -using GroupSize2D = ck_tile::QuantGroupShape>; - -// Type combinations for ABQuant tests -// Tuple format: -// clang-format off -using ABQuantTypes = ::testing::Types< - // PreshuffleQuant = false && TransposeC = false - // RCR layout with RowMajor AQ, ColumnMajor BQ - std::tuple ->; -// clang-format on - -// Test suite for ABQuant -TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantTypes); - -// AQuant tests - -TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest_PadK) -{ - this->run_test_with_validation(1024, 1024, 832); -} - -TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest_PadN) -{ - this->run_test_with_validation(1024, 832, 1024); -} - -TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest_PadM) -{ - this->run_test_with_validation(832, 1024, 1024); -} - -TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest_PadMNK) -{ - this->run_test_with_validation(832, 832, 832); -} - -TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest_PadNK) -{ - this->run_test_with_validation(1024, 832, 832); -} +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_quant_common.hpp" + +// Type combinations for ABQuant tests +// Tuple format: +// clang-format off +using ABQuantTypes = ::testing::Types< + // PreshuffleQuant = false && TransposeC = false + // RCR layout with RowMajor AQ, ColumnMajor BQ + std::tuple +>; +// clang-format on + +// Test suite for ABQuant +TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantTypes); + +// AQuant tests + +TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest_PadK) +{ + this->run_test_with_validation(1024, 1024, 832); +} + +TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest_PadN) +{ + this->run_test_with_validation(1024, 832, 1024); +} + +TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest_PadM) +{ + this->run_test_with_validation(832, 1024, 1024); +} + +TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest_PadMNK) +{ + this->run_test_with_validation(832, 832, 832); +} + +TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest_PadNK) +{ + this->run_test_with_validation(1024, 832, 832); +} diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_a4w4_preshuffle.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_a4w4_preshuffle.cpp index 43051c8d088f..3df77fc4fb5f 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_a4w4_preshuffle.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_a4w4_preshuffle.cpp @@ -1,44 +1,24 @@ -// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -// SPDX-License-Identifier: MIT - -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using Half = ck_tile::half_t; -using PkFP4 = ck_tile::pk_fp4_t; -using ABQuantGrouped = - std::integral_constant; - -// 1d block sizes for AQuant -using GroupSize1D = ck_tile::QuantGroupShape>; - -// 2d block sizes for BQuant -using GroupSize2D = ck_tile::QuantGroupShape>; - -// Type combinations for ABQuant tests -// Tuple format: -// clang-format off -using ABQuantTypes = ::testing::Types< - // RCR layout with RowMajor AQ, ColumnMajor BQ - // PreshuffleB = true && TransposeC = false - std::tuple ->; -// clang-format on - -// Test suite for ABQuant -TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantTypes); - -// AQuant tests -TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) -{ - this->run_test_with_validation(1024, 1024, 1024); -} +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_quant_common.hpp" + +// Type combinations for ABQuant tests +// Tuple format: +// clang-format off +using ABQuantTypes = ::testing::Types< + // RCR layout with RowMajor AQ, ColumnMajor BQ + // PreshuffleB = true && TransposeC = false + std::tuple +>; +// clang-format on + +// Test suite for ABQuant +TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantTypes); + +// AQuant tests +TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) +{ + this->run_test_with_validation(1024, 1024, 1024); +} diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_base.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_base.cpp index 2524f7887f70..e97459b892c6 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_base.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_base.cpp @@ -1,56 +1,38 @@ -// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -// SPDX-License-Identifier: MIT - -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using ABQuantGrouped = - std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; - -// 2d block sizes for BQuant -using GroupSize2D128N = ck_tile::QuantGroupShape>; - -// Type combinations for ABQuant tests -// Tuple format: -// clang-format off -using ABQuantTypes = ::testing::Types< - // 1D BScales; PreshuffleQuant = false && TransposeC = false (RCR layout with RowMajor AQ) - std::tuple, - std::tuple, - std::tuple, - std::tuple, - std::tuple, - std::tuple, - - // 2D B-scales; PreshuffleQuant = false && TransposeC = true (RCR layout with RowMajor AQ) - std::tuple, - std::tuple, - std::tuple, - std::tuple, - std::tuple, - std::tuple ->; -// clang-format on - -// Test suite for ABQuant -TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantTypes); - -// AQuant tests -TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) -{ - this->run_test_with_validation(1024, 1024, 1024); -} +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_quant_common.hpp" + +using GroupSize2D128N = ck_tile::QuantGroupShape>; + +// Type combinations for ABQuant tests +// Tuple format: +// clang-format off +using ABQuantTypes = ::testing::Types< + // 1D BScales; PreshuffleQuant = false && TransposeC = false (RCR layout with RowMajor AQ) + std::tuple, + std::tuple, + std::tuple, + std::tuple, + std::tuple, + std::tuple, + + // 2D B-scales; PreshuffleQuant = false && TransposeC = true (RCR layout with RowMajor AQ) + std::tuple, + std::tuple, + std::tuple, + std::tuple, + std::tuple, + std::tuple +>; +// clang-format on + +// Test suite for ABQuant +TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantTypes); + +// AQuant tests +TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) +{ + this->run_test_with_validation(1024, 1024, 1024); +} diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_eightwaves.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_eightwaves.cpp index baeb93ac0a45..746570f30dd5 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_eightwaves.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_eightwaves.cpp @@ -1,45 +1,27 @@ -// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -// SPDX-License-Identifier: MIT - -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using ABQuantGrouped = - std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; - -// 2d block sizes for BQuant -using GroupSize2D128N = ck_tile::QuantGroupShape>; -#ifdef CK_GFX950_SUPPORT -// Type combinations for ABQuant tests -// Tuple format: -// clang-format off -using ABQuantEightWavesTypes = ::testing::Types< - // PreshuffleQuant = false && TransposeC = false (RCR layout with RowMajor AQ) - std::tuple, - std::tuple ->; -// clang-format on - -// Test suite for ABQuant -TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantEightWavesTypes); - -// AQuant tests -TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) -{ - this->run_test_with_validation(1024, 1024, 1024); -} -#endif +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_quant_common.hpp" + +using GroupSize2D128N = ck_tile::QuantGroupShape>; +#ifdef CK_GFX950_SUPPORT +// Type combinations for ABQuant tests +// Tuple format: +// clang-format off +using ABQuantEightWavesTypes = ::testing::Types< + // PreshuffleQuant = false && TransposeC = false (RCR layout with RowMajor AQ) + std::tuple, + std::tuple +>; +// clang-format on + +// Test suite for ABQuant +TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantEightWavesTypes); + +// AQuant tests +TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) +{ + this->run_test_with_validation(1024, 1024, 1024); +} +#endif diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_padding.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_padding.cpp index 5247a4405de0..fe4ec0a42805 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_padding.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_padding.cpp @@ -1,39 +1,22 @@ -// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -// SPDX-License-Identifier: MIT - -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using ABQuantGrouped = - std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; - -// Type combinations for ABQuant padding padding tests -// Tuple format: -// clang-format off -using ABQuantPaddingTypes = ::testing::Types< - std::tuple ->; -// clang-format on - -// Test suite for ABQuant Padding -TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantPaddingTypes); - -// AQuant tests -TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) -{ - this->run_test_with_validation(1024, 832, 832); -} +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_quant_common.hpp" + +// Type combinations for ABQuant padding padding tests +// Tuple format: +// clang-format off +using ABQuantPaddingTypes = ::testing::Types< + std::tuple +>; +// clang-format on + +// Test suite for ABQuant Padding +TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantPaddingTypes); + +// AQuant tests +TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) +{ + this->run_test_with_validation(1024, 832, 832); +} diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_preshuffleQuant.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_preshuffleQuant.cpp index 1b554cc12abb..f949fd4e4788 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_preshuffleQuant.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_preshuffleQuant.cpp @@ -1,43 +1,25 @@ -// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -// SPDX-License-Identifier: MIT - -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using ABQuantGrouped = - std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; - -// 2d block sizes for BQuant -using GroupSize2D128N = ck_tile::QuantGroupShape>; - -// Type combinations for ABQuant tests -// Tuple format: -// clang-format off -using ABQuantPreshuffleQuantTypes = ::testing::Types< - std::tuple, - std::tuple ->; -// clang-format on - -// Test suite for ABQuant -TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantPreshuffleQuantTypes); - -// AQuant tests -TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) -{ - this->run_test_with_validation(1024, 1024, 1024); -} +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_quant_common.hpp" + +using GroupSize2D128N = ck_tile::QuantGroupShape>; + +// Type combinations for ABQuant tests +// Tuple format: +// clang-format off +using ABQuantPreshuffleQuantTypes = ::testing::Types< + std::tuple, + std::tuple +>; +// clang-format on + +// Test suite for ABQuant +TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantPreshuffleQuantTypes); + +// AQuant tests +TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) +{ + this->run_test_with_validation(1024, 1024, 1024); +} diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_preshuffle_2d.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_preshuffle_2d.cpp index 7d8b62616e55..a940c2fd0284 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_preshuffle_2d.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_preshuffle_2d.cpp @@ -1,47 +1,29 @@ -// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -// SPDX-License-Identifier: MIT - -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using ABQuantGrouped = - std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; - -// 2d block sizes for BQuant -using GroupSize2D128N = ck_tile::QuantGroupShape>; - -// Type combinations for ABQuant tests -// Tuple format: -// clang-format off -using ABQuantPreshuffleBTypes = ::testing::Types< - // 1D B-scales; PreshuffleQuant = false && TransposeC = false (RCR layout with RowMajor AQ) - std::tuple, - /// 2D B-scales; PreshuffleQuant = false && TransposeC = true (RCR layout with RowMajor AQ) - std::tuple, - std::tuple, - std::tuple ->; -// clang-format on - -// Test suite for ABQuant -TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantPreshuffleBTypes); - -// AQuant tests -TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) -{ - this->run_test_with_validation(1024, 1024, 1024); -} +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_quant_common.hpp" + +using GroupSize2D128N = ck_tile::QuantGroupShape>; + +// Type combinations for ABQuant tests +// Tuple format: +// clang-format off +using ABQuantPreshuffleBTypes = ::testing::Types< + // 1D B-scales; PreshuffleQuant = false && TransposeC = false (RCR layout with RowMajor AQ) + std::tuple, + /// 2D B-scales; PreshuffleQuant = false && TransposeC = true (RCR layout with RowMajor AQ) + std::tuple, + std::tuple, + std::tuple +>; +// clang-format on + +// Test suite for ABQuant +TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantPreshuffleBTypes); + +// AQuant tests +TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) +{ + this->run_test_with_validation(1024, 1024, 1024); +} diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_preshuffle_preshuffleQuant.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_preshuffle_preshuffleQuant.cpp index 0b845ac16d96..51e555479ded 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_preshuffle_preshuffleQuant.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_preshuffle_preshuffleQuant.cpp @@ -1,43 +1,25 @@ -// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -// SPDX-License-Identifier: MIT - -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using ABQuantGrouped = - std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; - -// 2d block sizes for BQuant -using GroupSize2D128N = ck_tile::QuantGroupShape>; - -// Type combinations for ABQuant tests -// Tuple format: -// clang-format off -using ABQuantPreshuffleQuantTypes = ::testing::Types< - std::tuple, GroupSize, GroupSize, ColumnMajor>, - std::tuple, GroupSize, GroupSize2D128N, ColumnMajor> ->; -// clang-format on - -// Test suite for ABQuant -TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantPreshuffleQuantTypes); - -// AQuant tests -TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) -{ - this->run_test_with_validation(1024, 1024, 1024); -} +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#include "test_gemm_quant_common.hpp" + +using GroupSize2D128N = ck_tile::QuantGroupShape>; + +// Type combinations for ABQuant tests +// Tuple format: +// clang-format off +using ABQuantPreshuffleQuantTypes = ::testing::Types< + std::tuple, GroupSize1D_128, GroupSize1D_128, ColumnMajor>, + std::tuple, GroupSize1D_128, GroupSize2D128N, ColumnMajor> +>; +// clang-format on + +// Test suite for ABQuant +TYPED_TEST_SUITE(TestCkTileGemmABQuant, ABQuantPreshuffleQuantTypes); + +// AQuant tests +TYPED_TEST(TestCkTileGemmABQuant, ABQuantGroupedTest) +{ + this->run_test_with_validation(1024, 1024, 1024); +} diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_splitk_decode.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_splitk_decode.cpp index 7732779d7a25..7f8fb70f9969 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_splitk_decode.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_splitk_decode.cpp @@ -1,22 +1,8 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using ABQuantGrouped = - std::integral_constant; +#include "test_gemm_quant_common.hpp" + using GroupSize1x1x128 = ck_tile::QuantGroupShape>; using GroupSize1x128x128 = ck_tile::QuantGroupShape>; diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_splitk_prefill.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_splitk_prefill.cpp index f746983d0676..8f58ef7c7fad 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_splitk_prefill.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_abquant_splitk_prefill.cpp @@ -1,22 +1,8 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using ABQuantGrouped = - std::integral_constant; +#include "test_gemm_quant_common.hpp" + using GroupSize1x1x128 = ck_tile::QuantGroupShape>; using GroupSize1x128x128 = ck_tile::QuantGroupShape>; diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_base_ccr.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_base_ccr.cpp index 0e04f9fc9e90..e66cf10ca868 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_base_ccr.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_base_ccr.cpp @@ -1,23 +1,9 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; using AQuantGrouped = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; // Type combinations for AQuant tests - CCR layout // Tuple format: >; // clang-format off using AQuantBaseCCRTypes = ::testing::Types< // CCR layout (ColumnMajor A, ColumnMajor B, RowMajor C with ColumnMajor AQ) - NEW layout support - std::tuple, - std::tuple, - std::tuple, - std::tuple + std::tuple, + std::tuple, + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_base_rcr.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_base_rcr.cpp index da32c063042d..671c878957f5 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_base_rcr.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_base_rcr.cpp @@ -1,23 +1,9 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; using AQuantGrouped = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; // Type combinations for AQuant tests - RCR layout base configuration // Tuple format: >; // clang-format off using AQuantBaseRCRTypes = ::testing::Types< // PreshuffleQuant = false && TransposeC = false (RCR layout with RowMajor AQ) - std::tuple, - std::tuple, - std::tuple, - std::tuple + std::tuple, + std::tuple, + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_base_rrr_crr.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_base_rrr_crr.cpp index 6e90c44764c8..e3b3c0953ab9 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_base_rrr_crr.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_base_rrr_crr.cpp @@ -1,23 +1,9 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; using AQuantGrouped = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; // Type combinations for AQuant tests - RRR and CRR layouts // Tuple format: >; // clang-format off using AQuantBaseRRRCRRTypes = ::testing::Types< // RRR layout (RowMajor A, RowMajor B, RowMajor C with RowMajor AQ) - std::tuple, - std::tuple, - std::tuple, - std::tuple, + std::tuple, + std::tuple, + std::tuple, + std::tuple, // CRR layout (ColumnMajor A, RowMajor B, RowMajor C with RowMajor AQ) - std::tuple, - std::tuple + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_mem_decode_interwave.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_mem_decode_interwave.cpp index a7ab4120a188..1ef57716c930 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_mem_decode_interwave.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_mem_decode_interwave.cpp @@ -1,33 +1,19 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; using AQuantGrouped = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; // Type combinations for AQuant tests - Mem Decode Interwave Configuration // Tuple format: // clang-format off using AQuantMemDecodeInterwaveTypes = ::testing::Types< - std::tuple, - std::tuple, - std::tuple, - std::tuple + std::tuple, + std::tuple, + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_mem_decode_intrawave.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_mem_decode_intrawave.cpp index 483138d71108..0c908a9d21cc 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_mem_decode_intrawave.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_mem_decode_intrawave.cpp @@ -1,33 +1,19 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; using AQuantGrouped = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; // Type combinations for AQuant tests - Mem Decode Intrawave Configuration // Tuple format: // clang-format off using AQuantMemDecodeIntrawaveTypes = ::testing::Types< - std::tuple, - std::tuple, - std::tuple, - std::tuple + std::tuple, + std::tuple, + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_mem_prefill_interwave.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_mem_prefill_interwave.cpp index 7e851d9bd39d..fde3ec977b07 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_mem_prefill_interwave.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_mem_prefill_interwave.cpp @@ -1,33 +1,19 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; using AQuantGrouped = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; // Type combinations for AQuant tests - Mem Prefill Interwave Configuration // Tuple format: // clang-format off using AQuantMemPrefillInterwaveTypes = ::testing::Types< - std::tuple, - std::tuple, - std::tuple, - std::tuple + std::tuple, + std::tuple, + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_prefill.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_prefill.cpp index 911af678df5a..50e882a1d1bf 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_prefill.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_prefill.cpp @@ -1,23 +1,9 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; using AQuantGrouped = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; // Type combinations for AQuant tests - Prefill Configuration // Tuple format: >; // clang-format off using AQuantPrefillTypes = ::testing::Types< // RCR layout - with the Prefill BlockTile Config. - std::tuple, - std::tuple, - std::tuple + std::tuple, + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_preshuffle.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_preshuffle.cpp index 35d15f935410..2a0876ea8244 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_preshuffle.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_preshuffle.cpp @@ -1,23 +1,9 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; using AQuantGrouped = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; // Type combinations for AQuant tests - PreshuffleQuant Configurations // Tuple format: >; // clang-format off using AQuantPreshuffleTypes = ::testing::Types< // PreshuffleQuant = true && TransposeC = false (with RowMajor AQ - PreshuffleQuant only supports RowMajor) - std::tuple, - std::tuple, - std::tuple, - std::tuple, + std::tuple, + std::tuple, + std::tuple, + std::tuple, // PreshuffleQuant = true && TransposeC = true (with RowMajor AQ - PreshuffleQuant only supports RowMajor) - std::tuple, - std::tuple, - std::tuple, - std::tuple + std::tuple, + std::tuple, + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_transpose_c.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_transpose_c.cpp index a2a4c2c38b60..5481419a4462 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_transpose_c.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_aquant_transpose_c.cpp @@ -1,23 +1,9 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; using AQuantGrouped = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; // Type combinations for AQuant tests - TransposeC Configuration // Tuple format: >; // clang-format off using AQuantTransposeCTypes = ::testing::Types< // PreshuffleQuant = false && TransposeC = true (with RowMajor AQ) - std::tuple, - std::tuple + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_1d_128.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_1d_128.cpp index 0e6e40b788c9..aa4006ec2397 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_1d_128.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_1d_128.cpp @@ -1,23 +1,7 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; +#include "test_gemm_quant_common.hpp" // Type combinations for BQuant tests - 1D GroupSize 128 // Tuple format: >; // clang-format off using BQuant1D128Types = ::testing::Types< // 1d cases with grouping only on k axis - std::tuple, - std::tuple, - std::tuple + std::tuple, + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_1d_64.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_1d_64.cpp index 1019caf1bcad..9f266b37be11 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_1d_64.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_1d_64.cpp @@ -1,23 +1,9 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; -using GroupSize64 = ck_tile::QuantGroupShape>; +using GroupSize64 = ck_tile::QuantGroupShape>; // Type combinations for BQuant tests - 1D GroupSize 64 // Tuple format: -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; using GroupSize2D128N = ck_tile::QuantGroupShape>; // Type combinations for BQuant tests - 2D Large N (128N) diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_2d_medium_n.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_2d_medium_n.cpp index 67d52ef874cf..409e044d41e8 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_2d_medium_n.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_2d_medium_n.cpp @@ -1,24 +1,8 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; - -// 2d block sizes for BQuant using GroupSize2D32N = ck_tile::QuantGroupShape>; using GroupSize2D64N = ck_tile::QuantGroupShape>; diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_2d_small_n.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_2d_small_n.cpp index 865713992d78..024c185012f8 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_2d_small_n.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_2d_small_n.cpp @@ -1,24 +1,8 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; - -// 2d block sizes for BQuant using GroupSize2D8N = ck_tile::QuantGroupShape>; using GroupSize2D16N = ck_tile::QuantGroupShape>; diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_microscale_ccr_1d_128.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_microscale_ccr_1d_128.cpp index 94572a80dc21..819eb0dafdb1 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_microscale_ccr_1d_128.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_microscale_ccr_1d_128.cpp @@ -1,23 +1,9 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using BF8 = ck_tile::bf8_t; -using BF16 = ck_tile::bf16_t; -using PkFP4 = ck_tile::pk_fp4_t; -using E8M0 = ck_tile::e8m0_t; -using BQuantGrouped = std::integral_constant; -using GroupSize128 = ck_tile::QuantGroupShape>; +using GroupSize128 = ck_tile::QuantGroupShape>; // Type combinations for BQuant tests - 1D GroupSize 128 // Tuple format: -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using FP16 = ck_tile::fp16_t; -using BF16 = ck_tile::bf16_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using PkFP4 = ck_tile::pk_fp4_t; -using E8M0 = ck_tile::e8m0_t; -using BQuantGrouped = std::integral_constant; -using GroupSize64 = ck_tile::QuantGroupShape>; +using GroupSize64 = ck_tile::QuantGroupShape>; // Type combinations for BQuant tests - 1D GroupSize 64 // Tuple format: -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using BF8 = ck_tile::bf8_t; -using BF16 = ck_tile::bf16_t; -using PkFP4 = ck_tile::pk_fp4_t; -using E8M0 = ck_tile::e8m0_t; -using BQuantGrouped = std::integral_constant; -using GroupSize128 = ck_tile::QuantGroupShape>; +using GroupSize128 = ck_tile::QuantGroupShape>; // Type combinations for BQuant tests - 1D GroupSize 128 // Tuple format: -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using BF8 = ck_tile::bf8_t; -using BF16 = ck_tile::bf16_t; -using PkFP4 = ck_tile::pk_fp4_t; -using E8M0 = ck_tile::e8m0_t; -using BQuantGrouped = std::integral_constant; -using GroupSize64 = ck_tile::QuantGroupShape>; +using GroupSize64 = ck_tile::QuantGroupShape>; // Type combinations for BQuant tests - 1D GroupSize 64 // Tuple format: -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using FP16 = ck_tile::fp16_t; -using BF16 = ck_tile::bf16_t; -using PkFP4 = ck_tile::pk_fp4_t; -using E8M0 = ck_tile::e8m0_t; -using BQuantGrouped = std::integral_constant; -using GroupSize128 = ck_tile::QuantGroupShape>; +using GroupSize128 = ck_tile::QuantGroupShape>; // Type combinations for BQuant tests - 1D GroupSize 128 // Tuple format: -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using FP16 = ck_tile::fp16_t; -using BF16 = ck_tile::bf16_t; -using PkFP4 = ck_tile::pk_fp4_t; -using E8M0 = ck_tile::e8m0_t; -using BQuantGrouped = std::integral_constant; -using GroupSize64 = ck_tile::QuantGroupShape>; +using GroupSize64 = ck_tile::QuantGroupShape>; // Type combinations for BQuant tests - 1D GroupSize 64 // Tuple format: -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using BF8 = ck_tile::bf8_t; -using BF16 = ck_tile::bf16_t; -using PkFP4 = ck_tile::pk_fp4_t; -using E8M0 = ck_tile::e8m0_t; -using BQuantGrouped = std::integral_constant; -using GroupSize128 = ck_tile::QuantGroupShape>; +using GroupSize128 = ck_tile::QuantGroupShape>; // Type combinations for BQuant tests - 1D GroupSize 128 // Tuple format: -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using BF8 = ck_tile::bf8_t; -using BF16 = ck_tile::bf16_t; -using PkFP4 = ck_tile::pk_fp4_t; -using E8M0 = ck_tile::e8m0_t; -using BQuantGrouped = std::integral_constant; -using GroupSize64 = ck_tile::QuantGroupShape>; +using GroupSize64 = ck_tile::QuantGroupShape>; // Type combinations for BQuant tests - 1D GroupSize 64 // Tuple format: -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; +#include "test_gemm_quant_common.hpp" // Type combinations for BQuant Preshuffle tests - Decode Config 1D // Tuple format: // clang-format off using BPreshuffleDecode1DTypes = ::testing::Types< - std::tuple, - std::tuple + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffleQuant_decode_2d.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffleQuant_decode_2d.cpp index fb4020bcd7e4..54f71f7c4953 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffleQuant_decode_2d.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffleQuant_decode_2d.cpp @@ -1,24 +1,8 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; - -// 2d block sizes for BQuant using GroupSize2D8N = ck_tile::QuantGroupShape>; using GroupSize2D16N = ck_tile::QuantGroupShape>; using GroupSize2D32N = ck_tile::QuantGroupShape>; diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffleQuant_prefill_1d.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffleQuant_prefill_1d.cpp index 0d4e4d5f0346..a65c3ab1f072 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffleQuant_prefill_1d.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffleQuant_prefill_1d.cpp @@ -1,33 +1,17 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; +#include "test_gemm_quant_common.hpp" // Type combinations for BQuant Preshuffle tests - Prefill Config 1D // Tuple format: // clang-format off using BPreshufflePrefill1DTypes = ::testing::Types< - std::tuple, - std::tuple, - std::tuple, - std::tuple + std::tuple, + std::tuple, + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffleQuant_prefill_2d.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffleQuant_prefill_2d.cpp index edc7bcaa0905..93da8003ee59 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffleQuant_prefill_2d.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffleQuant_prefill_2d.cpp @@ -1,24 +1,8 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; - -// 2d block sizes for BQuant using GroupSize2D8N = ck_tile::QuantGroupShape>; using GroupSize2D16N = ck_tile::QuantGroupShape>; using GroupSize2D32N = ck_tile::QuantGroupShape>; diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_decode_1d.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_decode_1d.cpp index cf599ebbfde2..f23c2f8c418d 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_decode_1d.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_decode_1d.cpp @@ -1,31 +1,15 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; +#include "test_gemm_quant_common.hpp" // Type combinations for BQuant Preshuffle tests - Decode Config 1D // Tuple format: // clang-format off using BPreshuffleDecode1DTypes = ::testing::Types< - std::tuple, - std::tuple + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_decode_2d.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_decode_2d.cpp index 66fb62e67e21..cce983348090 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_decode_2d.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_decode_2d.cpp @@ -1,24 +1,8 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; - -// 2d block sizes for BQuant using GroupSize2D8N = ck_tile::QuantGroupShape>; using GroupSize2D16N = ck_tile::QuantGroupShape>; using GroupSize2D32N = ck_tile::QuantGroupShape>; diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_prefill_1d.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_prefill_1d.cpp index 3f6dd225d717..1b3025df07c1 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_prefill_1d.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_prefill_1d.cpp @@ -1,33 +1,17 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; +#include "test_gemm_quant_common.hpp" // Type combinations for BQuant Preshuffle tests - Prefill Config 1D // Tuple format: // clang-format off using BPreshufflePrefill1DTypes = ::testing::Types< - std::tuple, - std::tuple, - std::tuple, - std::tuple + std::tuple, + std::tuple, + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_prefill_2d.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_prefill_2d.cpp index ace07a37ae58..e4f11e587b70 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_prefill_2d.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_prefill_2d.cpp @@ -1,24 +1,8 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" +#include "test_gemm_quant_common.hpp" -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; - -// 2d block sizes for BQuant using GroupSize2D8N = ck_tile::QuantGroupShape>; using GroupSize2D16N = ck_tile::QuantGroupShape>; using GroupSize2D32N = ck_tile::QuantGroupShape>; diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_tiled_permute.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_tiled_permute.cpp index 8a05f5812a8f..8a54bf05f62c 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_tiled_permute.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_preshuffle_tiled_permute.cpp @@ -1,32 +1,16 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; +#include "test_gemm_quant_common.hpp" // Type combinations for BQuant Preshuffle tests - TiledPermuteN Config // Tuple format: // clang-format off using BPreshuffleTiledPermuteTypes = ::testing::Types< - std::tuple, - std::tuple, - std::tuple + std::tuple, + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_splitk_decode.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_splitk_decode.cpp index ea1a8a1fbbd7..7ab7d22dc7a3 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_splitk_decode.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_bquant_splitk_decode.cpp @@ -1,23 +1,9 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; -using GroupSize128 = ck_tile::QuantGroupShape>; +#include "test_gemm_quant_common.hpp" + +using GroupSize128 = ck_tile::QuantGroupShape>; // Type combinations for BQuant split-K tests - Decode shape, GroupSize 128 // Tuple format: -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; -using GroupSize128 = ck_tile::QuantGroupShape>; +using GroupSize128 = ck_tile::QuantGroupShape>; // Type combinations for BQuant split-K tests - Prefill shape, GroupSize 128 // Tuple format: -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using PkInt4 = ck_tile::pk_int4_t; -using BQuantGrouped = std::integral_constant; -using GroupSize64 = ck_tile::QuantGroupShape>; +using GroupSize64 = ck_tile::QuantGroupShape>; using GroupSize2D64N = ck_tile::QuantGroupShape>; // Type combinations for BQuant tests - Transpose Layouts diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_common.hpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_common.hpp new file mode 100644 index 000000000000..167e4afc8ccd --- /dev/null +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_common.hpp @@ -0,0 +1,40 @@ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +// Common includes for all gemm quant tests +#include "ck_tile/host.hpp" +#include "ck_tile/ops/gemm.hpp" + +#include +#include + +#include "test_gemm_quant_fixtures.hpp" + +// Common layout aliases +using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; +using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; + +// Common data type aliases +using Half = ck_tile::half_t; +using FP16 = ck_tile::fp16_t; +using BF16 = ck_tile::bf16_t; +using FP8 = ck_tile::fp8_t; +using BF8 = ck_tile::bf8_t; +using E8M0 = ck_tile::e8m0_t; +using PkInt4 = ck_tile::pk_int4_t; +using PkFP4 = ck_tile::pk_fp4_t; + +// Common quant type aliases +using AQuantGrouped = std::integral_constant; +using BQuantGrouped = std::integral_constant; +using ABQuantGrouped = + std::integral_constant; +using RowColQuant = std::integral_constant; +using TensorQuant = std::integral_constant; + +// Common group size aliases +using GroupSize1D_128 = ck_tile::QuantGroupShape>; +using GroupSize1D_64 = ck_tile::QuantGroupShape>; +using GroupSize2D = ck_tile::QuantGroupShape>; diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_rowcol.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_rowcol.cpp index bb0fa2189987..4e93bdf692cb 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_rowcol.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_rowcol.cpp @@ -1,30 +1,15 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using RowColQuant = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; +#include "test_gemm_quant_common.hpp" // Type combinations for RowColQuant tests // Tuple format: // clang-format off using RowColQuantTypes = ::testing::Types< - std::tuple, - std::tuple + std::tuple, + std::tuple >; // clang-format on diff --git a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_tensor.cpp b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_tensor.cpp index 8b4c90f8b925..ce7a2552d287 100644 --- a/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_tensor.cpp +++ b/projects/composablekernel/test/ck_tile/gemm_block_scale/test_gemm_quant_tensor.cpp @@ -1,30 +1,15 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -#include "ck_tile/host.hpp" -#include "ck_tile/ops/gemm.hpp" - -#include -#include - -#include "test_gemm_quant_fixtures.hpp" - -// Type aliases for readability -using RowMajor = ck_tile::tensor_layout::gemm::RowMajor; -using ColumnMajor = ck_tile::tensor_layout::gemm::ColumnMajor; -using FP8 = ck_tile::fp8_t; -using BF8 = ck_tile::bf8_t; -using Half = ck_tile::half_t; -using TensorQuant = std::integral_constant; -using GroupSize = ck_tile::QuantGroupShape>; +#include "test_gemm_quant_common.hpp" // Type combinations for TensorQuant tests // Tuple format: // clang-format off using TensorQuantTypes = ::testing::Types< - std::tuple, - std::tuple + std::tuple, + std::tuple >; // clang-format on From 8bf8a24e2d1482ee6cfcb156b79b5c4c40ee5fbd Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Sat, 11 Apr 2026 00:34:19 -0400 Subject: [PATCH 49/49] CK: Extract contraction example DeviceOpInstance boilerplate into macro (#6324) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Depends on #6323 ## Summary Extract repeated `DeviceOpInstance` type alias boilerplate from 20 contraction example files into a single macro `CK_CONTRACTION_DEVICE_OP_INSTANCES(BASE, SUFFIX)` in `common_instances.hpp`. Each example file's 4 device-op-instance blocks (~56 lines) are replaced by 4 one-line macro calls. | Metric | Value | |--------|-------| | Files changed | 22 | | Insertions | +124 | | Deletions | −1,140 | | **Net lines removed** | **−1,016** | ### What changed | Before | After | |--------|-------| | 20 example files, each with 4 identical `DeviceOpInstance` type alias blocks (~14 lines each) | 1 macro definition in `common_instances.hpp` + 20 files with 4 one-line macro calls each | ### Readability assessment A code realist review confirmed this change **improves readability**: the 14-line `DeviceOpInstance` blocks were pure noise — identical across all 20 files and obscuring the actual example logic (data types, element operations). After the macro, each file's intent is immediately clear from the 4 one-liner macro calls, and a developer adding a new contraction example only needs to specify the varying parameters instead of copying 56 lines of boilerplate. ### GPU verification All 20 contraction examples verified on MI300X: **20/20 passed**. ### Cumulative cleanup series stats | PR | Description | Net lines | |----|-------------|-----------| | #6300 | Remove 61 dead `#if 0` blocks | −2,648 | | #6302 | Remove 41 commented-out dead code blocks | −2,861 | | #6303 | Remove 4 orphaned files | −3,886 | | #6323 | Extract gemm_quant test boilerplate | −693 | | This PR | Extract contraction example boilerplate | −1,016 | | **Total** | | **−11,104** | --- .../26_contraction/common_instances.hpp | 32 ++++++++ .../contraction_bilinear_xdl_bf16.cpp | 60 +-------------- ...raction_bilinear_xdl_bf16_compute_fp32.cpp | 60 +-------------- .../contraction_bilinear_xdl_fp16.cpp | 60 +-------------- ...raction_bilinear_xdl_fp16_compute_fp32.cpp | 60 +-------------- .../contraction_bilinear_xdl_fp32.cpp | 60 +-------------- ...raction_bilinear_xdl_fp32_compute_bf16.cpp | 60 +-------------- ...raction_bilinear_xdl_fp32_compute_fp16.cpp | 60 +-------------- .../contraction_bilinear_xdl_fp64.cpp | 60 +-------------- ...raction_bilinear_xdl_fp64_compute_fp32.cpp | 60 +-------------- .../contraction_scale_xdl_bf16.cpp | 60 +-------------- ...ontraction_scale_xdl_bf16_compute_fp32.cpp | 60 +-------------- .../contraction_scale_xdl_fp16.cpp | 60 +-------------- ...ontraction_scale_xdl_fp16_compute_fp32.cpp | 60 +-------------- .../contraction_scale_xdl_fp32.cpp | 60 +-------------- ...ontraction_scale_xdl_fp32_compute_bf16.cpp | 60 +-------------- ...ontraction_scale_xdl_fp32_compute_fp16.cpp | 60 +-------------- .../contraction_scale_xdl_fp64.cpp | 60 +-------------- ...ontraction_scale_xdl_fp64_compute_fp32.cpp | 60 +-------------- .../common_instances.hpp | 32 ++++++++ .../complex_contraction_bilinear_xdl_fp32.cpp | 60 +-------------- .../complex_contraction_bilinear_xdl_fp64.cpp | 60 +-------------- .../gpu/contraction/CMakeLists.txt | 6 ++ .../contraction_instance_common.hpp | 77 +++++++++++++++++++ ...16_bf16_bf16_compute_f32_kknn_instance.cpp | 64 +++------------ ...16_bf16_bf16_compute_f32_knnn_instance.cpp | 64 +++------------ ...16_bf16_bf16_compute_f32_mknn_instance.cpp | 64 +++------------ ...16_bf16_bf16_compute_f32_mnnn_instance.cpp | 64 +++------------ ...ffle_bf16_bf16_bf16_bf16_kknn_instance.cpp | 64 +++------------ ...ffle_bf16_bf16_bf16_bf16_knnn_instance.cpp | 64 +++------------ ...ffle_bf16_bf16_bf16_bf16_mknn_instance.cpp | 64 +++------------ ...ffle_bf16_bf16_bf16_bf16_mnnn_instance.cpp | 64 +++------------ ..._f16_f16_f16_compute_f32_kknn_instance.cpp | 64 +++------------ ..._f16_f16_f16_compute_f32_knnn_instance.cpp | 64 +++------------ ..._f16_f16_f16_compute_f32_mknn_instance.cpp | 64 +++------------ ..._f16_f16_f16_compute_f32_mnnn_instance.cpp | 64 +++------------ ..._shuffle_f16_f16_f16_f16_kknn_instance.cpp | 64 +++------------ ..._shuffle_f16_f16_f16_f16_knnn_instance.cpp | 64 +++------------ ..._shuffle_f16_f16_f16_f16_mknn_instance.cpp | 64 +++------------ ..._shuffle_f16_f16_f16_f16_mnnn_instance.cpp | 64 +++------------ ...f32_f32_f32_compute_bf16_kknn_instance.cpp | 64 +++------------ ...f32_f32_f32_compute_bf16_knnn_instance.cpp | 64 +++------------ ...f32_f32_f32_compute_bf16_mknn_instance.cpp | 64 +++------------ ...f32_f32_f32_compute_bf16_mnnn_instance.cpp | 64 +++------------ ..._f32_f32_f32_compute_f16_kknn_instance.cpp | 64 +++------------ ..._f32_f32_f32_compute_f16_knnn_instance.cpp | 64 +++------------ ..._f32_f32_f32_compute_f16_mknn_instance.cpp | 64 +++------------ ..._f32_f32_f32_compute_f16_mnnn_instance.cpp | 64 +++------------ ..._shuffle_f32_f32_f32_f32_kknn_instance.cpp | 64 +++------------ ..._shuffle_f32_f32_f32_f32_knnn_instance.cpp | 64 +++------------ ..._shuffle_f32_f32_f32_f32_mknn_instance.cpp | 64 +++------------ ..._shuffle_f32_f32_f32_f32_mnnn_instance.cpp | 64 +++------------ ..._f64_f64_f64_compute_f32_kknn_instance.cpp | 64 +++------------ ..._f64_f64_f64_compute_f32_knnn_instance.cpp | 64 +++------------ ..._f64_f64_f64_compute_f32_mknn_instance.cpp | 64 +++------------ ..._f64_f64_f64_compute_f32_mnnn_instance.cpp | 64 +++------------ ..._shuffle_f64_f64_f64_f64_kknn_instance.cpp | 64 +++------------ ..._shuffle_f64_f64_f64_f64_knnn_instance.cpp | 64 +++------------ ..._shuffle_f64_f64_f64_f64_mknn_instance.cpp | 64 +++------------ ..._shuffle_f64_f64_f64_f64_mnnn_instance.cpp | 64 +++------------ ...16_bf16_bf16_compute_f32_kknn_instance.cpp | 64 +++------------ ...16_bf16_bf16_compute_f32_knnn_instance.cpp | 64 +++------------ ...16_bf16_bf16_compute_f32_mknn_instance.cpp | 64 +++------------ ...16_bf16_bf16_compute_f32_mnnn_instance.cpp | 64 +++------------ ...ffle_bf16_bf16_bf16_bf16_kknn_instance.cpp | 64 +++------------ ...ffle_bf16_bf16_bf16_bf16_knnn_instance.cpp | 64 +++------------ ...ffle_bf16_bf16_bf16_bf16_mknn_instance.cpp | 64 +++------------ ...ffle_bf16_bf16_bf16_bf16_mnnn_instance.cpp | 64 +++------------ ..._f16_f16_f16_compute_f32_kknn_instance.cpp | 64 +++------------ ..._f16_f16_f16_compute_f32_knnn_instance.cpp | 64 +++------------ ..._f16_f16_f16_compute_f32_mknn_instance.cpp | 64 +++------------ ..._f16_f16_f16_compute_f32_mnnn_instance.cpp | 64 +++------------ ..._shuffle_f16_f16_f16_f16_kknn_instance.cpp | 66 +++------------- ..._shuffle_f16_f16_f16_f16_knnn_instance.cpp | 64 +++------------ ..._shuffle_f16_f16_f16_f16_mknn_instance.cpp | 64 +++------------ ..._shuffle_f16_f16_f16_f16_mnnn_instance.cpp | 64 +++------------ ...f32_f32_f32_compute_bf16_kknn_instance.cpp | 64 +++------------ ...f32_f32_f32_compute_bf16_knnn_instance.cpp | 64 +++------------ ...f32_f32_f32_compute_bf16_mknn_instance.cpp | 64 +++------------ ...f32_f32_f32_compute_bf16_mnnn_instance.cpp | 64 +++------------ ..._f32_f32_f32_compute_f16_kknn_instance.cpp | 64 +++------------ ..._f32_f32_f32_compute_f16_knnn_instance.cpp | 64 +++------------ ..._f32_f32_f32_compute_f16_mknn_instance.cpp | 64 +++------------ ..._f32_f32_f32_compute_f16_mnnn_instance.cpp | 64 +++------------ ..._shuffle_f32_f32_f32_f32_kknn_instance.cpp | 64 +++------------ ..._shuffle_f32_f32_f32_f32_knnn_instance.cpp | 64 +++------------ ..._shuffle_f32_f32_f32_f32_mknn_instance.cpp | 64 +++------------ ..._shuffle_f32_f32_f32_f32_mnnn_instance.cpp | 64 +++------------ ..._f64_f64_f64_compute_f32_kknn_instance.cpp | 64 +++------------ ..._f64_f64_f64_compute_f32_knnn_instance.cpp | 64 +++------------ ..._f64_f64_f64_compute_f32_mknn_instance.cpp | 64 +++------------ ..._f64_f64_f64_compute_f32_mnnn_instance.cpp | 64 +++------------ ..._shuffle_f64_f64_f64_f64_kknn_instance.cpp | 64 +++------------ ..._shuffle_f64_f64_f64_f64_knnn_instance.cpp | 64 +++------------ ..._shuffle_f64_f64_f64_f64_mknn_instance.cpp | 64 +++------------ ..._shuffle_f64_f64_f64_f64_mnnn_instance.cpp | 64 +++------------ ...f16_bf16_bf16_compute_f32_kkn_instance.cpp | 64 +++------------ ...f16_bf16_bf16_compute_f32_knn_instance.cpp | 64 +++------------ ...f16_bf16_bf16_compute_f32_mkn_instance.cpp | 64 +++------------ ...f16_bf16_bf16_compute_f32_mnn_instance.cpp | 64 +++------------ ..._c_shuffle_bf16_bf16_bf16_kkn_instance.cpp | 63 +++------------ ..._c_shuffle_bf16_bf16_bf16_knn_instance.cpp | 63 +++------------ ..._c_shuffle_bf16_bf16_bf16_mkn_instance.cpp | 63 +++------------ ..._c_shuffle_bf16_bf16_bf16_mnn_instance.cpp | 63 +++------------ ...e_f16_f16_f16_compute_f32_kkn_instance.cpp | 64 +++------------ ...e_f16_f16_f16_compute_f32_knn_instance.cpp | 64 +++------------ ...e_f16_f16_f16_compute_f32_mkn_instance.cpp | 64 +++------------ ...e_f16_f16_f16_compute_f32_mnn_instance.cpp | 64 +++------------ ...xdl_c_shuffle_f16_f16_f16_kkn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f16_f16_f16_knn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f16_f16_f16_mkn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f16_f16_f16_mnn_instance.cpp | 63 +++------------ ..._f32_f32_f32_compute_bf16_kkn_instance.cpp | 64 +++------------ ..._f32_f32_f32_compute_bf16_knn_instance.cpp | 64 +++------------ ..._f32_f32_f32_compute_bf16_mkn_instance.cpp | 64 +++------------ ..._f32_f32_f32_compute_bf16_mnn_instance.cpp | 64 +++------------ ...e_f32_f32_f32_compute_f16_kkn_instance.cpp | 64 +++------------ ...e_f32_f32_f32_compute_f16_knn_instance.cpp | 64 +++------------ ...e_f32_f32_f32_compute_f16_mkn_instance.cpp | 64 +++------------ ...e_f32_f32_f32_compute_f16_mnn_instance.cpp | 64 +++------------ ...xdl_c_shuffle_f32_f32_f32_kkn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f32_f32_f32_knn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f32_f32_f32_mkn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f32_f32_f32_mnn_instance.cpp | 63 +++------------ ...e_f64_f64_f64_compute_f32_kkn_instance.cpp | 64 +++------------ ...e_f64_f64_f64_compute_f32_knn_instance.cpp | 64 +++------------ ...e_f64_f64_f64_compute_f32_mkn_instance.cpp | 64 +++------------ ...e_f64_f64_f64_compute_f32_mnn_instance.cpp | 64 +++------------ ...xdl_c_shuffle_f64_f64_f64_kkn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f64_f64_f64_knn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f64_f64_f64_mkn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f64_f64_f64_mnn_instance.cpp | 63 +++------------ ...f16_bf16_bf16_compute_f32_kkn_instance.cpp | 64 +++------------ ...f16_bf16_bf16_compute_f32_knn_instance.cpp | 64 +++------------ ...f16_bf16_bf16_compute_f32_mkn_instance.cpp | 64 +++------------ ...f16_bf16_bf16_compute_f32_mnn_instance.cpp | 64 +++------------ ..._c_shuffle_bf16_bf16_bf16_kkn_instance.cpp | 63 +++------------ ..._c_shuffle_bf16_bf16_bf16_knn_instance.cpp | 63 +++------------ ..._c_shuffle_bf16_bf16_bf16_mkn_instance.cpp | 63 +++------------ ..._c_shuffle_bf16_bf16_bf16_mnn_instance.cpp | 63 +++------------ ...e_f16_f16_f16_compute_f32_kkn_instance.cpp | 64 +++------------ ...e_f16_f16_f16_compute_f32_knn_instance.cpp | 64 +++------------ ...e_f16_f16_f16_compute_f32_mkn_instance.cpp | 64 +++------------ ...e_f16_f16_f16_compute_f32_mnn_instance.cpp | 64 +++------------ ...xdl_c_shuffle_f16_f16_f16_kkn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f16_f16_f16_knn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f16_f16_f16_mkn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f16_f16_f16_mnn_instance.cpp | 63 +++------------ ..._f32_f32_f32_compute_bf16_kkn_instance.cpp | 64 +++------------ ..._f32_f32_f32_compute_bf16_knn_instance.cpp | 64 +++------------ ..._f32_f32_f32_compute_bf16_mkn_instance.cpp | 64 +++------------ ..._f32_f32_f32_compute_bf16_mnn_instance.cpp | 64 +++------------ ...e_f32_f32_f32_compute_f16_kkn_instance.cpp | 64 +++------------ ...e_f32_f32_f32_compute_f16_knn_instance.cpp | 64 +++------------ ...e_f32_f32_f32_compute_f16_mkn_instance.cpp | 64 +++------------ ...e_f32_f32_f32_compute_f16_mnn_instance.cpp | 64 +++------------ ...xdl_c_shuffle_f32_f32_f32_kkn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f32_f32_f32_knn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f32_f32_f32_mkn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f32_f32_f32_mnn_instance.cpp | 63 +++------------ ...e_f64_f64_f64_compute_f32_kkn_instance.cpp | 64 +++------------ ...e_f64_f64_f64_compute_f32_knn_instance.cpp | 64 +++------------ ...e_f64_f64_f64_compute_f32_mkn_instance.cpp | 64 +++------------ ...e_f64_f64_f64_compute_f32_mnn_instance.cpp | 64 +++------------ ...xdl_c_shuffle_f64_f64_f64_kkn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f64_f64_f64_knn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f64_f64_f64_mkn_instance.cpp | 63 +++------------ ...xdl_c_shuffle_f64_f64_f64_mnn_instance.cpp | 63 +++------------ 168 files changed, 1503 insertions(+), 9030 deletions(-) create mode 100644 projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction/CMakeLists.txt create mode 100644 projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction/contraction_instance_common.hpp diff --git a/projects/composablekernel/example/26_contraction/common_instances.hpp b/projects/composablekernel/example/26_contraction/common_instances.hpp index 457bae21aaef..808c54804200 100644 --- a/projects/composablekernel/example/26_contraction/common_instances.hpp +++ b/projects/composablekernel/example/26_contraction/common_instances.hpp @@ -194,3 +194,35 @@ using DeviceOpInstanceMN_FP64 = ck::tensor_operation::device:: //#####################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | DeviceContractionMultipleD_Xdl_CShuffle< NumDimM, NumDimN, NumDimK, ADataType, BDataType, AccDataType, CShuffleDataType, DsDataType, EDataType, AElementOp, BElementOp, CDEElementOp, GemmSpec, 1, 256, 128, 128, 16, 1, 1, 16, 16, 4, 4, S<4, 64, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 2, 1, 0, S<8, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 2, 1, 0, 1, 1, S<1, 16, 1, 16>, 1, ComputeDataType>; // clang-format on + +// Macro to instantiate all four layout variants of DeviceOpInstance. +// +// BASE: Generic (for fp16/bf16/fp32) or FP64 (for fp64 — different tile sizes) +// SUFFIX: NN for bilinear (DsDataType = Tuple), +// N for scale (DsDataType = Tuple<>) +// +// Requires these names to be defined in the calling TU before invocation: +// NumDimM, NumDimN, NumDimK, ADataType, BDataType, AccDataType, +// CShuffleDataType, DsDataType, EDataType, ComputeDataType, +// AElementOp, BElementOp, CDEElementOp +// +// Example: CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, NN); +// expands to DeviceOpInstanceKKNN, DeviceOpInstanceKNNN, +// DeviceOpInstanceMKNN, DeviceOpInstanceMNNN, +// and sets DeviceOpInstance = DeviceOpInstanceKKNN. +// clang-format off +#define CK_CONTRACTION_DEVICE_OP_INSTANCES(BASE, SUFFIX) \ + using DeviceOpInstanceKK##SUFFIX = DeviceOpInstanceKK_##BASE; \ + using DeviceOpInstanceKN##SUFFIX = DeviceOpInstanceKN_##BASE; \ + using DeviceOpInstanceMK##SUFFIX = DeviceOpInstanceMK_##BASE; \ + using DeviceOpInstanceMN##SUFFIX = DeviceOpInstanceMN_##BASE; \ + using DeviceOpInstance = DeviceOpInstanceKK##SUFFIX +// clang-format on diff --git a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_bf16.cpp b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_bf16.cpp index 8899b54fbf92..b5758ed428bd 100644 --- a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_bf16.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_bf16.cpp @@ -23,63 +23,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Bilinear; -using DeviceOpInstanceKKNN = DeviceOpInstanceKK_Generic; - -using DeviceOpInstanceKNNN = DeviceOpInstanceKN_Generic; - -using DeviceOpInstanceMKNN = DeviceOpInstanceMK_Generic; - -using DeviceOpInstanceMNNN = DeviceOpInstanceMN_Generic; - -using DeviceOpInstance = DeviceOpInstanceKKNN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, NN); #include "run_contraction_bilinear_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_bf16_compute_fp32.cpp b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_bf16_compute_fp32.cpp index 2dac449e99f4..be03613bd15d 100644 --- a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_bf16_compute_fp32.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_bf16_compute_fp32.cpp @@ -23,63 +23,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Bilinear; -using DeviceOpInstanceKKNN = DeviceOpInstanceKK_Generic; - -using DeviceOpInstanceKNNN = DeviceOpInstanceKN_Generic; - -using DeviceOpInstanceMKNN = DeviceOpInstanceMK_Generic; - -using DeviceOpInstanceMNNN = DeviceOpInstanceMN_Generic; - -using DeviceOpInstance = DeviceOpInstanceKKNN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, NN); #include "run_contraction_bilinear_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp16.cpp b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp16.cpp index 16e33e088602..5d6d401836bb 100644 --- a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp16.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp16.cpp @@ -23,63 +23,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Bilinear; -using DeviceOpInstanceKKNN = DeviceOpInstanceKK_Generic; - -using DeviceOpInstanceKNNN = DeviceOpInstanceKN_Generic; - -using DeviceOpInstanceMKNN = DeviceOpInstanceMK_Generic; - -using DeviceOpInstanceMNNN = DeviceOpInstanceMN_Generic; - -using DeviceOpInstance = DeviceOpInstanceKKNN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, NN); #include "run_contraction_bilinear_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp16_compute_fp32.cpp b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp16_compute_fp32.cpp index 494670bcca82..ded63dec25bd 100644 --- a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp16_compute_fp32.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp16_compute_fp32.cpp @@ -23,63 +23,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Bilinear; -using DeviceOpInstanceKKNN = DeviceOpInstanceKK_Generic; - -using DeviceOpInstanceKNNN = DeviceOpInstanceKN_Generic; - -using DeviceOpInstanceMKNN = DeviceOpInstanceMK_Generic; - -using DeviceOpInstanceMNNN = DeviceOpInstanceMN_Generic; - -using DeviceOpInstance = DeviceOpInstanceKKNN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, NN); #include "run_contraction_bilinear_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp32.cpp b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp32.cpp index e960199fc3f9..8779e1fab9b7 100644 --- a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp32.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp32.cpp @@ -23,63 +23,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Bilinear; -using DeviceOpInstanceKKNN = DeviceOpInstanceKK_Generic; - -using DeviceOpInstanceKNNN = DeviceOpInstanceKN_Generic; - -using DeviceOpInstanceMKNN = DeviceOpInstanceMK_Generic; - -using DeviceOpInstanceMNNN = DeviceOpInstanceMN_Generic; - -using DeviceOpInstance = DeviceOpInstanceKKNN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, NN); #include "run_contraction_bilinear_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp32_compute_bf16.cpp b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp32_compute_bf16.cpp index 2963152eb1d2..467672986ecb 100644 --- a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp32_compute_bf16.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp32_compute_bf16.cpp @@ -23,63 +23,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Bilinear; -using DeviceOpInstanceKKNN = DeviceOpInstanceKK_Generic; - -using DeviceOpInstanceKNNN = DeviceOpInstanceKN_Generic; - -using DeviceOpInstanceMKNN = DeviceOpInstanceMK_Generic; - -using DeviceOpInstanceMNNN = DeviceOpInstanceMN_Generic; - -using DeviceOpInstance = DeviceOpInstanceKKNN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, NN); #include "run_contraction_bilinear_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp32_compute_fp16.cpp b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp32_compute_fp16.cpp index 01966960ccae..dff5a0446a25 100644 --- a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp32_compute_fp16.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp32_compute_fp16.cpp @@ -23,63 +23,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Bilinear; -using DeviceOpInstanceKKNN = DeviceOpInstanceKK_Generic; - -using DeviceOpInstanceKNNN = DeviceOpInstanceKN_Generic; - -using DeviceOpInstanceMKNN = DeviceOpInstanceMK_Generic; - -using DeviceOpInstanceMNNN = DeviceOpInstanceMN_Generic; - -using DeviceOpInstance = DeviceOpInstanceKKNN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, NN); #include "run_contraction_bilinear_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp64.cpp b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp64.cpp index 1ea9bcedfd02..2d697f3e0716 100644 --- a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp64.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp64.cpp @@ -23,63 +23,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Bilinear; -using DeviceOpInstanceKKNN = DeviceOpInstanceKK_FP64; - -using DeviceOpInstanceKNNN = DeviceOpInstanceKN_FP64; - -using DeviceOpInstanceMKNN = DeviceOpInstanceMK_FP64; - -using DeviceOpInstanceMNNN = DeviceOpInstanceMN_FP64; - -using DeviceOpInstance = DeviceOpInstanceKKNN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(FP64, NN); #include "run_contraction_bilinear_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp64_compute_fp32.cpp b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp64_compute_fp32.cpp index 9e40e28485a2..341dad6d5b01 100644 --- a/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp64_compute_fp32.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_bilinear_xdl_fp64_compute_fp32.cpp @@ -23,63 +23,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Bilinear; -using DeviceOpInstanceKKNN = DeviceOpInstanceKK_FP64; - -using DeviceOpInstanceKNNN = DeviceOpInstanceKN_FP64; - -using DeviceOpInstanceMKNN = DeviceOpInstanceMK_FP64; - -using DeviceOpInstanceMNNN = DeviceOpInstanceMN_FP64; - -using DeviceOpInstance = DeviceOpInstanceKKNN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(FP64, NN); #include "run_contraction_bilinear_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_bf16.cpp b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_bf16.cpp index 586b02239713..003bc0274a6a 100644 --- a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_bf16.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_bf16.cpp @@ -22,63 +22,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Scale; -using DeviceOpInstanceKKN = DeviceOpInstanceKK_Generic; - -using DeviceOpInstanceKNN = DeviceOpInstanceKN_Generic; - -using DeviceOpInstanceMKN = DeviceOpInstanceMK_Generic; - -using DeviceOpInstanceMNN = DeviceOpInstanceMN_Generic; - -using DeviceOpInstance = DeviceOpInstanceKKN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, N); #include "run_contraction_scale_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_bf16_compute_fp32.cpp b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_bf16_compute_fp32.cpp index 9e4a02967a04..bada39204e80 100644 --- a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_bf16_compute_fp32.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_bf16_compute_fp32.cpp @@ -22,63 +22,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Scale; -using DeviceOpInstanceKKN = DeviceOpInstanceKK_Generic; - -using DeviceOpInstanceKNN = DeviceOpInstanceKN_Generic; - -using DeviceOpInstanceMKN = DeviceOpInstanceMK_Generic; - -using DeviceOpInstanceMNN = DeviceOpInstanceMN_Generic; - -using DeviceOpInstance = DeviceOpInstanceKKN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, N); #include "run_contraction_scale_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp16.cpp b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp16.cpp index 1f29e16223d5..4f3adef47af3 100644 --- a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp16.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp16.cpp @@ -22,63 +22,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Scale; -using DeviceOpInstanceKKN = DeviceOpInstanceKK_Generic; - -using DeviceOpInstanceKNN = DeviceOpInstanceKN_Generic; - -using DeviceOpInstanceMKN = DeviceOpInstanceMK_Generic; - -using DeviceOpInstanceMNN = DeviceOpInstanceMN_Generic; - -using DeviceOpInstance = DeviceOpInstanceKKN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, N); #include "run_contraction_scale_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp16_compute_fp32.cpp b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp16_compute_fp32.cpp index 878011afd15b..9be3b616f65e 100644 --- a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp16_compute_fp32.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp16_compute_fp32.cpp @@ -22,63 +22,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Scale; -using DeviceOpInstanceKKN = DeviceOpInstanceKK_Generic; - -using DeviceOpInstanceKNN = DeviceOpInstanceKN_Generic; - -using DeviceOpInstanceMKN = DeviceOpInstanceMK_Generic; - -using DeviceOpInstanceMNN = DeviceOpInstanceMN_Generic; - -using DeviceOpInstance = DeviceOpInstanceKKN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, N); #include "run_contraction_scale_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp32.cpp b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp32.cpp index 5d8aa7b9c5d1..d7754ef5462e 100644 --- a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp32.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp32.cpp @@ -22,63 +22,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Scale; -using DeviceOpInstanceKKN = DeviceOpInstanceKK_Generic; - -using DeviceOpInstanceKNN = DeviceOpInstanceKN_Generic; - -using DeviceOpInstanceMKN = DeviceOpInstanceMK_Generic; - -using DeviceOpInstanceMNN = DeviceOpInstanceMN_Generic; - -using DeviceOpInstance = DeviceOpInstanceKKN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, N); #include "run_contraction_scale_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp32_compute_bf16.cpp b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp32_compute_bf16.cpp index 57b1052a8303..deaf7e7bdce5 100644 --- a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp32_compute_bf16.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp32_compute_bf16.cpp @@ -22,63 +22,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Scale; -using DeviceOpInstanceKKN = DeviceOpInstanceKK_Generic; - -using DeviceOpInstanceKNN = DeviceOpInstanceKN_Generic; - -using DeviceOpInstanceMKN = DeviceOpInstanceMK_Generic; - -using DeviceOpInstanceMNN = DeviceOpInstanceMN_Generic; - -using DeviceOpInstance = DeviceOpInstanceKKN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, N); #include "run_contraction_scale_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp32_compute_fp16.cpp b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp32_compute_fp16.cpp index ae23986bc945..de5209671251 100644 --- a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp32_compute_fp16.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp32_compute_fp16.cpp @@ -22,63 +22,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Scale; -using DeviceOpInstanceKKN = DeviceOpInstanceKK_Generic; - -using DeviceOpInstanceKNN = DeviceOpInstanceKN_Generic; - -using DeviceOpInstanceMKN = DeviceOpInstanceMK_Generic; - -using DeviceOpInstanceMNN = DeviceOpInstanceMN_Generic; - -using DeviceOpInstance = DeviceOpInstanceKKN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, N); #include "run_contraction_scale_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp64.cpp b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp64.cpp index 66f22ce63cfa..3d5d23968fed 100644 --- a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp64.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp64.cpp @@ -22,63 +22,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Scale; -using DeviceOpInstanceKKN = DeviceOpInstanceKK_FP64; - -using DeviceOpInstanceKNN = DeviceOpInstanceKN_FP64; - -using DeviceOpInstanceMKN = DeviceOpInstanceMK_FP64; - -using DeviceOpInstanceMNN = DeviceOpInstanceMN_FP64; - -using DeviceOpInstance = DeviceOpInstanceKKN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(FP64, N); #include "run_contraction_scale_example.inc" diff --git a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp64_compute_fp32.cpp b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp64_compute_fp32.cpp index 2d72be815728..ee2533ca0ad5 100644 --- a/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp64_compute_fp32.cpp +++ b/projects/composablekernel/example/26_contraction/contraction_scale_xdl_fp64_compute_fp32.cpp @@ -22,63 +22,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Scale; -using DeviceOpInstanceKKN = DeviceOpInstanceKK_FP64; - -using DeviceOpInstanceKNN = DeviceOpInstanceKN_FP64; - -using DeviceOpInstanceMKN = DeviceOpInstanceMK_FP64; - -using DeviceOpInstanceMNN = DeviceOpInstanceMN_FP64; - -using DeviceOpInstance = DeviceOpInstanceKKN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(FP64, N); #include "run_contraction_scale_example.inc" diff --git a/projects/composablekernel/example/66_complex_contraction_bilinear/common_instances.hpp b/projects/composablekernel/example/66_complex_contraction_bilinear/common_instances.hpp index cb6157b29b6d..3ae168cb727b 100644 --- a/projects/composablekernel/example/66_complex_contraction_bilinear/common_instances.hpp +++ b/projects/composablekernel/example/66_complex_contraction_bilinear/common_instances.hpp @@ -194,3 +194,35 @@ using DeviceOpInstanceMN_FP64 = ck::tensor_operation::device:: //#####################################| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | DeviceContractionMultipleD_Xdl_CShuffle< NumDimM, NumDimN, NumDimK, ADataType, BDataType, AccDataType, CShuffleDataType, DsDataType, EDataType, AElementOp, BElementOp, CDEElementOp, GemmSpec, 1, 256, 128, 128, 16, 1, 1, 16, 16, 4, 4, S<4, 64, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 2, 1, 0, S<8, 32, 1>, S<0, 2, 1>, S<0, 2, 1>, 1, 2, 1, 0, 1, 1, S<1, 16, 1, 16>, 1, ComputeDataType>; // clang-format on + +// Macro to instantiate all four layout variants of DeviceOpInstance. +// +// BASE: Generic (for fp16/bf16/fp32) or FP64 (for fp64 — different tile sizes) +// SUFFIX: NN for bilinear (DsDataType = Tuple), +// N for scale (DsDataType = Tuple<>) +// +// Requires these names to be defined in the calling TU before invocation: +// NumDimM, NumDimN, NumDimK, ADataType, BDataType, AccDataType, +// CShuffleDataType, DsDataType, EDataType, ComputeDataType, +// AElementOp, BElementOp, CDEElementOp +// +// Example: CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, NN); +// expands to DeviceOpInstanceKKNN, DeviceOpInstanceKNNN, +// DeviceOpInstanceMKNN, DeviceOpInstanceMNNN, +// and sets DeviceOpInstance = DeviceOpInstanceKKNN. +// clang-format off +#define CK_CONTRACTION_DEVICE_OP_INSTANCES(BASE, SUFFIX) \ + using DeviceOpInstanceKK##SUFFIX = DeviceOpInstanceKK_##BASE; \ + using DeviceOpInstanceKN##SUFFIX = DeviceOpInstanceKN_##BASE; \ + using DeviceOpInstanceMK##SUFFIX = DeviceOpInstanceMK_##BASE; \ + using DeviceOpInstanceMN##SUFFIX = DeviceOpInstanceMN_##BASE; \ + using DeviceOpInstance = DeviceOpInstanceKK##SUFFIX +// clang-format on diff --git a/projects/composablekernel/example/66_complex_contraction_bilinear/complex_contraction_bilinear_xdl_fp32.cpp b/projects/composablekernel/example/66_complex_contraction_bilinear/complex_contraction_bilinear_xdl_fp32.cpp index e2cae7a1f8cb..7533281f1a25 100644 --- a/projects/composablekernel/example/66_complex_contraction_bilinear/complex_contraction_bilinear_xdl_fp32.cpp +++ b/projects/composablekernel/example/66_complex_contraction_bilinear/complex_contraction_bilinear_xdl_fp32.cpp @@ -23,63 +23,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Bilinear; -using DeviceOpInstanceKKNN = DeviceOpInstanceKK_Generic; - -using DeviceOpInstanceKNNN = DeviceOpInstanceKN_Generic; - -using DeviceOpInstanceMKNN = DeviceOpInstanceMK_Generic; - -using DeviceOpInstanceMNNN = DeviceOpInstanceMN_Generic; - -using DeviceOpInstance = DeviceOpInstanceKKNN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(Generic, NN); #include "run_complex_contraction_bilinear_example.inc" diff --git a/projects/composablekernel/example/66_complex_contraction_bilinear/complex_contraction_bilinear_xdl_fp64.cpp b/projects/composablekernel/example/66_complex_contraction_bilinear/complex_contraction_bilinear_xdl_fp64.cpp index a2021b5eaae1..a41e1f178551 100644 --- a/projects/composablekernel/example/66_complex_contraction_bilinear/complex_contraction_bilinear_xdl_fp64.cpp +++ b/projects/composablekernel/example/66_complex_contraction_bilinear/complex_contraction_bilinear_xdl_fp64.cpp @@ -23,63 +23,9 @@ using AElementOp = ck::tensor_operation::element_wise::PassThrough; using BElementOp = ck::tensor_operation::element_wise::PassThrough; using CDEElementOp = ck::tensor_operation::element_wise::Bilinear; -using DeviceOpInstanceKKNN = DeviceOpInstanceKK_FP64; - -using DeviceOpInstanceKNNN = DeviceOpInstanceKN_FP64; - -using DeviceOpInstanceMKNN = DeviceOpInstanceMK_FP64; - -using DeviceOpInstanceMNNN = DeviceOpInstanceMN_FP64; - -using DeviceOpInstance = DeviceOpInstanceKKNN; +// Instantiate DeviceOpInstance for all four layout variants (KK, KN, MK, MN). +// See common_instances.hpp for macro definition and available BASE/SUFFIX options. +CK_CONTRACTION_DEVICE_OP_INSTANCES(FP64, NN); #include "run_complex_contraction_bilinear_example.inc" diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction/CMakeLists.txt b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction/CMakeLists.txt new file mode 100644 index 000000000000..cd0d93c5e901 --- /dev/null +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction/CMakeLists.txt @@ -0,0 +1,6 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +# This directory contains only shared header files (contraction_instance_common.hpp). +# There are no source files to compile here — the header is included by the +# contraction_bilinear/ and contraction_scale/ instance directories. diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction/contraction_instance_common.hpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction/contraction_instance_common.hpp new file mode 100644 index 000000000000..e9f838107e00 --- /dev/null +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction/contraction_instance_common.hpp @@ -0,0 +1,77 @@ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +#pragma once + +// This (ifndef) is a hack to use customized behavior for buffer load rather than using default +// setting Don't use this hack unless absolutely necessary! +// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op +#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 + +#include + +#include "ck/ck.hpp" +#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" +#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" +#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" +#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" + +// Macro to generate a contraction device operation instance definition and its +// registration function. Each invocation produces one using-alias and one +// add_device_* function inside ck::tensor_operation::device::instance. +// +// Parameters: +// INST_TPL — instance template (e.g. device_contraction_kk_instance, +// device_contraction_f64_kk_instance) +// OP_NAME — lowercase operation name for identifier construction +// (bilinear or scale) +// CDE_OP — C++ element-wise operation type for template argument +// (Bilinear or Scale) +// NDIM_VAL — number of dimensions (2 or 6) +// NAME_SUFFIX — data-type and layout suffix for the generated names +// (e.g. f32_f32_f32_f32_kknn, bf16_bf16_bf16_bf16_compute_f32_knnn) +// ADATA — ADataType +// BDATA — BDataType +// ACC — AccDataType +// CSHUFFLE — CShuffleDataType +// DS_TUPLE — DsDataType (e.g. F32_Tuple, Empty_Tuple) +// EDATA — EDataType +// COMPUTE — ComputeDataType +// +// Example — bilinear, F32, kk layout, 2D: +// +// CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, +// bilinear, Bilinear, 2, f32_f32_f32_f32_kknn, +// F32, F32, F32, F32, F32_Tuple, F32, F32) +// +// Expands to: +// using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance = ...; +// void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance(...) +// { ... } +// +// clang-format off +#define CK_CONTRACTION_INSTANCE(INST_TPL, OP_NAME, CDE_OP, NDIM_VAL, \ + NAME_SUFFIX, ADATA, BDATA, ACC, CSHUFFLE, DS_TUPLE, EDATA, COMPUTE) \ + \ +namespace ck { \ +namespace tensor_operation { \ +namespace device { \ +namespace instance { \ + \ +using device_contraction_##OP_NAME##_m##NDIM_VAL##_n##NDIM_VAL##_k##NDIM_VAL##_xdl_c_shuffle_##NAME_SUFFIX##_instance = \ + INST_TPL; \ + \ +void add_device_contraction_##OP_NAME##_m##NDIM_VAL##_n##NDIM_VAL##_k##NDIM_VAL##_xdl_c_shuffle_##NAME_SUFFIX##_instance( \ + std::vector>>& instances) \ +{ \ + add_device_operation_instances(instances, \ + device_contraction_##OP_NAME##_m##NDIM_VAL##_n##NDIM_VAL##_k##NDIM_VAL##_xdl_c_shuffle_##NAME_SUFFIX##_instance{}); \ +} \ + \ +} /* namespace instance */ \ +} /* namespace device */ \ +} /* namespace tensor_operation */ \ +} /* namespace ck */ +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_kknn_instance.cpp index c8f6053c44ed..1a4ce88a3926 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_kknn_instance = - device_contraction_kk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + bilinear, Bilinear, 2, bf16_bf16_bf16_bf16_compute_f32_kknn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_knnn_instance.cpp index fb1002f1aaf9..cdfcab69afd8 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_knnn_instance = - device_contraction_kn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + bilinear, Bilinear, 2, bf16_bf16_bf16_bf16_compute_f32_knnn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mknn_instance.cpp index 5918beb9ad43..b1ca1603b4fd 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mknn_instance = - device_contraction_mk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + bilinear, Bilinear, 2, bf16_bf16_bf16_bf16_compute_f32_mknn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mnnn_instance.cpp index fccd91e5be93..bd7f73d2ed8b 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mnnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + bilinear, Bilinear, 2, bf16_bf16_bf16_bf16_compute_f32_mnnn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_kknn_instance.cpp index ce57ee2d0790..964d2a069038 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_kknn_instance = - device_contraction_kk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + bilinear, Bilinear, 2, bf16_bf16_bf16_bf16_kknn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_knnn_instance.cpp index e1e5dbb434d8..ac8ac661e391 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_knnn_instance = - device_contraction_kn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + bilinear, Bilinear, 2, bf16_bf16_bf16_bf16_knnn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_mknn_instance.cpp index db9840639010..281673f6a8c2 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_mknn_instance = - device_contraction_mk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + bilinear, Bilinear, 2, bf16_bf16_bf16_bf16_mknn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_mnnn_instance.cpp index 5c7032e8547f..3ac1cef7beec 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_mnnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_bf16_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + bilinear, Bilinear, 2, bf16_bf16_bf16_bf16_mnnn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_kknn_instance.cpp index 89cb35495b9b..5b410c24a078 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_kknn_instance = - device_contraction_kk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + bilinear, Bilinear, 2, f16_f16_f16_f16_compute_f32_kknn, + F16, F16, F32, F16, F16_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_knnn_instance.cpp index c25ebfb59875..9982149b2e0c 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_knnn_instance = - device_contraction_kn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + bilinear, Bilinear, 2, f16_f16_f16_f16_compute_f32_knnn, + F16, F16, F32, F16, F16_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mknn_instance.cpp index 9815d2f4e38f..0b6f0a858990 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mknn_instance = - device_contraction_mk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + bilinear, Bilinear, 2, f16_f16_f16_f16_compute_f32_mknn, + F16, F16, F32, F16, F16_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mnnn_instance.cpp index c1735b1fe1af..a2092c8c5c3a 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mnnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + bilinear, Bilinear, 2, f16_f16_f16_f16_compute_f32_mnnn, + F16, F16, F32, F16, F16_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_kknn_instance.cpp index a0c8376980b3..188a674c3f26 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_kknn_instance = - device_contraction_kk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + bilinear, Bilinear, 2, f16_f16_f16_f16_kknn, + F16, F16, F32, F16, F16_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_knnn_instance.cpp index 0798f7a9b60f..e083e27460fe 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_knnn_instance = - device_contraction_kn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + bilinear, Bilinear, 2, f16_f16_f16_f16_knnn, + F16, F16, F32, F16, F16_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_mknn_instance.cpp index 7da837148240..8986de8f8254 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_mknn_instance = - device_contraction_mk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + bilinear, Bilinear, 2, f16_f16_f16_f16_mknn, + F16, F16, F32, F16, F16_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_mnnn_instance.cpp index 49267e086705..7a80a9e6f053 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_mnnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_f16_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + bilinear, Bilinear, 2, f16_f16_f16_f16_mnnn, + F16, F16, F32, F16, F16_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_kknn_instance.cpp index 008d5720af5b..ddb619c3f8fe 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_kknn_instance = - device_contraction_kk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + bilinear, Bilinear, 2, f32_f32_f32_f32_compute_bf16_kknn, + F32, F32, F32, F32, F32_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_knnn_instance.cpp index 9b927385ef89..e2abf1c0571e 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_knnn_instance = - device_contraction_kn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + bilinear, Bilinear, 2, f32_f32_f32_f32_compute_bf16_knnn, + F32, F32, F32, F32, F32_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mknn_instance.cpp index a398194f64b1..bc1965c9004a 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mknn_instance = - device_contraction_mk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + bilinear, Bilinear, 2, f32_f32_f32_f32_compute_bf16_mknn, + F32, F32, F32, F32, F32_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mnnn_instance.cpp index 3726f97709fe..4390179324de 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mnnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + bilinear, Bilinear, 2, f32_f32_f32_f32_compute_bf16_mnnn, + F32, F32, F32, F32, F32_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_kknn_instance.cpp index 41fa523b5f74..eae059b621bc 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_kknn_instance = - device_contraction_kk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + bilinear, Bilinear, 2, f32_f32_f32_f32_compute_f16_kknn, + F32, F32, F32, F32, F32_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_knnn_instance.cpp index 898c5a79cc18..b3a72e5f9977 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_knnn_instance = - device_contraction_kn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + bilinear, Bilinear, 2, f32_f32_f32_f32_compute_f16_knnn, + F32, F32, F32, F32, F32_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mknn_instance.cpp index 64db3364a3b9..627489886dd5 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mknn_instance = - device_contraction_mk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + bilinear, Bilinear, 2, f32_f32_f32_f32_compute_f16_mknn, + F32, F32, F32, F32, F32_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mnnn_instance.cpp index ad548f38e73d..8442ea8faee6 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mnnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + bilinear, Bilinear, 2, f32_f32_f32_f32_compute_f16_mnnn, + F32, F32, F32, F32, F32_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance.cpp index 3e36bfd30b43..9344bb06decf 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance = - device_contraction_kk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + bilinear, Bilinear, 2, f32_f32_f32_f32_kknn, + F32, F32, F32, F32, F32_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_knnn_instance.cpp index b67121316b11..72bec728d911 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_knnn_instance = - device_contraction_kn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + bilinear, Bilinear, 2, f32_f32_f32_f32_knnn, + F32, F32, F32, F32, F32_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_mknn_instance.cpp index 94228aa307a8..7e4a69f634ed 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_mknn_instance = - device_contraction_mk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + bilinear, Bilinear, 2, f32_f32_f32_f32_mknn, + F32, F32, F32, F32, F32_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_mnnn_instance.cpp index 28184344c3ea..9516290b2344 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_mnnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_f32_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + bilinear, Bilinear, 2, f32_f32_f32_f32_mnnn, + F32, F32, F32, F32, F32_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_kknn_instance.cpp index f2d107c37d17..2f7ddf0a38c2 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_kknn_instance = - device_contraction_f64_kk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kk_instance, + bilinear, Bilinear, 2, f64_f64_f64_f64_compute_f32_kknn, + F64, F64, F32, F64, F64_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_knnn_instance.cpp index dcf8c05eda1f..074035870f7f 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_knnn_instance = - device_contraction_f64_kn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kn_instance, + bilinear, Bilinear, 2, f64_f64_f64_f64_compute_f32_knnn, + F64, F64, F32, F64, F64_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mknn_instance.cpp index fe2e1108e930..70e4a0ca807b 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mknn_instance = - device_contraction_f64_mk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mk_instance, + bilinear, Bilinear, 2, f64_f64_f64_f64_compute_f32_mknn, + F64, F64, F32, F64, F64_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mnnn_instance.cpp index 420a1f07ebc9..03d36ce10c7a 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mnnn_instance = - device_contraction_f64_mn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mn_instance, + bilinear, Bilinear, 2, f64_f64_f64_f64_compute_f32_mnnn, + F64, F64, F32, F64, F64_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_kknn_instance.cpp index 1c5917cbc6f5..a3e48e8fe0e8 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_kknn_instance = - device_contraction_f64_kk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kk_instance, + bilinear, Bilinear, 2, f64_f64_f64_f64_kknn, + F64, F64, F64, F64, F64_Tuple, F64, F64) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_knnn_instance.cpp index 6b87fcf1d8f0..b6391d36ed2e 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_knnn_instance = - device_contraction_f64_kn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kn_instance, + bilinear, Bilinear, 2, f64_f64_f64_f64_knnn, + F64, F64, F64, F64, F64_Tuple, F64, F64) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_mknn_instance.cpp index 03469cd96be0..3a96d9c8a4ba 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_mknn_instance = - device_contraction_f64_mk_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mk_instance, + bilinear, Bilinear, 2, f64_f64_f64_f64_mknn, + F64, F64, F64, F64, F64_Tuple, F64, F64) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_mnnn_instance.cpp index 5171a38dece6..fc4f651f7535 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/2D/device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_mnnn_instance = - device_contraction_f64_mn_instance; - -void add_device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_f64_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mn_instance, + bilinear, Bilinear, 2, f64_f64_f64_f64_mnnn, + F64, F64, F64, F64, F64_Tuple, F64, F64) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_kknn_instance.cpp index 961b78427f45..26e9a1801bc7 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_kknn_instance = - device_contraction_kk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + bilinear, Bilinear, 6, bf16_bf16_bf16_bf16_compute_f32_kknn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_knnn_instance.cpp index 5cd869249ded..419b1ce339f9 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_knnn_instance = - device_contraction_kn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + bilinear, Bilinear, 6, bf16_bf16_bf16_bf16_compute_f32_knnn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mknn_instance.cpp index aa8ad904a5a2..9b6490cfda1b 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mknn_instance = - device_contraction_mk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + bilinear, Bilinear, 6, bf16_bf16_bf16_bf16_compute_f32_mknn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mnnn_instance.cpp index 80b4de606073..931820ecb8c9 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mnnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_compute_f32_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + bilinear, Bilinear, 6, bf16_bf16_bf16_bf16_compute_f32_mnnn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_kknn_instance.cpp index 77fae91ffeee..35b76bb568c7 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_kknn_instance = - device_contraction_kk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + bilinear, Bilinear, 6, bf16_bf16_bf16_bf16_kknn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_knnn_instance.cpp index 9b8cacc5e150..7a558ca4a896 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_knnn_instance = - device_contraction_kn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + bilinear, Bilinear, 6, bf16_bf16_bf16_bf16_knnn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_mknn_instance.cpp index 50a76452564a..020ac2ca394d 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_mknn_instance = - device_contraction_mk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + bilinear, Bilinear, 6, bf16_bf16_bf16_bf16_mknn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_mnnn_instance.cpp index 78aa99fa6e5c..c21320392785 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_mnnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_bf16_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + bilinear, Bilinear, 6, bf16_bf16_bf16_bf16_mnnn, + BF16, BF16, F32, BF16, BF16_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_kknn_instance.cpp index 2342b0db676b..0896074b15d0 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_kknn_instance = - device_contraction_kk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + bilinear, Bilinear, 6, f16_f16_f16_f16_compute_f32_kknn, + F16, F16, F32, F16, F16_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_knnn_instance.cpp index 130d56c5ca26..b9b7e22544e7 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_knnn_instance = - device_contraction_kn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + bilinear, Bilinear, 6, f16_f16_f16_f16_compute_f32_knnn, + F16, F16, F32, F16, F16_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mknn_instance.cpp index 90222accc158..86affeec0022 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mknn_instance = - device_contraction_mk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + bilinear, Bilinear, 6, f16_f16_f16_f16_compute_f32_mknn, + F16, F16, F32, F16, F16_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mnnn_instance.cpp index 9b731a95cf43..2315f6116820 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mnnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_compute_f32_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + bilinear, Bilinear, 6, f16_f16_f16_f16_compute_f32_mnnn, + F16, F16, F32, F16, F16_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_kknn_instance.cpp index e738e54f069b..dae7e5780a1d 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_kknn_instance.cpp @@ -1,60 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_kknn_instance = - device_contraction_kk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_kknn_instance( - std::vector>>& instances) -{ - printf("[CK_DEBUG] f16+f16+f16+f16_kknn_instance: before add, size=%zu\n", instances.size()); - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_kknn_instance{}); - printf("[CK_DEBUG] f16+f16+f16+f16_kknn_instance: after add, size=%zu\n", instances.size()); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + bilinear, Bilinear, 6, f16_f16_f16_f16_kknn, + F16, F16, F32, F16, F16_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_knnn_instance.cpp index 4bc5b1684a46..319f5a87de4a 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_knnn_instance = - device_contraction_kn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + bilinear, Bilinear, 6, f16_f16_f16_f16_knnn, + F16, F16, F32, F16, F16_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_mknn_instance.cpp index e320fbe11a9e..03739391cdfc 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_mknn_instance = - device_contraction_mk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + bilinear, Bilinear, 6, f16_f16_f16_f16_mknn, + F16, F16, F32, F16, F16_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_mnnn_instance.cpp index bbb90a6af42c..d40fcae6fff2 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_mnnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_f16_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + bilinear, Bilinear, 6, f16_f16_f16_f16_mnnn, + F16, F16, F32, F16, F16_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_kknn_instance.cpp index b95aa0d5ba28..36e8a19263fb 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_kknn_instance = - device_contraction_kk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + bilinear, Bilinear, 6, f32_f32_f32_f32_compute_bf16_kknn, + F32, F32, F32, F32, F32_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_knnn_instance.cpp index e2f62c234251..8b3d2c6420b6 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_knnn_instance = - device_contraction_kn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + bilinear, Bilinear, 6, f32_f32_f32_f32_compute_bf16_knnn, + F32, F32, F32, F32, F32_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mknn_instance.cpp index 80b6b6ecf87c..7c6a8b8d83b7 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mknn_instance = - device_contraction_mk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + bilinear, Bilinear, 6, f32_f32_f32_f32_compute_bf16_mknn, + F32, F32, F32, F32, F32_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mnnn_instance.cpp index 181ad86e1bbb..8b08570f6c1f 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mnnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_bf16_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + bilinear, Bilinear, 6, f32_f32_f32_f32_compute_bf16_mnnn, + F32, F32, F32, F32, F32_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_kknn_instance.cpp index 514da56a0f91..881436f505be 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_kknn_instance = - device_contraction_kk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + bilinear, Bilinear, 6, f32_f32_f32_f32_compute_f16_kknn, + F32, F32, F32, F32, F32_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_knnn_instance.cpp index 61dda90cbca3..6b2d7b14c50b 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_knnn_instance = - device_contraction_kn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + bilinear, Bilinear, 6, f32_f32_f32_f32_compute_f16_knnn, + F32, F32, F32, F32, F32_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mknn_instance.cpp index 301bde04b812..bb91b6879be0 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mknn_instance = - device_contraction_mk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + bilinear, Bilinear, 6, f32_f32_f32_f32_compute_f16_mknn, + F32, F32, F32, F32, F32_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mnnn_instance.cpp index 09dbdff021ef..d35107af675d 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mnnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_compute_f16_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + bilinear, Bilinear, 6, f32_f32_f32_f32_compute_f16_mnnn, + F32, F32, F32, F32, F32_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance.cpp index fe7b520219ff..f56045888af9 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance = - device_contraction_kk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + bilinear, Bilinear, 6, f32_f32_f32_f32_kknn, + F32, F32, F32, F32, F32_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_knnn_instance.cpp index c99a1439e188..5a591fb479bf 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_knnn_instance = - device_contraction_kn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + bilinear, Bilinear, 6, f32_f32_f32_f32_knnn, + F32, F32, F32, F32, F32_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_mknn_instance.cpp index 7ae0833b19a6..42010cb957b7 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_mknn_instance = - device_contraction_mk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + bilinear, Bilinear, 6, f32_f32_f32_f32_mknn, + F32, F32, F32, F32, F32_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_mnnn_instance.cpp index f0cd251985d9..ca015c306dcc 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_mnnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_f32_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + bilinear, Bilinear, 6, f32_f32_f32_f32_mnnn, + F32, F32, F32, F32, F32_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_kknn_instance.cpp index a14b00a7f203..3254d2a5f1df 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_kknn_instance = - device_contraction_f64_kk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kk_instance, + bilinear, Bilinear, 6, f64_f64_f64_f64_compute_f32_kknn, + F64, F64, F32, F64, F64_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_knnn_instance.cpp index e7194022514a..a2831f076084 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_knnn_instance = - device_contraction_f64_kn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kn_instance, + bilinear, Bilinear, 6, f64_f64_f64_f64_compute_f32_knnn, + F64, F64, F32, F64, F64_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mknn_instance.cpp index d093671e25e5..cede3aa1a4ca 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mknn_instance = - device_contraction_f64_mk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mk_instance, + bilinear, Bilinear, 6, f64_f64_f64_f64_compute_f32_mknn, + F64, F64, F32, F64, F64_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mnnn_instance.cpp index 3e0ac565e268..bbee01fa581b 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mnnn_instance = - device_contraction_f64_mn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_compute_f32_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mn_instance, + bilinear, Bilinear, 6, f64_f64_f64_f64_compute_f32_mnnn, + F64, F64, F32, F64, F64_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_kknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_kknn_instance.cpp index c4c8cd13d57f..c6fc9eecf3a8 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_kknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_kknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_kknn_instance = - device_contraction_f64_kk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_kknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_kknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kk_instance, + bilinear, Bilinear, 6, f64_f64_f64_f64_kknn, + F64, F64, F64, F64, F64_Tuple, F64, F64) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_knnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_knnn_instance.cpp index 7e056c48243e..4c0dabed1a62 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_knnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_knnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_knnn_instance = - device_contraction_f64_kn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_knnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_knnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kn_instance, + bilinear, Bilinear, 6, f64_f64_f64_f64_knnn, + F64, F64, F64, F64, F64_Tuple, F64, F64) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_mknn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_mknn_instance.cpp index dd11af63b46d..7154fa880160 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_mknn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_mknn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_mknn_instance = - device_contraction_f64_mk_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_mknn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_mknn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mk_instance, + bilinear, Bilinear, 6, f64_f64_f64_f64_mknn, + F64, F64, F64, F64, F64_Tuple, F64, F64) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_mnnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_mnnn_instance.cpp index 990e862e77ca..bd24c620e30a 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_mnnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_bilinear/6D/device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_mnnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_mnnn_instance = - device_contraction_f64_mn_instance; - -void add_device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_mnnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_bilinear_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_f64_mnnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mn_instance, + bilinear, Bilinear, 6, f64_f64_f64_f64_mnnn, + F64, F64, F64, F64, F64_Tuple, F64, F64) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_kkn_instance.cpp index a3acedbcc405..a0ff8391d214 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_kkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_kkn_instance = - device_contraction_kk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + scale, Scale, 2, bf16_bf16_bf16_compute_f32_kkn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_knn_instance.cpp index c5c365ec26ca..bf5a255afdea 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_knn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_knn_instance = - device_contraction_kn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + scale, Scale, 2, bf16_bf16_bf16_compute_f32_knn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mkn_instance.cpp index 58ab346942ec..8c26b797a764 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mkn_instance = - device_contraction_mk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + scale, Scale, 2, bf16_bf16_bf16_compute_f32_mkn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mnn_instance.cpp index 8c9f6fc57b90..c93b43da7bf5 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + scale, Scale, 2, bf16_bf16_bf16_compute_f32_mnn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_kkn_instance.cpp index c85f8cc998f3..9d32d0eb4597 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_kkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_kkn_instance = - device_contraction_kk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + scale, Scale, 2, bf16_bf16_bf16_kkn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_knn_instance.cpp index d4a25d40cbb8..8474e996c2e8 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_knn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_knn_instance = - device_contraction_kn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + scale, Scale, 2, bf16_bf16_bf16_knn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_mkn_instance.cpp index 7be8a0a694e5..6c8c7ac837ea 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_mkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_mkn_instance = - device_contraction_mk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + scale, Scale, 2, bf16_bf16_bf16_mkn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_mnn_instance.cpp index b2a4c020e67b..e971273a2f72 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_mnn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_mnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_bf16_bf16_bf16_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + scale, Scale, 2, bf16_bf16_bf16_mnn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_kkn_instance.cpp index 9a9d3e16fb66..8026a5f3b95c 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_kkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_kkn_instance = - device_contraction_kk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + scale, Scale, 2, f16_f16_f16_compute_f32_kkn, + F16, F16, F32, F16, Empty_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_knn_instance.cpp index d158d5eb99da..697474954678 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_knn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_knn_instance = - device_contraction_kn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + scale, Scale, 2, f16_f16_f16_compute_f32_knn, + F16, F16, F32, F16, Empty_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_mkn_instance.cpp index a263d0b8ca4a..fb80ab9df1a9 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_mkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_mkn_instance = - device_contraction_mk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + scale, Scale, 2, f16_f16_f16_compute_f32_mkn, + F16, F16, F32, F16, Empty_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_mnn_instance.cpp index eb9fa3714e09..87f337c67f68 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_mnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_mnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_compute_f32_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + scale, Scale, 2, f16_f16_f16_compute_f32_mnn, + F16, F16, F32, F16, Empty_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_kkn_instance.cpp index 52042dd0451a..e8de33728bc9 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_kkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_kkn_instance = - device_contraction_kk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + scale, Scale, 2, f16_f16_f16_kkn, + F16, F16, F32, F16, Empty_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_knn_instance.cpp index 2b6aed8ed4f4..e87816b00fce 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_knn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_knn_instance = - device_contraction_kn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + scale, Scale, 2, f16_f16_f16_knn, + F16, F16, F32, F16, Empty_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_mkn_instance.cpp index 07cbbf87c68e..2e13b536f2a2 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_mkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_mkn_instance = - device_contraction_mk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + scale, Scale, 2, f16_f16_f16_mkn, + F16, F16, F32, F16, Empty_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_mnn_instance.cpp index 2cc4bfb718dd..eccce81df91a 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_mnn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_mnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f16_f16_f16_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + scale, Scale, 2, f16_f16_f16_mnn, + F16, F16, F32, F16, Empty_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_kkn_instance.cpp index 50fe1a696f12..6464ffeddc49 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_kkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_kkn_instance = - device_contraction_kk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + scale, Scale, 2, f32_f32_f32_compute_bf16_kkn, + F32, F32, F32, F32, Empty_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_knn_instance.cpp index 6aab79f31268..26bf60755902 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_knn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_knn_instance = - device_contraction_kn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + scale, Scale, 2, f32_f32_f32_compute_bf16_knn, + F32, F32, F32, F32, Empty_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_mkn_instance.cpp index e6f24424abdb..e236ad71f4f9 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_mkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_mkn_instance = - device_contraction_mk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + scale, Scale, 2, f32_f32_f32_compute_bf16_mkn, + F32, F32, F32, F32, Empty_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_mnn_instance.cpp index 60b760bfce10..3ccd1820e009 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_mnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_mnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_bf16_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + scale, Scale, 2, f32_f32_f32_compute_bf16_mnn, + F32, F32, F32, F32, Empty_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_kkn_instance.cpp index 19992c96fd04..f60ef81681c8 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_kkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_kkn_instance = - device_contraction_kk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + scale, Scale, 2, f32_f32_f32_compute_f16_kkn, + F32, F32, F32, F32, Empty_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_knn_instance.cpp index a13e315e381a..da0ffaf8f0bf 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_knn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_knn_instance = - device_contraction_kn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + scale, Scale, 2, f32_f32_f32_compute_f16_knn, + F32, F32, F32, F32, Empty_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_mkn_instance.cpp index 3b4aaa7a5b83..a1567d9c827e 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_mkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_mkn_instance = - device_contraction_mk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + scale, Scale, 2, f32_f32_f32_compute_f16_mkn, + F32, F32, F32, F32, Empty_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_mnn_instance.cpp index 48e190574faf..098602f203ab 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_mnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_mnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_compute_f16_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + scale, Scale, 2, f32_f32_f32_compute_f16_mnn, + F32, F32, F32, F32, Empty_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_kkn_instance.cpp index 1b8bceb65de6..483b4eb86904 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_kkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_kkn_instance = - device_contraction_kk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + scale, Scale, 2, f32_f32_f32_kkn, + F32, F32, F32, F32, Empty_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_knn_instance.cpp index a09ebae1dd9a..71b17712b30b 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_knn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_knn_instance = - device_contraction_kn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + scale, Scale, 2, f32_f32_f32_knn, + F32, F32, F32, F32, Empty_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_mkn_instance.cpp index 4172958f2aff..91b6b1d92742 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_mkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_mkn_instance = - device_contraction_mk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + scale, Scale, 2, f32_f32_f32_mkn, + F32, F32, F32, F32, Empty_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_mnn_instance.cpp index c8c9ce43480e..cbba0786e297 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_mnn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_mnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f32_f32_f32_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + scale, Scale, 2, f32_f32_f32_mnn, + F32, F32, F32, F32, Empty_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_kkn_instance.cpp index bb44557ba800..dcd7cf50c4f3 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_kkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_kkn_instance = - device_contraction_f64_kk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kk_instance, + scale, Scale, 2, f64_f64_f64_compute_f32_kkn, + F64, F64, F32, F64, Empty_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_knn_instance.cpp index 91c96bd6798c..13ac1b4cbb73 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_knn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_knn_instance = - device_contraction_f64_kn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kn_instance, + scale, Scale, 2, f64_f64_f64_compute_f32_knn, + F64, F64, F32, F64, Empty_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_mkn_instance.cpp index 0fe142fc59a8..e012e157a7fb 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_mkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_mkn_instance = - device_contraction_f64_mk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mk_instance, + scale, Scale, 2, f64_f64_f64_compute_f32_mkn, + F64, F64, F32, F64, Empty_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_mnn_instance.cpp index 28d337d24639..5bda23685640 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_mnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_mnn_instance = - device_contraction_f64_mn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_compute_f32_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mn_instance, + scale, Scale, 2, f64_f64_f64_compute_f32_mnn, + F64, F64, F32, F64, Empty_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_kkn_instance.cpp index 39e29cd3e8ef..8ab00c937c16 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_kkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_kkn_instance = - device_contraction_f64_kk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kk_instance, + scale, Scale, 2, f64_f64_f64_kkn, + F64, F64, F64, F64, Empty_Tuple, F64, F64) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_knn_instance.cpp index ef4dd284e526..fb33d7d76195 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_knn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_knn_instance = - device_contraction_f64_kn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kn_instance, + scale, Scale, 2, f64_f64_f64_knn, + F64, F64, F64, F64, Empty_Tuple, F64, F64) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_mkn_instance.cpp index 78effae8e273..571cea261e99 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_mkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_mkn_instance = - device_contraction_f64_mk_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mk_instance, + scale, Scale, 2, f64_f64_f64_mkn, + F64, F64, F64, F64, Empty_Tuple, F64, F64) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_mnn_instance.cpp index 465a80b1b0ad..9847c021d584 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/2D/device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_mnn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_mnn_instance = - device_contraction_f64_mn_instance; - -void add_device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m2_n2_k2_xdl_c_shuffle_f64_f64_f64_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mn_instance, + scale, Scale, 2, f64_f64_f64_mnn, + F64, F64, F64, F64, Empty_Tuple, F64, F64) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_kkn_instance.cpp index a472f793e444..134fca49365b 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_kkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_kkn_instance = - device_contraction_kk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + scale, Scale, 6, bf16_bf16_bf16_compute_f32_kkn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_knn_instance.cpp index c4bddd6c6ec2..062f8468f702 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_knn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_knn_instance = - device_contraction_kn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + scale, Scale, 6, bf16_bf16_bf16_compute_f32_knn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mkn_instance.cpp index 3a1c9c3fb91b..c6b7784f2792 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mkn_instance = - device_contraction_mk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + scale, Scale, 6, bf16_bf16_bf16_compute_f32_mkn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mnn_instance.cpp index d23c00519101..30f483036a70 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_compute_f32_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + scale, Scale, 6, bf16_bf16_bf16_compute_f32_mnn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_kkn_instance.cpp index 9244f6a132c1..9118dba4f17c 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_kkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_kkn_instance = - device_contraction_kk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + scale, Scale, 6, bf16_bf16_bf16_kkn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_knn_instance.cpp index 99e80e0e28d6..713eff33cbfe 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_knn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_knn_instance = - device_contraction_kn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + scale, Scale, 6, bf16_bf16_bf16_knn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_mkn_instance.cpp index 77ca8c0d16b3..1b78e11f70d0 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_mkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_mkn_instance = - device_contraction_mk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + scale, Scale, 6, bf16_bf16_bf16_mkn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_mnn_instance.cpp index 564fe537bb41..2a70c27f207a 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_mnn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_mnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_bf16_bf16_bf16_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + scale, Scale, 6, bf16_bf16_bf16_mnn, + BF16, BF16, F32, BF16, Empty_Tuple, BF16, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_kkn_instance.cpp index 69f074caf01c..80bc1cbe7242 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_kkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_kkn_instance = - device_contraction_kk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + scale, Scale, 6, f16_f16_f16_compute_f32_kkn, + F16, F16, F32, F16, Empty_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_knn_instance.cpp index dbad11727c60..5564fcb64ffb 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_knn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_knn_instance = - device_contraction_kn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + scale, Scale, 6, f16_f16_f16_compute_f32_knn, + F16, F16, F32, F16, Empty_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_mkn_instance.cpp index a53e7801ea09..19c73e48b854 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_mkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_mkn_instance = - device_contraction_mk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + scale, Scale, 6, f16_f16_f16_compute_f32_mkn, + F16, F16, F32, F16, Empty_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_mnn_instance.cpp index 977497d387a4..1acb62c960dc 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_mnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_mnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_compute_f32_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + scale, Scale, 6, f16_f16_f16_compute_f32_mnn, + F16, F16, F32, F16, Empty_Tuple, F16, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_kkn_instance.cpp index dfc187562a37..28d2d84510ff 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_kkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_kkn_instance = - device_contraction_kk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + scale, Scale, 6, f16_f16_f16_kkn, + F16, F16, F32, F16, Empty_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_knn_instance.cpp index 50d951a99c66..ba247621ff80 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_knn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_knn_instance = - device_contraction_kn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + scale, Scale, 6, f16_f16_f16_knn, + F16, F16, F32, F16, Empty_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_mkn_instance.cpp index 460c5c4b4923..32d601c9b7f0 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_mkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_mkn_instance = - device_contraction_mk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + scale, Scale, 6, f16_f16_f16_mkn, + F16, F16, F32, F16, Empty_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_mnn_instance.cpp index bee17f3386bc..fb66208b9360 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_mnn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_mnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f16_f16_f16_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + scale, Scale, 6, f16_f16_f16_mnn, + F16, F16, F32, F16, Empty_Tuple, F16, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_kkn_instance.cpp index 5f737132afc0..c78f64bfcaed 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_kkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_kkn_instance = - device_contraction_kk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + scale, Scale, 6, f32_f32_f32_compute_bf16_kkn, + F32, F32, F32, F32, Empty_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_knn_instance.cpp index 1dbebe89f798..fde6062baae7 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_knn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_knn_instance = - device_contraction_kn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + scale, Scale, 6, f32_f32_f32_compute_bf16_knn, + F32, F32, F32, F32, Empty_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_mkn_instance.cpp index 4c609db46aab..7d3ae3348ed9 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_mkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_mkn_instance = - device_contraction_mk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + scale, Scale, 6, f32_f32_f32_compute_bf16_mkn, + F32, F32, F32, F32, Empty_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_mnn_instance.cpp index 9005335eafdc..899ba7aac5a4 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_mnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_mnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_bf16_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + scale, Scale, 6, f32_f32_f32_compute_bf16_mnn, + F32, F32, F32, F32, Empty_Tuple, F32, BF16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_kkn_instance.cpp index 4623b2e5d808..afc0c0a588bf 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_kkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_kkn_instance = - device_contraction_kk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + scale, Scale, 6, f32_f32_f32_compute_f16_kkn, + F32, F32, F32, F32, Empty_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_knn_instance.cpp index 952ad237a89e..7d084a8b45eb 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_knn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_knn_instance = - device_contraction_kn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + scale, Scale, 6, f32_f32_f32_compute_f16_knn, + F32, F32, F32, F32, Empty_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_mkn_instance.cpp index 8273c319b820..821bc2798fa4 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_mkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_mkn_instance = - device_contraction_mk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + scale, Scale, 6, f32_f32_f32_compute_f16_mkn, + F32, F32, F32, F32, Empty_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_mnn_instance.cpp index cf22f7a729c9..3fe62bb11746 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_mnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_mnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_compute_f16_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + scale, Scale, 6, f32_f32_f32_compute_f16_mnn, + F32, F32, F32, F32, Empty_Tuple, F32, F16) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_kkn_instance.cpp index a4659d4d904c..a29453355647 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_kkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_kkn_instance = - device_contraction_kk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kk_instance, + scale, Scale, 6, f32_f32_f32_kkn, + F32, F32, F32, F32, Empty_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_knn_instance.cpp index 72adf0f03db4..fa38bc2ef82c 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_knn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_knn_instance = - device_contraction_kn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_kn_instance, + scale, Scale, 6, f32_f32_f32_knn, + F32, F32, F32, F32, Empty_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_mkn_instance.cpp index d70c2bb4c5f8..5752bc169ab8 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_mkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_mkn_instance = - device_contraction_mk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mk_instance, + scale, Scale, 6, f32_f32_f32_mkn, + F32, F32, F32, F32, Empty_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_mnn_instance.cpp index 7fa3458ab012..1cae73eb8a52 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_mnn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_mnn_instance = - device_contraction_mn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f32_f32_f32_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_mn_instance, + scale, Scale, 6, f32_f32_f32_mnn, + F32, F32, F32, F32, Empty_Tuple, F32, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_kkn_instance.cpp index 877545e3382e..1f171a141352 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_kkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_kkn_instance = - device_contraction_f64_kk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kk_instance, + scale, Scale, 6, f64_f64_f64_compute_f32_kkn, + F64, F64, F32, F64, Empty_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_knn_instance.cpp index df51431b2337..66a8eae42790 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_knn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_knn_instance = - device_contraction_f64_kn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kn_instance, + scale, Scale, 6, f64_f64_f64_compute_f32_knn, + F64, F64, F32, F64, Empty_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_mkn_instance.cpp index 3bbdf8486584..9c5e9fd1bbcc 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_mkn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_mkn_instance = - device_contraction_f64_mk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mk_instance, + scale, Scale, 6, f64_f64_f64_compute_f32_mkn, + F64, F64, F32, F64, Empty_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_mnn_instance.cpp index 127c47c5a379..579e9559739c 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_mnn_instance.cpp @@ -1,58 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_mnn_instance = - device_contraction_f64_mn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, - device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_compute_f32_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mn_instance, + scale, Scale, 6, f64_f64_f64_compute_f32_mnn, + F64, F64, F32, F64, Empty_Tuple, F64, F32) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_kkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_kkn_instance.cpp index f05a685d177f..c3357a6f910f 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_kkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_kkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_kkn_instance = - device_contraction_f64_kk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_kkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_kkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kk_instance, + scale, Scale, 6, f64_f64_f64_kkn, + F64, F64, F64, F64, Empty_Tuple, F64, F64) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_knn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_knn_instance.cpp index 34bc800fcf94..447db7fab4df 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_knn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_knn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// k/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_knn_instance = - device_contraction_f64_kn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_knn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_knn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_kn_instance, + scale, Scale, 6, f64_f64_f64_knn, + F64, F64, F64, F64, Empty_Tuple, F64, F64) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_mkn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_mkn_instance.cpp index 180d1b527373..059689ff5e60 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_mkn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_mkn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/k/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_mkn_instance = - device_contraction_f64_mk_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_mkn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_mkn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mk_instance, + scale, Scale, 6, f64_f64_f64_mkn, + F64, F64, F64, F64, Empty_Tuple, F64, F64) +// clang-format on diff --git a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_mnn_instance.cpp b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_mnn_instance.cpp index bb6f5c668562..393b7ac6f34e 100644 --- a/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_mnn_instance.cpp +++ b/projects/composablekernel/library/src/tensor_operation_instance/gpu/contraction_scale/6D/device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_mnn_instance.cpp @@ -1,57 +1,12 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -// This (ifndef) is a hack to use customized behavior for buffer load rather than using default -// setting Don't use this hack unless absolutely necessary! -// FIXME: make the behavior of buffer load a configurable (template) parameter of each device op -#define CK_EXPERIMENTAL_USE_BUFFER_LOAD_OOB_CHECK_OFFSET_TRICK 1 - -#include - -#include "ck/ck.hpp" -#include "ck/tensor_operation/gpu/device/device_contraction_multiple_d.hpp" -#include "ck/tensor_operation/gpu/element/element_wise_operation.hpp" -#include "ck/library/tensor_operation_instance/gpu/contraction/device_contraction_instance.hpp" -#include "ck/library/tensor_operation_instance/add_device_operation_instance.hpp" - -namespace ck { -namespace tensor_operation { -namespace device { -namespace instance { - -// A[m0, m1, k0, k1] * B[n0, n1, k0, k1] + D[m0, m1, n0, n1] = E[m0, m1, n0, n1] -// m/n/n/n are the fast changing dimension for A/B/D/E -using device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_mnn_instance = - device_contraction_f64_mn_instance; - -void add_device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_mnn_instance( - std::vector>>& instances) -{ - add_device_operation_instances( - instances, device_contraction_scale_m6_n6_k6_xdl_c_shuffle_f64_f64_f64_mnn_instance{}); -} - -} // namespace instance -} // namespace device -} // namespace tensor_operation -} // namespace ck +#include "../../contraction/contraction_instance_common.hpp" + +// Instantiate contraction device operation and register via add_device_* function. +// See contraction_instance_common.hpp for macro definition and parameter documentation. +// clang-format off +CK_CONTRACTION_INSTANCE(device_contraction_f64_mn_instance, + scale, Scale, 6, f64_f64_f64_mnn, + F64, F64, F64, F64, Empty_Tuple, F64, F64) +// clang-format on