Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ template <typename PrecType_, typename WorkspaceType_>
struct GemmConfigTwoStage_Wmma : public GemmConfigComputeV3_WMMA<PrecType_>
{
using WorkspaceType = ck_tile::remove_cvref_t<WorkspaceType_>;

static constexpr bool EnableKPadFallback = false; // TODO invoker not supported yet
};

struct SplitKTwoStageInvoker
Expand Down Expand Up @@ -79,12 +81,19 @@ struct SplitKTwoStageInvoker
ck_tile::element_wise::PassThrough,
ck_tile::element_wise::PassThrough,
AComputeDataType,
BComputeDataType>;
BComputeDataType,
GemmConfig::FixedVectorSize,
GemmConfig::VectorSizeA,
GemmConfig::VectorSizeB>;
Comment thread
chris-tsiaousis-hpc marked this conversation as resolved.
Resolved
using WorkspaceType = ck_tile::remove_cvref_t<typename GemmConfig::WorkspaceType>;

using GemmPipeline = typename PipelineTypeTraits<
GemmConfig::Pipeline>::template GemmPipeline<UniversalGemmProblem>;

// The first kernel writes an intermediate WorkspaceType buffer rather than the final C
// tensor. A second elementwise kernel converts that buffer to CDataType and writes C.
// Because VectorSizeC controls stores to C, it is not passed to
// this intermediate-output epilogue, which selects its own store width.
using GemmEpilogue = ck_tile::CShuffleEpilogue<
ck_tile::CShuffleEpilogueProblem<ADataType,
BDataType,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,14 @@
#include <string>
#include <variant>

// Max. vectorized global memory access in bytes.
static constexpr ck_tile::index_t kMaxVectorBytes = 16;

// Number of elements of type T in 1 max-width access.
template <typename T>
static constexpr ck_tile::index_t kMaxVectorElems =
static_cast<ck_tile::index_t>(kMaxVectorBytes / sizeof(T));

struct GemmConfigBase
{
static constexpr bool kPadM = false;
Expand Down Expand Up @@ -41,6 +49,36 @@ struct GemmConfigBase
static constexpr ck_tile::DataCachePrefetchKind DataCachePrefetchB =
ck_tile::DataCachePrefetchKind::None;
static constexpr bool Async = false;

static constexpr bool FixedVectorSize = false;
// If FixedVectorSize==true: use these vector sizes for A/B loads and C store
static constexpr ck_tile::index_t VectorSizeA = 1;
static constexpr ck_tile::index_t VectorSizeB = 1;
static constexpr ck_tile::index_t VectorSizeC = 1;

// Enable for gfx1250 RCR
Comment thread
chris-tsiaousis-hpc marked this conversation as resolved.
Resolved
static constexpr bool EnableKPadFallback = false;
};

// A,B vector sizes must divide K_Warp_Tile.
template <typename GemmConfig,
ck_tile::index_t VectorSizeA_,
ck_tile::index_t VectorSizeB_,
ck_tile::index_t VectorSizeC_>
struct GemmConfigFixedVectorSize : public GemmConfig
{
// Split-K partitions are aligned to K_Warp_Tile: preserve split-K remainder
static_assert(GemmConfig::K_Warp_Tile % VectorSizeA_ == 0 &&
GemmConfig::K_Warp_Tile % VectorSizeB_ == 0,
"A/B vector width must divide K_Warp_Tile");

// Enable K padding
static constexpr bool kPadK = true;

static constexpr bool FixedVectorSize = true;
static constexpr ck_tile::index_t VectorSizeA = VectorSizeA_;
static constexpr ck_tile::index_t VectorSizeB = VectorSizeB_;
static constexpr ck_tile::index_t VectorSizeC = VectorSizeC_;
};

template <typename PrecType>
Expand Down Expand Up @@ -193,13 +231,18 @@ struct GemmConfigComputeV3_WMMA : public GemmConfigBase
static constexpr ck_tile::GemmPipeline Pipeline = ck_tile::GemmPipeline::COMPUTE_V3;

static constexpr int kBlockPerCu = 2;

static constexpr bool EnableKPadFallback = true;
Comment thread
chris-tsiaousis-hpc marked this conversation as resolved.
Resolved
};

template <typename PrecType>
struct GemmConfigComputeV3_WMMA_ClusterLaunch : public GemmConfigComputeV3_WMMA<PrecType>
{
static constexpr ck_tile::index_t kClusterSizeM = 2;
static constexpr ck_tile::index_t kClusterSizeN = 2;

// KPad fallback not validated on the cluster-launch path yet, disable for now.
static constexpr bool EnableKPadFallback = false;
};

template <typename PrecType>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
#include "ck_tile/host/tensor_shuffle_utils.hpp"
#include "ck_tile/ops/common/utils.hpp"

#include <type_traits>

template <typename Layout>
static constexpr inline auto is_row_major(Layout layout_)
{
Expand Down Expand Up @@ -347,31 +349,64 @@ int run_gemm_example_with_layouts(ck_tile::ArgParser& arg_parser,
c_m_n_dev_buf.SetZero();
c_m_n_dev_result.SetZero();

float ave_time = invoke_gemm<GemmConfig,
Invoker,
ADataTypeBuf,
BDataTypeBuf,
ck_tile::tuple<>,
AccDataType,
CDataType,
ALayout,
BLayout,
ck_tile::tuple<>,
CLayout>(a_m_k_dev_buf,
b_k_n_dev_buf,
c_m_n_dev_buf,
M,
N,
K,
stride_A,
stride_B,
stride_C,
kbatch,
n_warmup,
n_repeat,
persistent,
flush_cache,
rotating_count);
#ifdef CK_USE_GFX1250
Comment thread
chris-tsiaousis-hpc marked this conversation as resolved.
Resolved
// Enable fallback: gfx1250, RCR, PackedSize=1
constexpr bool k_pad_fallback_eligible =
GemmConfig::EnableKPadFallback && !GemmConfig::FixedVectorSize &&
std::is_same_v<ADataTypeBuf, BDataTypeBuf> &&
std::is_same_v<ALayout, ck_tile::tensor_layout::gemm::RowMajor> &&
std::is_same_v<BLayout, ck_tile::tensor_layout::gemm::ColumnMajor> &&
std::is_same_v<CLayout, ck_tile::tensor_layout::gemm::RowMajor> &&
ck_tile::numeric_traits<ADataTypeBuf>::PackedSize == 1 &&
ck_tile::numeric_traits<BDataTypeBuf>::PackedSize == 1;
#else
constexpr bool k_pad_fallback_eligible = false;
#endif

// Invoke GEMM
auto run_invoke = [&](auto cfg_tag) {
using Cfg = typename decltype(cfg_tag)::type;
return invoke_gemm<Cfg,
Invoker,
ADataTypeBuf,
BDataTypeBuf,
ck_tile::tuple<>,
AccDataType,
CDataType,
ALayout,
BLayout,
ck_tile::tuple<>,
CLayout>(a_m_k_dev_buf,
b_k_n_dev_buf,
c_m_n_dev_buf,
M,
N,
K,
stride_A,
stride_B,
stride_C,
kbatch,
n_warmup,
n_repeat,
persistent,
flush_cache,
rotating_count);
};

float ave_time;
if constexpr(k_pad_fallback_eligible)
{
using KPadConfig = GemmConfigFixedVectorSize<GemmConfig, 1, 1, kMaxVectorElems<CDataType>>;

ave_time = (K % kMaxVectorElems<ADataTypeBuf> != 0)
? run_invoke(std::type_identity<KPadConfig>{}) // unaligned K
: run_invoke(std::type_identity<GemmConfig>{}); // aligned K
}
else
{
// Do not use KPad fallback
ave_time = run_invoke(std::type_identity<GemmConfig>{});
}

c_m_n_dev_buf.FromDevice(c_m_n_dev_result.data());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,10 @@ struct UniversalInvoker
ck_tile::element_wise::PassThrough,
ck_tile::element_wise::PassThrough,
AComputeDataType,
BComputeDataType>;
BComputeDataType,
GemmConfig::FixedVectorSize,
GemmConfig::VectorSizeA,
GemmConfig::VectorSizeB>;

using GemmPipeline = typename PipelineTypeTraits<
GemmConfig::Pipeline>::template GemmPipeline<UniversalGemmProblem>;
Expand All @@ -127,8 +130,8 @@ struct UniversalInvoker
GemmConfig::K_Warp_Tile,
UniversalGemmProblem::TransposeC,
GemmConfig::NumWaveGroups,
false, /*FixedVectorSize_*/
1, /*VectorSizeC_*/
GemmConfig::FixedVectorSize,
GemmConfig::VectorSizeC,
1, /*BlockedXDLN_PerWarp_*/
GemmConfig::DoubleSmemBuffer, /*DoubleSmemBuffer*/
AComputeDataType,
Expand Down
11 changes: 10 additions & 1 deletion projects/composablekernel/test/ck_tile/gemm/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,16 @@ if(GPU_TARGETS MATCHES "gfx94|gfx95|gfx90a|gfx11|gfx12")
test_ck_tile_gemm_pipeline_compv4_wmma
test_ck_tile_gemm_pipeline_persistent_wmma
)


if(SUPPORTED_GPU_TARGETS MATCHES "gfx1250")
Comment thread
chris-tsiaousis-hpc marked this conversation as resolved.
Resolved
# Test GEMM with K not a multiple of the A/B load vector width
add_gtest_executable(test_gemm_unaligned_k test_gemm_unaligned_k.cpp)
target_include_directories(test_gemm_unaligned_k PRIVATE
${CMAKE_SOURCE_DIR}/example/ck_tile/03_gemm)
target_compile_options(test_gemm_unaligned_k PRIVATE ${EXAMPLE_GEMM_COMPILE_OPTIONS})
list(APPEND CK_TILE_GEMM_TEST_TARGETS test_gemm_unaligned_k)
endif()

if(GPU_TARGETS MATCHES "gfx125")
add_gtest_executable(test_ck_tile_gemm_pipeline_tdm_wmma test_gemm_pipeline_comp_tdm_wmma.cpp)
target_compile_options(test_ck_tile_gemm_pipeline_tdm_wmma PRIVATE ${EXAMPLE_GEMM_COMPILE_OPTIONS})
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
// Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
// SPDX-License-Identifier: MIT

#include "gemm_utils.hpp"
#include "run_gemm_example.inc"
#include "universal_gemm_invoker.hpp"

#include "ck_tile/host.hpp"

#include "gtest/gtest.h"

#include <tuple>

#ifdef CK_USE_GFX1250
Comment thread
chris-tsiaousis-hpc marked this conversation as resolved.
Resolved

using Row = ck_tile::tensor_layout::gemm::RowMajor;
using Col = ck_tile::tensor_layout::gemm::ColumnMajor;

template <typename PrecType, typename CDataType>
using KPaddingConfig =
GemmConfigFixedVectorSize<GemmConfigComputeV3_WMMA<PrecType>, 1, 1, kMaxVectorElems<CDataType>>;

template <typename Tuple>
class TestGemmUnalignedK : public ::testing::Test
{
protected:
using ADataType = std::tuple_element_t<0, Tuple>;
using BDataType = std::tuple_element_t<1, Tuple>;
using AccDataType = std::tuple_element_t<2, Tuple>;
using CDataType = std::tuple_element_t<3, Tuple>;

void RunAndVerify(int K, int k_batch = 1)
{
constexpr int M = 128;
constexpr int N = 128;

const ck_tile::index_t stride_A = K;
const ck_tile::index_t stride_B = K;
const ck_tile::index_t stride_C = N;

// Host tensors
ck_tile::HostTensor<ADataType> a_m_k(
ck_tile::host_tensor_descriptor(M, K, stride_A, ck_tile::bool_constant<true>{}));
ck_tile::HostTensor<BDataType> b_k_n(
ck_tile::host_tensor_descriptor(K, N, stride_B, ck_tile::bool_constant<false>{}));
ck_tile::HostTensor<CDataType> gpu_result(
ck_tile::host_tensor_descriptor(M, N, stride_C, ck_tile::bool_constant<true>{}));

ck_tile::FillUniformDistributionIntegerValue<ADataType>{-5, 5, 11939}(a_m_k);
ck_tile::FillUniformDistributionIntegerValue<BDataType>{-5, 5, 11940}(b_k_n);

// Device buffers
ck_tile::DeviceMem a_buf(a_m_k.get_element_space_size_in_bytes());
ck_tile::DeviceMem b_buf(b_k_n.get_element_space_size_in_bytes());
ck_tile::DeviceMem c_buf(gpu_result.get_element_space_size_in_bytes());
a_buf.ToDevice(a_m_k.data());
b_buf.ToDevice(b_k_n.data());
c_buf.SetZero();

ck_tile::GemmHostArgs args = {a_buf.GetDeviceBuffer(),
b_buf.GetDeviceBuffer(),
c_buf.GetDeviceBuffer(),
k_batch,
M,
N,
K,
stride_A,
stride_B,
stride_C};

// Run UniversalInvoker::gemm()
UniversalInvoker::gemm<KPaddingConfig<ADataType, CDataType>,
ADataType,
BDataType,
ck_tile::tuple<>,
AccDataType,
CDataType,
Row,
Col,
ck_tile::tuple<>,
Row,
/*Persistent=*/false,
ck_tile::element_wise::PassThrough>(
args, ck_tile::stream_config{nullptr, false});

c_buf.FromDevice(gpu_result.data());

ck_tile::HostTensor<CDataType> host_reference(
ck_tile::host_tensor_descriptor(M, N, stride_C, ck_tile::bool_constant<true>{}));
host_reference.SetZero();
ck_tile::reference_gemm<ADataType, BDataType, AccDataType, CDataType>(
a_m_k, b_k_n, host_reference);

const float max_accumulated_value =
*std::max_element(host_reference.mData.begin(), host_reference.mData.end());
const auto rtol_atol = calculate_rtol_atol<ADataType, BDataType, AccDataType, CDataType>(
K, k_batch, max_accumulated_value);

// Compare both results
EXPECT_TRUE(do_verify(gpu_result, host_reference, rtol_atol, "GPU"))
<< "K=" << K << ", k_batch=" << k_batch;
}
};

using UnalignedKDataTypes =
::testing::Types<std::tuple<ck_tile::half_t, ck_tile::half_t, float, ck_tile::half_t>,
std::tuple<ck_tile::bf16_t, ck_tile::bf16_t, float, ck_tile::bf16_t>,
std::tuple<ck_tile::fp8_t, ck_tile::fp8_t, float, ck_tile::half_t>,
std::tuple<ck_tile::bf8_t, ck_tile::bf8_t, float, ck_tile::half_t>,
std::tuple<ck_tile::int8_t, ck_tile::int8_t, int32_t, int32_t>>;
TYPED_TEST_SUITE(TestGemmUnalignedK, UnalignedKDataTypes);

TYPED_TEST(TestGemmUnalignedK, SmallKAndTail)
{
// Cover a K smaller than the maximum load width and a full K tile plus a tail.
constexpr int KTile = GemmConfigComputeV3_WMMA<typename TestFixture::ADataType>::K_Tile;
for(int K : {5, KTile + 5})
{
this->RunAndVerify(K);
}
}

TYPED_TEST(TestGemmUnalignedK, SplitK)
{
// The first split is a full warp tile; the second contains the K tail.
constexpr int K = GemmConfigComputeV3_WMMA<typename TestFixture::ADataType>::K_Warp_Tile + 5;
this->RunAndVerify(K, /*k_batch=*/2);
}

#endif // CK_USE_GFX1250
Loading