From d7fb2080895b43829d3880d86978722a8cbbffce Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Tue, 22 Sep 2026 20:34:55 -0400 Subject: [PATCH 01/32] feat(rocke): share logical dtype and packed operand contracts --- .../rocke/platform/cpp/core/dtypes.cpp | 25 ++ .../rocke/platform/cpp/core/ir/core_types.cpp | 32 +++ .../platform/cpp/core/lower_hip/core.cpp | 57 ++++ .../rocke/platform/cpp/core/storage.cpp | 265 ++++++++++++++++++ .../rocke/platform/cpp/helpers/mma_io.cpp | 206 ++++++++++++++ .../rocke/platform/cpp/helpers/quant.cpp | 45 +-- .../rocke/platform/cpp/include/rocke/dtypes.h | 18 ++ .../rocke/helper_rocke.helpers.mma_io.h | 41 +++ .../rocke/helper_rocke.helpers.quant.h | 44 +-- .../rocke/platform/cpp/include/rocke/ir.h | 15 + .../platform/cpp/include/rocke/storage.h | 114 ++++++++ .../cpp/include/rocke/wmma_scale_internal.h | 56 +++- .../architecture/dtype_storage_packing.md | 124 ++++++++ .../platform/python/rocke/core/__init__.py | 19 +- .../python/rocke/core/arch/wmma_scale.py | 68 ++++- .../platform/python/rocke/core/dtypes.py | 67 +++++ .../rocke/platform/python/rocke/core/ir.py | 36 +++ .../platform/python/rocke/core/lower_hip.py | 37 +++ .../platform/python/rocke/core/storage.py | 229 +++++++++++++++ .../platform/python/rocke/helpers/mma_io.py | 162 +++++++++++ .../platform/python/rocke/helpers/quant.py | 37 ++- .../rocke/platform/tests/CMakeLists.txt | 9 + .../rocke/platform/tests/core/test_dtypes.py | 49 ++++ .../rocke/platform/tests/core/test_mma_io.py | 147 ++++++++++ .../platform/tests/core/test_storage.cpp | 186 ++++++++++++ .../rocke/platform/tests/core/test_storage.py | 94 +++++++ 26 files changed, 2093 insertions(+), 89 deletions(-) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.mma_io.h create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/storage.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/dtypes.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/dtypes.cpp index 0f717a76e35f..b5ab7d5507ae 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/dtypes.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/dtypes.cpp @@ -137,3 +137,28 @@ const char* rocke_normalize_dtype(const char* name, char* lowered, size_t cap) } return lowered; } + +static const rocke_dtype_info_t k_dtype_info[] = { + {"fp32", ROCKE_DTYPE_FLOAT, 32}, {"fp16", ROCKE_DTYPE_FLOAT, 16}, + {"bf16", ROCKE_DTYPE_FLOAT, 16}, {"fp8e4m3", ROCKE_DTYPE_FLOAT, 8}, + {"bf8e5m2", ROCKE_DTYPE_FLOAT, 8}, {"fp6e2m3", ROCKE_DTYPE_FLOAT, 6}, + {"fp6e3m2", ROCKE_DTYPE_FLOAT, 6}, {"fp4e2m1", ROCKE_DTYPE_FLOAT, 4}, + {"e8m0", ROCKE_DTYPE_FLOAT, 8}, {"e4m3", ROCKE_DTYPE_FLOAT, 8}, + {"e5m3", ROCKE_DTYPE_FLOAT, 8}, {"i4", ROCKE_DTYPE_INTEGER, 4}, + {"iu4", ROCKE_DTYPE_INTEGER, 4}, {"i8", ROCKE_DTYPE_INTEGER, 8}, + {"iu8", ROCKE_DTYPE_INTEGER, 8}, {"i16", ROCKE_DTYPE_INTEGER, 16}, + {"i32", ROCKE_DTYPE_INTEGER, 32}, {"i64", ROCKE_DTYPE_INTEGER, 64}, + {"i1", ROCKE_DTYPE_PREDICATE, 1}, +}; + +const rocke_dtype_info_t* rocke_dtype_info(const char* name) +{ + if(!name) + return NULL; + char scratch[64]; + const char* key = rocke_normalize_dtype(name, scratch, sizeof(scratch)); + for(const auto& info : k_dtype_info) + if(strcmp(key, info.name) == 0) + return &info; + return NULL; +} diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp index bafe79e1f905..d5e5bb48ee58 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp @@ -21,6 +21,7 @@ #include #include "rocke/arena.h" +#include "rocke/dtypes.h" #include "rocke/ir.h" #include "rocke/ir_internal.h" @@ -55,6 +56,13 @@ ROCKE_SCALAR_SINGLETON(rocke_f32, ROCKE_SCALAR_F32, "f32") ROCKE_SCALAR_SINGLETON(rocke_fp8e4m3, ROCKE_SCALAR_FP8E4M3, "fp8e4m3") ROCKE_SCALAR_SINGLETON(rocke_bf8e5m2, ROCKE_SCALAR_BF8E5M2, "bf8e5m2") +ROCKE_SCALAR_SINGLETON(rocke_fp4e2m1, ROCKE_SCALAR_FP4E2M1, "fp4e2m1") +ROCKE_SCALAR_SINGLETON(rocke_fp6e2m3, ROCKE_SCALAR_FP6E2M3, "fp6e2m3") +ROCKE_SCALAR_SINGLETON(rocke_fp6e3m2, ROCKE_SCALAR_FP6E3M2, "fp6e3m2") +ROCKE_SCALAR_SINGLETON(rocke_e8m0, ROCKE_SCALAR_E8M0, "e8m0") +ROCKE_SCALAR_SINGLETON(rocke_e4m3, ROCKE_SCALAR_E4M3, "e4m3") +ROCKE_SCALAR_SINGLETON(rocke_e5m3, ROCKE_SCALAR_E5M3, "e5m3") + #undef ROCKE_SCALAR_SINGLETON const rocke_type_t* rocke_scalar_by_name(const char* name) @@ -81,9 +89,33 @@ const rocke_type_t* rocke_scalar_by_name(const char* name) return rocke_fp8e4m3(); if(strcmp(name, "bf8e5m2") == 0) return rocke_bf8e5m2(); + if(strcmp(name, "fp4e2m1") == 0) + return rocke_fp4e2m1(); + if(strcmp(name, "fp6e2m3") == 0) + return rocke_fp6e2m3(); + if(strcmp(name, "fp6e3m2") == 0) + return rocke_fp6e3m2(); + if(strcmp(name, "e8m0") == 0) + return rocke_e8m0(); + if(strcmp(name, "e4m3") == 0) + return rocke_e4m3(); + if(strcmp(name, "e5m3") == 0) + return rocke_e5m3(); return NULL; } +const rocke_type_t* rocke_dtype_to_ir_type(const char* dtype) +{ + const rocke_dtype_info_t* info = rocke_dtype_info(dtype); + if(!info) + return NULL; + if(strcmp(info->name, "fp16") == 0) + return rocke_f16(); + if(strcmp(info->name, "fp32") == 0) + return rocke_f32(); + return rocke_scalar_by_name(info->name); +} + /* VectorType(elem, count) -> "vec<{elem}x{count}>" */ const rocke_type_t* rocke_vector_type(rocke_ir_builder_t* b, const rocke_type_t* elem, int count) { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/core.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/core.cpp index 02cb5a9176cd..039b225b394e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/core.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/core.cpp @@ -895,6 +895,60 @@ rocke_hip_arch_t rocke_hip_arch_from_gfx(const char* gfx) /* ============================== public entry ======================== */ +/* Mirrors _extra_vector_declarations: preserve the static prologue and add + * only widths actually encountered, in parameter/operand/result walk order. */ +static void + h_extra_vector_type(ckc::rocke_h_lowerer_t* lw, const rocke_type_t* t, rocke_strbuf_t* out) +{ + if(!t) + return; + if(t->kind == ROCKE_TYPE_PTR) + h_extra_vector_type(lw, t->pointee, out); + else if(t->kind == ROCKE_TYPE_SMEM) + h_extra_vector_type(lw, t->elem, out); + else if(t->kind == ROCKE_TYPE_VECTOR) + { + const char* name = ckc::rocke_h_type_to_hip(lw, t); + const char* prefix + = strcmp(t->elem->name, "i1") == 0 + ? "boolx" + : ckc::rocke_h_vec_prefix_checked(lw, t->elem->name, true, "vector type"); + const char* scalar + = strcmp(prefix, "i8x") == 0 ? "int8_t" : ckc::rocke_h_hip_scalar(t->elem->name); + const char* macro + = rocke_arena_printf(&lw->b->arena, "_ROCKE_VEC(%s, %s, %d)", scalar, prefix, t->count); + const char* declaration + = rocke_arena_printf(&lw->b->arena, + "using %s = %s __attribute__((ext_vector_type(%d)));", + name, + scalar, + t->count); + if(!macro || !declaration) + ckc::rocke_h_fail(lw, ROCKE_ERR_OOM, "vector declaration allocation failed"); + if(!strstr(ROCKE_HIP_PROLOGUE, macro) && !strstr(rocke_strbuf_cstr(out), declaration)) + { + rocke_strbuf_append(out, declaration); + rocke_strbuf_append_char(out, '\n'); + } + } +} + +static void h_extra_vector_region(ckc::rocke_h_lowerer_t* lw, + const rocke_region_t* region, + rocke_strbuf_t* out) +{ + for(int i = 0; i < region->num_ops; ++i) + { + const auto* op = region->ops[i]; + for(int j = 0; j < op->num_operands; ++j) + h_extra_vector_type(lw, op->operands[j]->type, out); + for(int j = 0; j < op->num_results; ++j) + h_extra_vector_type(lw, op->results[j]->type, out); + for(int j = 0; j < op->num_regions; ++j) + h_extra_vector_region(lw, op->regions[j], out); + } +} + rocke_status_t rocke_lower_kernel_to_hip(rocke_ir_builder_t* b, const rocke_kernel_def_t* kernel, const rocke_lower_hip_opts_t* opts, @@ -993,6 +1047,9 @@ rocke_status_t rocke_lower_kernel_to_hip(rocke_ir_builder_t* b, { rocke_strbuf_append(out, ROCKE_HIP_PROLOGUE); rocke_strbuf_append_char(out, '\n'); + for(int j = 0; j < kernel->num_params; ++j) + h_extra_vector_type(&lw, kernel->params[j]->type, out); + h_extra_vector_region(&lw, kernel->body, out); } /* head */ rocke_strbuf_appendf(out, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp new file mode 100644 index 000000000000..b1a2dda9cb6c --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp @@ -0,0 +1,265 @@ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +/* Mirrors core/storage.py: little-endian patterns, checked sizes and coordinates. + * Matrix and scale operands share these mechanics without sharing semantics. + */ +#include "rocke/storage.h" + +#include + +static bool valid(const rocke_bit_packing_t* p) +{ + return p && p->element_bits >= 1 && p->element_bits <= p->slot_bits && p->slot_bits <= 64; +} + +static bool add(uint64_t a, uint64_t b, uint64_t* out) +{ + if(!out || a > UINT64_MAX - b) + return false; + *out = a + b; + return true; +} + +static bool mul(uint64_t a, uint64_t b, uint64_t* out) +{ + if(!out || (b && a > UINT64_MAX / b)) + return false; + *out = a * b; + return true; +} + +bool rocke_bit_packing_init(rocke_bit_packing_t* out, int element_bits, int slot_bits) +{ + rocke_bit_packing_t p = {element_bits, slot_bits ? slot_bits : element_bits}; + if(!out || !valid(&p)) + return false; + *out = p; + return true; +} + +bool rocke_bit_packing_group(const rocke_bit_packing_t* p, + int carrier_bits, + uint64_t* values, + uint64_t* carriers) +{ + if(!valid(p) || !values || !carriers + || (carrier_bits != 8 && carrier_bits != 16 && carrier_bits != 32 && carrier_bits != 64)) + return false; + int a = p->slot_bits, b = carrier_bits; + while(b) + { + int next = a % b; + a = b; + b = next; + } + *values = carrier_bits / a; + *carriers = p->slot_bits / a; + return true; +} + +bool rocke_bit_packing_offset(const rocke_bit_packing_t* p, uint64_t index, uint64_t* bits) +{ + return valid(p) && mul(index, p->slot_bits, bits); +} + +bool rocke_bit_packing_bytes(const rocke_bit_packing_t* p, + uint64_t count, + uint64_t bit_offset, + uint64_t* bytes) +{ + uint64_t bits; + if(!bytes || !rocke_bit_packing_offset(p, count, &bits)) + return false; + if(!count) + { + *bytes = 0; + return true; + } + if(!add(bits, bit_offset, &bits)) + return false; + *bytes = bits / 8 + (bits % 8 != 0); + return true; +} + +bool rocke_bit_pack(const rocke_bit_packing_t* p, + const uint64_t* patterns, + size_t count, + uint64_t bit_offset, + uint8_t* dst, + size_t dst_size) +{ + uint64_t bytes; + if(!rocke_bit_packing_bytes(p, count, bit_offset, &bytes) || bytes > dst_size + || (count && (!patterns || !dst))) + return false; + for(size_t i = 0; i < count; ++i) + if(p->element_bits < 64 && patterns[i] >= (uint64_t(1) << p->element_bits)) + return false; + if(bytes) + memset(dst, 0, (size_t)bytes); + for(size_t i = 0; i < count; ++i) + { + const uint64_t pos = bit_offset + i * uint64_t(p->slot_bits); + for(int bit = 0; bit < p->element_bits; ++bit) + { + const uint64_t at = pos + bit; + dst[at / 8] |= uint8_t(((patterns[i] >> bit) & 1) << (at % 8)); + } + } + return true; +} + +bool rocke_bit_unpack(const rocke_bit_packing_t* p, + const uint8_t* src, + size_t src_size, + size_t count, + uint64_t bit_offset, + uint64_t* patterns) +{ + uint64_t bytes; + if(!rocke_bit_packing_bytes(p, count, bit_offset, &bytes) || bytes > src_size + || (count && (!patterns || !src))) + return false; + for(size_t i = 0; i < count; ++i) + { + patterns[i] = 0; + const uint64_t pos = bit_offset + i * uint64_t(p->slot_bits); + for(int bit = 0; bit < p->element_bits; ++bit) + { + const uint64_t at = pos + bit; + patterns[i] |= uint64_t((src[at / 8] >> (at % 8)) & 1) << bit; + } + } + return true; +} + +bool rocke_fragment_packing_init(rocke_fragment_packing_t* out, + const rocke_bit_packing_t* p, + uint64_t count, + int carrier_bits, + uint64_t carrier_count) +{ + uint64_t values, carriers, bits, capacity; + if(!out || !rocke_bit_packing_group(p, carrier_bits, &values, &carriers) + || !rocke_bit_packing_offset(p, count, &bits) || !mul(carrier_count, carrier_bits, &capacity) + || bits > capacity) + return false; + *out = {*p, count, carrier_bits, carrier_count}; + return true; +} + +bool rocke_fragment_pack(const rocke_fragment_packing_t* p, + const uint64_t* patterns, + size_t count, + uint64_t* carriers, + size_t carrier_count) +{ + rocke_fragment_packing_t checked; + if(!p + || !rocke_fragment_packing_init( + &checked, &p->packing, p->count, p->carrier_bits, p->carrier_count) + || count != p->count || carrier_count != p->carrier_count || (count && !patterns) + || (carrier_count && !carriers)) + return false; + for(size_t i = 0; i < count; ++i) + if(p->packing.element_bits < 64 && patterns[i] >= (uint64_t(1) << p->packing.element_bits)) + return false; + for(size_t i = 0; i < carrier_count; ++i) + carriers[i] = 0; + for(size_t i = 0; i < count; ++i) + for(int bit = 0; bit < p->packing.element_bits; ++bit) + { + uint64_t at = i * uint64_t(p->packing.slot_bits) + bit; + carriers[at / p->carrier_bits] |= ((patterns[i] >> bit) & 1) << (at % p->carrier_bits); + } + return true; +} + +bool rocke_tensor_storage_init(rocke_tensor_storage_t* out, + const char* dtype, + uint64_t rows, + uint64_t cols, + uint64_t row_stride_bytes, + int slot_bits, + uint64_t base_bit_offset, + uint64_t alignment_bytes) +{ + rocke_tensor_storage_t p = {}; + p.dtype = rocke_dtype_info(dtype); + uint64_t row_bytes, bytes; + if(!out || !p.dtype || !alignment_bytes || (alignment_bytes & (alignment_bytes - 1)) + || !rocke_bit_packing_init(&p.packing, p.dtype->encoded_bits, slot_bits) + || !rocke_bit_packing_bytes(&p.packing, cols, base_bit_offset, &row_bytes)) + return false; + p.rows = rows; + p.cols = cols; + p.row_stride_bytes = row_stride_bytes == UINT64_MAX ? row_bytes : row_stride_bytes; + p.base_bit_offset = base_bit_offset; + p.alignment_bytes = alignment_bytes; + if(p.row_stride_bytes < row_bytes || !rocke_tensor_storage_bytes(&p, &bytes)) + return false; + *out = p; + return true; +} + +bool rocke_tensor_storage_bytes(const rocke_tensor_storage_t* p, uint64_t* bytes) +{ + if(!p || !bytes || !valid(&p->packing)) + return false; + if(!p->rows || !p->cols) + { + *bytes = 0; + return true; + } + uint64_t row_bytes, prefix; + return rocke_bit_packing_bytes(&p->packing, p->cols, p->base_bit_offset, &row_bytes) + && mul(p->rows - 1, p->row_stride_bytes, &prefix) && add(prefix, row_bytes, bytes); +} + +bool rocke_tensor_storage_address( + const rocke_tensor_storage_t* p, uint64_t row, uint64_t col, uint64_t* byte, int* shift) +{ + if(!p || !byte || !shift || row >= p->rows || col >= p->cols) + return false; + uint64_t bits, prefix; + if(!rocke_bit_packing_offset(&p->packing, col, &bits) || !add(bits, p->base_bit_offset, &bits) + || !mul(row, p->row_stride_bytes, &prefix) || !add(prefix, bits / 8, byte)) + return false; + *shift = bits % 8; + return true; +} + +bool rocke_matrix_fragment_layout_init(rocke_matrix_fragment_layout_t* out, + const rocke_fragment_packing_t* fragment, + int chunk_elements, + int lane_groups, + int lanes_per_group) +{ + rocke_fragment_packing_t checked; + uint64_t bits; + if(!out || !fragment || chunk_elements <= 0 || lane_groups <= 0 || lanes_per_group <= 0 + || fragment->count % chunk_elements + || !rocke_fragment_packing_init(&checked, + &fragment->packing, + fragment->count, + fragment->carrier_bits, + fragment->carrier_count) + || !rocke_bit_packing_offset(&fragment->packing, chunk_elements, &bits) || bits % 8) + return false; + *out = {*fragment, chunk_elements, lane_groups, lanes_per_group}; + return true; +} + +bool rocke_matrix_fragment_coord( + const rocke_matrix_fragment_layout_t* p, int lane, uint64_t slot, uint64_t* row, uint64_t* k) +{ + if(!p || !row || !k || lane < 0 || p->lanes_per_group <= 0 || p->lane_groups <= 0 + || p->chunk_elements <= 0 || uint64_t(lane) >= uint64_t(p->lane_groups) * p->lanes_per_group + || slot >= p->fragment.count) + return false; + uint64_t chunk; + *row = lane % p->lanes_per_group; + return mul(slot / p->chunk_elements, p->lane_groups, &chunk) + && add(chunk, lane / p->lanes_per_group, &chunk) && mul(chunk, p->chunk_elements, &chunk) + && add(chunk, slot % p->chunk_elements, k); +} diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp new file mode 100644 index 000000000000..ecb70efaefb2 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp @@ -0,0 +1,206 @@ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +/* Descriptor-driven transport. Mirrors helpers/mma_io.py in emission order. */ +#include "rocke/helper_rocke.helpers.mma_io.h" + +#include +#include + +#include "rocke/error_boundary.hpp" + +const rocke_type_t* rocke_storage_ir_type(const char* dtype) +{ + const rocke_dtype_info_t* info = rocke_dtype_info(dtype); + if(!info) + return NULL; + if(info->encoded_bits % 8 || strcmp(info->name, "e8m0") == 0 || strcmp(info->name, "e4m3") == 0 + || strcmp(info->name, "e5m3") == 0) + return rocke_i8(); + return rocke_dtype_to_ir_type(info->name); +} + +static uint64_t gcd(uint64_t a, uint64_t b) +{ + while(b) + { + uint64_t next = a % b; + a = b; + b = next; + } + return a; +} + +rocke_value_t* rocke_h_load_matrix_fragment(rocke_ir_builder_t* b, + rocke_value_t* ptr, + rocke_value_t* row_base, + rocke_value_t* lane_group, + uint64_t k0, + const rocke_tensor_storage_t* storage, + const rocke_matrix_fragment_layout_t* layout, + const rocke_type_t* carrier_type) +{ + return ckc::guard_builder(b, [&]() -> rocke_value_t* { + if(!rocke_i_live(b)) + return NULL; + if(!storage || !storage->dtype || !layout || !carrier_type || !ptr || !row_base + || !lane_group) + ckc::raise_status(ROCKE_ERR_VALUE, "null matrix fragment argument"); + const auto& packing = layout->fragment; + rocke_matrix_fragment_layout_t checked; + if(!rocke_matrix_fragment_layout_init(&checked, + &packing, + layout->chunk_elements, + layout->lane_groups, + layout->lanes_per_group) + || !packing.count || packing.count > INT_MAX || packing.carrier_count > INT_MAX) + ckc::raise_status(ROCKE_ERR_VALUE, "invalid matrix fragment chunk layout"); + const rocke_type_t* unit_type = rocke_storage_ir_type(storage->dtype->name); + if(!unit_type || ptr->type->kind != ROCKE_TYPE_PTR + || !rocke_type_eq(ptr->type->pointee, unit_type)) + ckc::raise_status(ROCKE_ERR_VALUE, "matrix pointer storage type mismatch"); + const uint64_t unit_bytes = rocke_dtype_info(unit_type->name)->encoded_bits / 8; + if(storage->packing.element_bits != packing.packing.element_bits + || storage->packing.slot_bits != packing.packing.slot_bits || storage->base_bit_offset) + ckc::raise_status(ROCKE_ERR_VALUE, + "matrix fragment requires matching packing and a zero bit origin"); + const uint64_t span = packing.count * layout->lane_groups; + if(k0 > storage->cols || span > storage->cols - k0) + ckc::raise_status(ROCKE_ERR_VALUE, "matrix fragment exceeds the packed row"); + uint64_t origin_bits, chunk_bytes; + if(!rocke_bit_packing_offset(&storage->packing, k0, &origin_bits) + || !rocke_bit_packing_bytes(&packing.packing, layout->chunk_elements, 0, &chunk_bytes) + || origin_bits % (8 * unit_bytes) || chunk_bytes % unit_bytes) + ckc::raise_status(ROCKE_ERR_VALUE, + "matrix fragment is not aligned to pointer storage units"); + const rocke_dtype_info_t* carrier_info = rocke_dtype_info(carrier_type->name); + if(!carrier_info || carrier_info->encoded_bits != packing.carrier_bits) + ckc::raise_status(ROCKE_ERR_VALUE, "matrix carrier type width mismatch"); + const uint64_t payload_bits = packing.count * packing.packing.slot_bits; + if(payload_bits % packing.carrier_bits) + ckc::raise_status(ROCKE_ERR_VALUE, + "matrix fragment payload must occupy whole carriers"); + const int live_carriers = payload_bits / packing.carrier_bits; + const int padding = packing.carrier_count - live_carriers; + if(padding && !rocke_type_eq(carrier_type, rocke_i32())) + ckc::raise_status(ROCKE_ERR_VALUE, + "padded matrix fragments currently require i32 carriers"); + const uint64_t chunk_units = chunk_bytes / unit_bytes; + const uint64_t origin_bytes = origin_bits / 8; + if(origin_bytes / unit_bytes > INT_MAX || chunk_units * layout->lane_groups > INT_MAX) + ckc::raise_status(ROCKE_ERR_VALUE, "matrix fragment offset exceeds i32 range"); + int alignment = gcd(gcd(storage->alignment_bytes, storage->row_stride_bytes), + gcd(chunk_bytes, origin_bytes)); + auto* lane_chunk = rocke_b_mul(b, lane_group, rocke_b_const_i32(b, chunk_units)); + auto* step_base = rocke_b_add(b, row_base, rocke_b_const_i32(b, origin_bytes / unit_bytes)); + /* Collect loads before concatenation to preserve Python SSA numbering. */ + rocke_value_t** chunks = (rocke_value_t**)rocke_arena_alloc( + &b->arena, size_t(packing.count) * sizeof(rocke_value_t*)); + if(!chunks) + ckc::raise_status(ROCKE_ERR_OOM, "matrix fragment allocation failed"); + int num_chunks = 0; + for(uint64_t j = 0; j < packing.count / layout->chunk_elements; ++j) + { + auto* offset = rocke_b_add( + b, + rocke_b_add( + b, step_base, rocke_b_const_i32(b, j * layout->lane_groups * chunk_units)), + lane_chunk); + uint64_t remaining = chunk_units, consumed = 0; + const int max_width = unit_bytes == 4 ? 8 : 16; + while(remaining) + { + int width = 1; + while(width < max_width && uint64_t(width * 2) <= remaining) + width *= 2; + auto* at + = consumed ? rocke_b_add(b, offset, rocke_b_const_i32(b, consumed)) : offset; + int load_align = gcd(alignment, consumed * unit_bytes); + auto* value + = width == 1 ? rocke_b_vector_splat( + b, rocke_b_global_load(b, ptr, at, unit_type, load_align), 1) + : rocke_b_global_load_vN(b, ptr, at, unit_type, width, load_align); + chunks[num_chunks++] = value; + consumed += width; + remaining -= width; + } + } + auto* payload = chunks[0]; + for(int i = 1; i < num_chunks; ++i) + payload = rocke_b_vec_concat(b, payload, chunks[i]); + const auto* payload_type = rocke_vector_type(b, carrier_type, live_carriers); + if(!rocke_type_eq(payload->type, payload_type)) + payload = rocke_b_bitcast(b, payload, payload_type); + if(padding) + payload = rocke_b_vec_concat( + b, payload, rocke_b_vector_splat(b, rocke_b_const_i32(b, 0), padding)); + return payload; + }); +} + +rocke_status_t rocke_h_pack_fragment_bits(rocke_ir_builder_t* b, + rocke_load_bits_fn load_bits, + void* ctx, + const rocke_fragment_packing_t* fragment, + rocke_value_t** words, + size_t word_count) +{ + return ckc::guard_status(b, [&]() -> rocke_status_t { + if(!rocke_i_live(b)) + return ROCKE_ERR_VALUE; + rocke_fragment_packing_t checked; + if(!fragment || !load_bits || !words + || !rocke_fragment_packing_init(&checked, + &fragment->packing, + fragment->count, + fragment->carrier_bits, + fragment->carrier_count) + || word_count != fragment->carrier_count || fragment->count > INT_MAX) + ckc::raise_status(ROCKE_ERR_VALUE, "invalid pattern packing arguments"); + const auto& f = *fragment; + if(f.carrier_bits != 32 && f.carrier_bits != 64) + ckc::raise_status(ROCKE_ERR_VALUE, + "IR pattern packing currently requires i32 or i64 carriers"); + const auto* word_type = f.carrier_bits == 64 ? rocke_i64() : rocke_i32(); + auto constant = [&](uint64_t value) { + return f.carrier_bits == 64 ? rocke_b_const_i64(b, int64_t(value)) + : rocke_b_const_i32(b, int32_t(value)); + }; + for(size_t j = 0; j < word_count; ++j) + words[j] = constant(0); + for(uint64_t j = 0; j < f.count; ++j) + { + auto* pattern = load_bits(b, (int)j, ctx); + if(!pattern + || (!rocke_type_eq(pattern->type, rocke_i8()) + && !rocke_type_eq(pattern->type, rocke_i16()) + && !rocke_type_eq(pattern->type, rocke_i32()) + && !rocke_type_eq(pattern->type, rocke_i64()))) + ckc::raise_status(ROCKE_ERR_VALUE, + "pattern packing requires unsigned patterns in integer carriers"); + const int bits = rocke_dtype_info(pattern->type->name)->encoded_bits; + if(bits < f.packing.element_bits) + ckc::raise_status(ROCKE_ERR_VALUE, "pattern carrier is smaller than encoded width"); + if(bits < f.carrier_bits) + pattern = rocke_b_zext(b, pattern, word_type); + else if(bits > f.carrier_bits) + ckc::raise_status(ROCKE_ERR_VALUE, "pattern carrier is wider than output carrier"); + uint64_t start = j * f.packing.slot_bits; + int remaining = f.packing.element_bits, consumed = 0; + while(remaining) + { + const uint64_t word = start / f.carrier_bits; + const int shift = start % f.carrier_bits; + const int take + = remaining < f.carrier_bits - shift ? remaining : f.carrier_bits - shift; + auto* part = consumed ? rocke_b_lshr(b, pattern, constant(consumed)) : pattern; + if(take < remaining) + part = rocke_b_land(b, part, constant((uint64_t(1) << take) - 1)); + words[word] = rocke_b_lor(b, words[word], rocke_b_shl(b, part, constant(shift))); + start += take; + consumed += take; + remaining -= take; + } + } + return ROCKE_OK; + }); +} diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/quant.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/quant.cpp index e6e92d1206e2..fcb13a085464 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/quant.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/quant.cpp @@ -1,37 +1,8 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT -/* - * helper_rocke.helpers.quant.c -- C99 port of - * rocke.helpers.quant.quant_ir_type. - * - * Faithful translation of: - * - * _QDTYPE_ALIAS = { - * "i8": "i8", "int8": "i8", - * "fp8e4m3": "fp8e4m3", "fp8": "fp8e4m3", "fp8_e4m3": "fp8e4m3", - * "bf8e5m2": "bf8e5m2", "bf8": "bf8e5m2", "fp8_e5m2": "bf8e5m2", - * } - * - * def _canon(qdtype: str) -> QDType: - * if qdtype not in _QDTYPE_ALIAS: - * raise ValueError( - * f"unsupported quant dtype {qdtype!r}; expected one of " - * f"{sorted(_QDTYPE_ALIAS)}") - * return _QDTYPE_ALIAS[qdtype] - * - * def quant_ir_type(qdtype: str) -> Type: - * canon = _canon(qdtype) - * if canon == "i8": return I8 - * if canon == "fp8e4m3": return FP8E4M3 - * if canon == "bf8e5m2": return BF8E5M2 - * raise ValueError(f"unreachable: canon={canon!r}") - * - * Mapping invariants (must stay byte-identical to the Python so downstream IR is - * identical): - * "i8" / "int8" -> rocke_i8() (Python I8) - * "fp8e4m3" / "fp8" / "fp8_e4m3" -> rocke_fp8e4m3() (Python FP8E4M3) - * "bf8e5m2" / "bf8" / "fp8_e5m2" -> rocke_bf8e5m2() (Python BF8E5M2) - * else -> NULL (Python ValueError) +/* Mirrors helpers/quant.py::quant_ir_type. Preserve the existing quant aliases + * and delegate logical type resolution to rocke_dtype_to_ir_type. FP4/FP6 + * recognition is independent of scalar quantization/conversion support. */ #include "rocke/helper_rocke.helpers.quant.h" @@ -49,20 +20,24 @@ const rocke_type_t* rocke_quant_ir_type(const char* qdtype) /* `_canon` -> "i8" : keys "i8", "int8". */ if(strcmp(qdtype, "i8") == 0 || strcmp(qdtype, "int8") == 0) { - return rocke_i8(); + return rocke_dtype_to_ir_type("i8"); } /* `_canon` -> "fp8e4m3" : keys "fp8e4m3", "fp8", "fp8_e4m3". */ if(strcmp(qdtype, "fp8e4m3") == 0 || strcmp(qdtype, "fp8") == 0 || strcmp(qdtype, "fp8_e4m3") == 0) { - return rocke_fp8e4m3(); + return rocke_dtype_to_ir_type("fp8e4m3"); } /* `_canon` -> "bf8e5m2" : keys "bf8e5m2", "bf8", "fp8_e5m2". */ if(strcmp(qdtype, "bf8e5m2") == 0 || strcmp(qdtype, "bf8") == 0 || strcmp(qdtype, "fp8_e5m2") == 0) { - return rocke_bf8e5m2(); + return rocke_dtype_to_ir_type("bf8e5m2"); } + if(strcmp(qdtype, "fp4") == 0 || strcmp(qdtype, "fp4e2m1") == 0 || strcmp(qdtype, "fp6") == 0 + || strcmp(qdtype, "fp6e2m3") == 0 || strcmp(qdtype, "bf6") == 0 + || strcmp(qdtype, "fp6e3m2") == 0) + return rocke_dtype_to_ir_type(qdtype); /* Python: _canon raises ValueError. No builder here, so signal via NULL. */ return NULL; } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/dtypes.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/dtypes.h index f7da21efad31..6d814029b952 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/dtypes.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/dtypes.h @@ -28,6 +28,24 @@ extern "C" { * Mirrors core/dtypes.py::normalize_dtype (strip + lower + _DTYPE_ALIASES.get). */ const char* rocke_normalize_dtype(const char* name, char* scratch, size_t scratch_cap); +/* Borrowed immutable encoding descriptor, valid for the process lifetime. */ +typedef enum rocke_dtype_category +{ + ROCKE_DTYPE_FLOAT, + ROCKE_DTYPE_INTEGER, + ROCKE_DTYPE_PREDICATE +} rocke_dtype_category_t; + +typedef struct rocke_dtype_info +{ + const char* name; + rocke_dtype_category_t category; + int encoded_bits; +} rocke_dtype_info_t; + +/* NULL for unknown encodings. No architecture or operation support implied. */ +const rocke_dtype_info_t* rocke_dtype_info(const char* name); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.mma_io.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.mma_io.h new file mode 100644 index 000000000000..8f5db63aa960 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.mma_io.h @@ -0,0 +1,41 @@ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +/* Mirrors helpers/mma_io.py; descriptors expand to ordinary IR before serialization. + * storage_ir_type -> rocke_storage_ir_type + * load_matrix_fragment -> rocke_h_load_matrix_fragment + * pack_fragment_bits -> rocke_h_pack_fragment_bits + */ +#ifndef ROCKE_HELPER_MMA_IO_H +#define ROCKE_HELPER_MMA_IO_H + +#include "rocke/ir.h" +#include "rocke/storage.h" + +#ifdef __cplusplus +extern "C" { +#endif + +const rocke_type_t* rocke_storage_ir_type(const char* dtype); +rocke_value_t* rocke_h_load_matrix_fragment(rocke_ir_builder_t* b, + rocke_value_t* ptr, + rocke_value_t* row_base, + rocke_value_t* lane_group, + uint64_t k0, + const rocke_tensor_storage_t* storage, + const rocke_matrix_fragment_layout_t* layout, + const rocke_type_t* carrier_type); + +/* Callback returns a canonical unsigned pattern in an integer carrier. + * ctx and output pointer array are borrowed for the duration of this call. */ +typedef rocke_value_t* (*rocke_load_bits_fn)(rocke_ir_builder_t* b, int index, void* ctx); +rocke_status_t rocke_h_pack_fragment_bits(rocke_ir_builder_t* b, + rocke_load_bits_fn load_bits, + void* ctx, + const rocke_fragment_packing_t* fragment, + rocke_value_t** words, + size_t word_count); + +#ifdef __cplusplus +} +#endif +#endif /* ROCKE_HELPER_MMA_IO_H */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.quant.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.quant.h index 540e797219ea..2b24e6d2e172 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.quant.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.quant.h @@ -3,42 +3,11 @@ * * rocke/helper_rocke.helpers.quant.h -- C99 port of rocke.helpers.quant. * - * Scope of THIS file: only `quant_ir_type` is ported here (the other quant - * helpers in helpers/quant.py -- quantize_scalar_f32, dequantize_scalar_to_f32, - * pack_quant_chunk_local_f32, store_packed_chunk_local, quant_max_abs, - * ir_to_qdtype -- are out of scope for this phase). The function maps a - * quant-dtype alias spelling string to the canonical IR scalar type, - * byte-identically to the Python: - * - * def quant_ir_type(qdtype: str) -> Type: - * canon = _canon(qdtype) - * if canon == "i8": return I8 - * if canon == "fp8e4m3": return FP8E4M3 - * if canon == "bf8e5m2": return BF8E5M2 - * raise ValueError(...) - * - * with `_canon` normalising the accepted aliases: - * - * "i8" / "int8" -> "i8" - * "fp8e4m3" / "fp8" / "fp8_e4m3" -> "fp8e4m3" - * "bf8e5m2" / "bf8" / "fp8_e5m2" -> "bf8e5m2" - * - * and raising ValueError for any other spelling. - * - * The Python helper takes no IRBuilder and raises ValueError on an unsupported - * dtype. C99 has no exceptions, so we expose two faithful spellings, mirroring - * the sibling helper_rocke.helpers.io port: - * - * rocke_quant_ir_type(qdtype) - * Pure map. Returns the interned scalar singleton - * (rocke_i8()/rocke_fp8e4m3()/rocke_bf8e5m2()) for any accepted alias; returns - * NULL for anything else (the analog of "raise ValueError"). No error - * state is set because there is no builder. - * - * rocke_b_quant_ir_type(b, qdtype) - * Builder-aware spelling matching the rest of the C port's sticky-error - * model: on an unsupported dtype it records ROCKE_ERR_VALUE + a message on - * the builder (mirroring the Python ValueError text) and returns NULL. + * Type resolution only; scalar quantization and conversion remain separate. + * rocke_quant_ir_type delegates to rocke_dtype_to_ir_type for nominal types, + * preserving the existing quant aliases and adding FP4/FP6 recognition. + * Returns a borrowed static singleton or NULL for an unsupported name. + * The builder-aware variant also records the corresponding error. */ #ifndef ROCKE_HELPER_ROCKE_HELPERS_QUANT_H #define ROCKE_HELPER_ROCKE_HELPERS_QUANT_H @@ -55,6 +24,9 @@ extern "C" { * "i8", "int8" -> rocke_i8() * "fp8e4m3", "fp8", "fp8_e4m3" -> rocke_fp8e4m3() * "bf8e5m2", "bf8", "fp8_e5m2" -> rocke_bf8e5m2() + * "fp4", "fp4e2m1" -> rocke_fp4e2m1() + * "fp6", "fp6e2m3" -> rocke_fp6e2m3() + * "bf6", "fp6e3m2" -> rocke_fp6e3m2() * Returns NULL for any other value (the Python ValueError path). */ const rocke_type_t* rocke_quant_ir_type(const char* qdtype); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h index 59ae31feced3..131ea637e341 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h @@ -105,6 +105,12 @@ typedef enum rocke_scalar_kind ROCKE_SCALAR_F32, ROCKE_SCALAR_FP8E4M3, ROCKE_SCALAR_BF8E5M2, + ROCKE_SCALAR_FP4E2M1, + ROCKE_SCALAR_FP6E2M3, + ROCKE_SCALAR_FP6E3M2, + ROCKE_SCALAR_E8M0, + ROCKE_SCALAR_E4M3, + ROCKE_SCALAR_E5M3, ROCKE_SCALAR__COUNT } rocke_scalar_kind_t; @@ -574,6 +580,15 @@ const rocke_type_t* rocke_f16(void); const rocke_type_t* rocke_f32(void); const rocke_type_t* rocke_fp8e4m3(void); const rocke_type_t* rocke_bf8e5m2(void); +const rocke_type_t* rocke_fp4e2m1(void); +const rocke_type_t* rocke_fp6e2m3(void); +const rocke_type_t* rocke_fp6e3m2(void); +const rocke_type_t* rocke_e8m0(void); +const rocke_type_t* rocke_e4m3(void); +const rocke_type_t* rocke_e5m3(void); + +/* Logical dtype resolver; NULL for unknown or unrepresented encodings. */ +const rocke_type_t* rocke_dtype_to_ir_type(const char* dtype); /* Look up a scalar singleton by canonical name ("i32",...); NULL if unknown. */ const rocke_type_t* rocke_scalar_by_name(const char* name); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h new file mode 100644 index 000000000000..3ea1794d6369 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h @@ -0,0 +1,114 @@ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +/* Shared bit storage; mirrors core/storage.py. + * BitPacking -> rocke_bit_packing_t + * FragmentPacking -> rocke_fragment_packing_t + * TensorStorage -> rocke_tensor_storage_t + * All descriptors are caller-owned. Functions return false for invalid inputs, + * overflow or insufficient buffers; no exceptions cross this C API. + */ +#ifndef ROCKE_STORAGE_H +#define ROCKE_STORAGE_H + +#include +#include +#include + +#include "rocke/dtypes.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct rocke_bit_packing +{ + int element_bits; + int slot_bits; /* 0 selects dense packing at construction. */ +} rocke_bit_packing_t; + +typedef struct rocke_fragment_packing +{ + rocke_bit_packing_t packing; + uint64_t count; + int carrier_bits; + uint64_t carrier_count; +} rocke_fragment_packing_t; + +typedef struct rocke_matrix_fragment_layout +{ + rocke_fragment_packing_t fragment; + int chunk_elements; + int lane_groups; + int lanes_per_group; +} rocke_matrix_fragment_layout_t; + +bool rocke_matrix_fragment_layout_init(rocke_matrix_fragment_layout_t* out, + const rocke_fragment_packing_t* fragment, + int chunk_elements, + int lane_groups, + int lanes_per_group); +bool rocke_matrix_fragment_coord( + const rocke_matrix_fragment_layout_t* p, int lane, uint64_t slot, uint64_t* row, uint64_t* k); + +typedef struct rocke_tensor_storage +{ + const rocke_dtype_info_t* dtype; /* Borrowed static descriptor. */ + uint64_t rows; + uint64_t cols; + uint64_t row_stride_bytes; + uint64_t base_bit_offset; + uint64_t alignment_bytes; + rocke_bit_packing_t packing; +} rocke_tensor_storage_t; + +bool rocke_bit_packing_init(rocke_bit_packing_t* out, int element_bits, int slot_bits); +bool rocke_bit_packing_group(const rocke_bit_packing_t* p, + int carrier_bits, + uint64_t* values, + uint64_t* carriers); +bool rocke_bit_packing_offset(const rocke_bit_packing_t* p, uint64_t index, uint64_t* bits); +bool rocke_bit_packing_bytes(const rocke_bit_packing_t* p, + uint64_t count, + uint64_t bit_offset, + uint64_t* bytes); +/* Pack writes a fresh zero-padded stream; caller must exclusively own dst. + * Unpack reads only the allocated bytes, including a partial final byte. */ +bool rocke_bit_pack(const rocke_bit_packing_t* p, + const uint64_t* patterns, + size_t count, + uint64_t bit_offset, + uint8_t* dst, + size_t dst_size); +bool rocke_bit_unpack(const rocke_bit_packing_t* p, + const uint8_t* src, + size_t src_size, + size_t count, + uint64_t bit_offset, + uint64_t* patterns); +bool rocke_fragment_packing_init(rocke_fragment_packing_t* out, + const rocke_bit_packing_t* p, + uint64_t count, + int carrier_bits, + uint64_t carrier_count); +bool rocke_fragment_pack(const rocke_fragment_packing_t* p, + const uint64_t* patterns, + size_t count, + uint64_t* carriers, + size_t carrier_count); +/* row_stride_bytes=UINT64_MAX selects a minimal packed row stride. */ +bool rocke_tensor_storage_init(rocke_tensor_storage_t* out, + const char* dtype, + uint64_t rows, + uint64_t cols, + uint64_t row_stride_bytes, + int slot_bits, + uint64_t base_bit_offset, + uint64_t alignment_bytes); +bool rocke_tensor_storage_bytes(const rocke_tensor_storage_t* p, uint64_t* bytes); +bool rocke_tensor_storage_address( + const rocke_tensor_storage_t* p, uint64_t row, uint64_t col, uint64_t* byte, int* shift); + +#ifdef __cplusplus +} +#endif +#endif /* ROCKE_STORAGE_H */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h index c85df088e922..08585dd6b734 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h @@ -14,6 +14,7 @@ #include "rocke/arch_target.h" #include "rocke/error.hpp" #include "rocke/ir.h" +#include "rocke/storage.h" typedef struct rocke_scale_packing { @@ -23,6 +24,52 @@ typedef struct rocke_scale_packing int block_k; } rocke_scale_packing_t; +typedef struct rocke_scale_association +{ + int block_k; +} rocke_scale_association_t; + +static inline rocke_scale_association_t rocke_scale_association(const rocke_scale_packing_t* p) +{ + if(p->block_k <= 0) + ckc::raise_status(ROCKE_ERR_VALUE, "scale block_k must be positive"); + return {p->block_k}; +} + +static inline rocke_fragment_packing_t rocke_scale_fragment(const rocke_scale_packing_t* p) +{ + rocke_bit_packing_t bits; + rocke_fragment_packing_t result; + rocke_scale_association(p); + if(p->count != 1 && p->count != 2 && p->count != 4 && p->count != 8) + ckc::raise_status(ROCKE_ERR_VALUE, "carrier_bits must be 8, 16, 32, or 64"); + if(!rocke_bit_packing_init(&bits, rocke_dtype_info("e8m0")->encoded_bits, 0) + || !rocke_fragment_packing_init(&result, &bits, p->count, p->count * 8, 1)) + ckc::raise_status(ROCKE_ERR_VALUE, "invalid scale packing"); + return result; +} + +static inline rocke_matrix_fragment_layout_t rocke_scaled_matrix_layout(const char* dtype, + int abi_words) +{ + const rocke_dtype_info_t* info = rocke_dtype_info(dtype); + int chunk = 0; + if(info && (strcmp(info->name, "fp8e4m3") == 0 || strcmp(info->name, "bf8e5m2") == 0)) + chunk = 16; + else if(info + && (strcmp(info->name, "fp4e2m1") == 0 || strcmp(info->name, "fp6e2m3") == 0 + || strcmp(info->name, "fp6e3m2") == 0)) + chunk = 32; + rocke_bit_packing_t bits; + rocke_fragment_packing_t fragment; + rocke_matrix_fragment_layout_t result; + if(!chunk || abi_words < 0 || !rocke_bit_packing_init(&bits, info->encoded_bits, 0) + || !rocke_fragment_packing_init(&fragment, &bits, 64, 32, abi_words) + || !rocke_matrix_fragment_layout_init(&result, &fragment, chunk, 2, 16)) + ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled matrix layout"); + return result; +} + typedef struct rocke_scaled_wmma_op { const char* op_id; @@ -45,6 +92,13 @@ static inline const rocke_mma_op_t* rocke_gfx1250_scaled_wmma(const char* op_id) return atom && strcmp(atom->family, "wmma_scaled") == 0 ? atom : NULL; } +static inline rocke_matrix_fragment_layout_t + rocke_scaled_wmma_matrix_layout(const rocke_mma_op_t* atom, bool for_b) +{ + return rocke_scaled_matrix_layout(for_b ? atom->b_dtype : atom->a_dtype, + for_b ? atom->b_frag_len : atom->a_frag_len); +} + static inline rocke_scaled_wmma_op_t rocke_scaled_wmma_contract(const rocke_mma_op_t* atom) { rocke_scaled_wmma_op_t spec = {}; @@ -104,7 +158,7 @@ static inline const rocke_mma_op_t* rocke_gfx1250_scaled_wmma_from_op(const rock static inline int rocke_scale_word_bits(const rocke_scale_packing_t* packing) { - return packing->count * 8; + return rocke_scale_fragment(packing).carrier_bits; } #endif /* ROCKE_WMMA_SCALE_INTERNAL_H */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md new file mode 100644 index 000000000000..37d8d51b91f0 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md @@ -0,0 +1,124 @@ +# Logical dtypes, storage, and operand packing + +Matrix inputs and scales use three separate contracts: + +1. A logical dtype identifies the numerical encoding and its bit width. +2. Tensor storage describes how elements occupy an addressable buffer. +3. An instruction fragment describes lane mapping, carrier capacity, and padding. + +The selected atom determines supported operand combinations and instruction +selectors. An integer carrier does not identify the numerical format. + +## Contents + +- [Logical dtypes](#logical-dtypes) +- [Common packing](#common-packing) +- [Tensor storage](#tensor-storage) +- [Matrix and scale layouts](#matrix-and-scale-layouts) +- [Mirroring and follow-up work](#mirroring-and-follow-up-work) + +## Logical dtypes + +[`dtype_info`](../../python/rocke/core/dtypes.py) and +[`rocke_dtype_info`](../../cpp/include/rocke/dtypes.h) return target-independent +encoding descriptors. `dtype_to_ir_type` and `rocke_dtype_to_ir_type` resolve +logical types, including distinct FP4 E2M1, FP6 E2M3, FP6 E3M2, and scale types. +`quant_ir_type` delegates type resolution to this common resolver. + +Recognition and serialization do not enable scalar conversion, arithmetic, or +an instruction on a target. In this implementation, FP4/FP6 and the nominal scale +types are used in authoring metadata; storage helpers expand their transport into +existing integer operations before serialization. Raw scalar lowering of these +types remains unsupported. Existing FP16/BF16 and FP8/BF8 lowering is unchanged. + +Scale-format validation remains separate from matrix alias normalization: +E5M3 is not BF8 E5M2. Registering a scale dtype does not enable a backend selector. + +## Common packing + +`BitPacking` / `rocke_bit_packing_t` describes unsigned bit patterns with an +encoded width and slot width. Dense packing uses equal widths. A wider slot has +unused high bits, which packers zero. The current encoding is least-significant +bit first in a little-endian byte stream. + +| Format | Dense group occupying whole 32-bit words | +|---|---| +| FP16/BF16 | 2 elements in 1 word | +| FP8/BF8 | 4 elements in 1 word | +| FP6/BF6 | 16 elements across 3 words | +| FP4 | 8 elements in 1 word | + +FP6 elements can cross byte and word boundaries. Packing cannot be expressed +generally as `32 // element_bits` elements per word. + +`FragmentPacking` / `rocke_fragment_packing_t` combines bit packing with a logical +count and a carrier width/count. Capacity is checked independently of payload +size; unused carriers are zero. Carriers may hold integer bits or typed values, +depending on the instruction ABI. + +Host pack/unpack routines process bit patterns only. They do not round, clamp, +decode floats, or apply scales. Their tests use literal expected bits and +independent integer arithmetic; numerical tests remain separate. + +## Tensor storage + +`TensorStorage` / `rocke_tensor_storage_t` initially supports a two-dimensional +logical shape with a contiguous packed inner axis, an explicit byte row stride, +bit offset, and base alignment. Nonnegative sizes and offsets are checked against +the native uint64 range. A/B descriptors are independent. + +For example, dense FP6 uses four elements in three bytes; `slot_bits=8` describes +one six-bit pattern in each byte instead. Both have the same logical dtype. +Address calculation includes the bit offset, and allocation bounds include the +partial tail byte. Empty views require no accesses. + +[`storage_ir_type`](../../python/rocke/helpers/mma_io.py) chooses an addressable unit. FP4/FP6 use I8 +units while retaining their logical identities in the storage descriptor. +This function is deliberately separate from `dtype_to_ir_type`. + +The fragment loader accepts complete, byte-aligned chunks. It derives load +alignment from base alignment, row stride, chunk spacing, and origin. A 24-byte +FP6 chunk becomes 16-byte and 8-byte loads without reading past the chunk. +Partial fragments and nonzero bit origins are rejected by this initial loader. +The host bit packer writes a fresh, exclusively owned output buffer; concurrent +packed stores are not provided. + +## Matrix and scale layouts + +`MatrixFragmentLayout` describes contiguous K chunks interleaved between lane +groups. Coordinate mapping belongs to the atom-specific layout, not to the +generic bit packer. `ScaledWmmaOp.matrix_layout(operand)` selects independent +A/B layouts. Existing `a_frag_len`/`b_frag_len` retain their ABI-vector meaning. + +The current gfx1250 scaled matrix layouts have 64 elements per lane and sixteen +i32 carriers. FP8 occupies all sixteen words. FP4 occupies eight and pads eight. +The FP6 transport descriptor occupies twelve and pads four, without enabling +FP6 catalog entries or numerical conversions. + +`ScaleAssociation` records `block_k`: how many source K elements share one +scale. This is separate from the bit layout. Existing `ScalePacking(count, +block_k)` remains a compatibility adapter, exposing both the association and a +shared `FragmentPacking`. + +Current scaled-WMMA scale fragments hold four eight-bit patterns in i32 for K32, +or eight in i64 for K16. The first K group occupies the low byte. The A/B scale +coordinate maps determine which scales each lane loads. The same +`pack_fragment_bits` helper can pack six-bit fields that cross word boundaries. + +## Mirroring and follow-up work + +The Python and C++ helpers expand descriptors before serialization. Their emitted +IR and HIP source are tested byte for byte, including FP16/BF16 typed carriers, +FP4/FP6/FP8 matrix payloads, and scale word packing. Existing GEMM signatures +remain compatible. HIP emits declarations for any encountered vector widths +absent from its fixed compatibility prologue. + +The shared descriptor/helper changes can be consumed by both FP4 and FP6 +branches. FP4/FP8 builder migration can use them directly. Future FP6 numerical +integration should consume the six-bit storage and fragment descriptors and +retain separate target and conversion validation. + +First-class tensor-view/fragment IR nodes, arbitrary packed axes, masked partial +tiles, concurrent packed stores, and scalar low-bit conversions are separate +extensions. Any descriptor retained in future IR nodes must serialize, validate, +participate in cache keys, and have a native implementation. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/__init__.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/__init__.py index bd0f96fc8493..57596e6ba255 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/__init__.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/__init__.py @@ -45,12 +45,19 @@ apply_codegen_policy, codegen_policy_for_kernel, ) -from .dtypes import normalize_dtype +from .dtypes import DTypeCategory, DTypeInfo, dtype_info, normalize_dtype from .ir import ( BF16, F16, F32, FP8E4M3, + FP4E2M1, + FP6E2M3, + FP6E3M2, + E8M0, + E4M3, + E5M3, + dtype_to_ir_type, I1, I8, I32, @@ -96,6 +103,16 @@ __all__ = [ "normalize_dtype", + "DTypeCategory", + "DTypeInfo", + "dtype_info", + "dtype_to_ir_type", + "FP4E2M1", + "FP6E2M3", + "FP6E3M2", + "E8M0", + "E4M3", + "E5M3", "CodegenPolicy", "SchedulerStrategy", "apply_codegen_policy", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py index 2febb7514b05..7f0284ed983c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py @@ -7,6 +7,36 @@ from dataclasses import dataclass from .target import ArchTarget, MmaOp, MmaScaleDType +from ..dtypes import dtype_info +from ..storage import BitPacking, FragmentPacking, MatrixFragmentLayout + + +@dataclass(frozen=True) +class ScaleAssociation: + """Number of source K elements sharing one scale.""" + + block_k: int + + def __post_init__(self) -> None: + if self.block_k <= 0: + raise ValueError("scale block_k must be positive") + + +def scaled_matrix_layout(dtype: str, abi_words: int) -> MatrixFragmentLayout: + """gfx1250 scaled operand layout, independent of atom availability. + + FP6 describes the transport contract for future catalog integration. + """ + info = dtype_info(dtype) + chunks = {"fp8e4m3": 16, "bf8e5m2": 16, "fp6e2m3": 32, "fp6e3m2": 32, "fp4e2m1": 32} + if info.name not in chunks: + raise ValueError(f"unsupported scaled matrix layout {dtype!r}") + return MatrixFragmentLayout( + FragmentPacking(BitPacking(info.encoded_bits), 64, 32, abi_words), + chunks[info.name], + 2, + 16, + ) @dataclass(frozen=True) @@ -20,13 +50,31 @@ class ScalePacking: count: int block_k: int + def __post_init__(self) -> None: + self.association + self.fragment + + @property + def association(self) -> ScaleAssociation: + return ScaleAssociation(self.block_k) + + @property + def packing(self) -> BitPacking: + return BitPacking(dtype_info("e8m0").encoded_bits) + + @property + def fragment(self) -> FragmentPacking: + return FragmentPacking( + self.packing, self.count, self.count * self.packing.element_bits, 1 + ) + @property def element_bits(self) -> int: - return 8 + return self.packing.element_bits @property def word_bits(self) -> int: - return self.count * self.element_bits + return self.fragment.carrier_bits @property def llvm_type(self) -> str: @@ -42,6 +90,22 @@ class ScaledWmmaOp: scale_formats: tuple[int, int] scales: ScalePacking + def matrix_layout(self, operand: str) -> MatrixFragmentLayout: + if operand == "a": + return scaled_matrix_layout(self.atom.a_dtype, self.atom.a_frag_len) + if operand == "b": + return scaled_matrix_layout(self.atom.b_dtype, self.atom.b_frag_len) + raise ValueError("matrix operand must be 'a' or 'b'") + + def scale_packing(self, operand: str) -> ScalePacking: + if operand == "a": + count = self.atom.a_scale_frag_len + elif operand == "b": + count = self.atom.b_scale_frag_len + else: + raise ValueError("scale operand must be 'a' or 'b'") + return ScalePacking(count, self.atom.scale_block_k) + @property def op_id(self) -> str: return self.atom.op_id diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/dtypes.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/dtypes.py index 583b591b1390..3808f87f3ad0 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/dtypes.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/dtypes.py @@ -9,6 +9,9 @@ from __future__ import annotations +from dataclasses import dataclass +from enum import Enum + # Canonical dtype spellings used as catalog keys. Instance/spec dtype strings # are normalised through this map so "f16"/"half" and "fp16" all resolve. _DTYPE_ALIASES = { @@ -48,3 +51,67 @@ def normalize_dtype(name: str) -> str: """Map a dtype spelling to its canonical catalog key.""" key = name.strip().lower() return _DTYPE_ALIASES.get(key, key) + + +class DTypeCategory(str, Enum): + FLOAT = "float" + INTEGER = "integer" + PREDICATE = "predicate" + + +@dataclass(frozen=True) +class DTypeInfo: + """Logical encoding, independent of storage slots and instruction support.""" + + name: str + category: DTypeCategory + encoded_bits: int + + +_DTYPE_INFO = { + name: DTypeInfo(name, category, bits) + for category, entries in ( + ( + DTypeCategory.FLOAT, + ( + ("fp32", 32), + ("fp16", 16), + ("bf16", 16), + ("fp8e4m3", 8), + ("bf8e5m2", 8), + ("fp6e2m3", 6), + ("fp6e3m2", 6), + ("fp4e2m1", 4), + ("e8m0", 8), + ("e4m3", 8), + ("e5m3", 8), + ), + ), + ( + DTypeCategory.INTEGER, + ( + ("i4", 4), + ("iu4", 4), + ("i8", 8), + ("iu8", 8), + ("i16", 16), + ("i32", 32), + ("i64", 64), + ), + ), + (DTypeCategory.PREDICATE, (("i1", 1),)), + ) + for name, bits in entries +} + + +def dtype_info(name: str) -> DTypeInfo: + """Resolve an encoding; recognition alone does not enable an operation. + + Scale names remain distinct keys. In particular e5m3 is not bf8e5m2. + Operand-role validation belongs to the selected atom. + """ + key = normalize_dtype(name) + if key not in _DTYPE_INFO: + raise ValueError(f"unknown dtype {name!r}") + return _DTYPE_INFO[key] diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py index cab47560124e..6dc0175c467d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py @@ -30,6 +30,7 @@ from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple from .arch import target as _arch +from .dtypes import dtype_info # ----------------------------- Types -------------------------------------- @@ -52,6 +53,41 @@ def __repr__(self) -> str: F32 = Type("f32") FP8E4M3 = Type("fp8e4m3") BF8E5M2 = Type("bf8e5m2") +FP4E2M1 = Type("fp4e2m1") +FP6E2M3 = Type("fp6e2m3") +FP6E3M2 = Type("fp6e3m2") +E8M0 = Type("e8m0") +E4M3 = Type("e4m3") +E5M3 = Type("e5m3") + + +def dtype_to_ir_type(dtype: str) -> Type: + """Resolve a logical type without claiming scalar operation support. + + Packed memory is described separately; a low-bit type is never an I8 alias. + """ + info = dtype_info(dtype) + types = { + "i1": I1, + "i8": I8, + "i16": I16, + "i32": I32, + "i64": I64, + "fp16": F16, + "bf16": BF16, + "fp32": F32, + "fp8e4m3": FP8E4M3, + "bf8e5m2": BF8E5M2, + "fp4e2m1": FP4E2M1, + "fp6e2m3": FP6E2M3, + "fp6e3m2": FP6E3M2, + "e8m0": E8M0, + "e4m3": E4M3, + "e5m3": E5M3, + } + if info.name not in types: + raise ValueError(f"no logical IR type for dtype {info.name!r}") + return types[info.name] # AMDGPU buffer-load AUX-byte cache-coherency hints. The AUX field of diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py index f0628a9fdee8..22e38bca4581 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py @@ -24,6 +24,7 @@ PtrType, Region, SmemType, + Type, Value, VectorType, ) @@ -2440,6 +2441,41 @@ def _find_enclosing_for(region: Region, target: Op) -> Optional[Op]: return None +def _extra_vector_declarations(kernel: KernelDef) -> list[str]: + """Declare encountered widths absent from the fixed compatibility prologue.""" + declarations: dict[str, str] = {} + + def visit_type(t: Type) -> None: + if isinstance(t, PtrType): + visit_type(t.pointee) + elif isinstance(t, SmemType): + visit_type(t.elem) + elif isinstance(t, VectorType): + name = _type_to_hip(t) + prefix = ( + "boolx" + if t.elem.name == "i1" + else _vec_prefix(t.elem.name, "vector type") + ) + scalar = "int8_t" if prefix == "i8x" else _HIP_TYPE[t.elem.name] + if f"_ROCKE_VEC({scalar}, {prefix}, {t.count})" not in HIP_PROLOGUE: + declarations[name] = ( + f"using {name} = {scalar} __attribute__((ext_vector_type({t.count})));" + ) + + def visit_region(region: Region) -> None: + for op in region.ops: + for value in (*op.operands, *op.results): + visit_type(value.type) + for child in op.regions: + visit_region(child) + + for param in kernel.params: + visit_type(param.type) + visit_region(kernel.body) + return list(declarations.values()) + + def lower_kernel_to_hip( kernel: KernelDef, *, @@ -2507,6 +2543,7 @@ def lower_kernel_to_hip( parts: List[str] = [] if include_prologue: parts.append(HIP_PROLOGUE) + parts.extend(_extra_vector_declarations(kernel)) parts.append(head) if smem_block: parts.append(smem_block) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/storage.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/storage.py new file mode 100644 index 000000000000..7c59be34fc0f --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/storage.py @@ -0,0 +1,229 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Bit storage shared by matrix elements and scales, independent of an ISA.""" + +from __future__ import annotations + +from dataclasses import dataclass +from math import gcd +from collections.abc import Sequence + +from .dtypes import DTypeInfo, dtype_info + +_MAX_BITS = (1 << 64) - 1 + + +def _checked(value: int) -> int: + if not isinstance(value, int) or not 0 <= value <= _MAX_BITS: + raise ValueError("storage size or offset exceeds uint64 range") + return value + + +@dataclass(frozen=True) +class BitPacking: + """Low-bit-first patterns in a little-endian bit stream. + + slot_bits=None means dense packing. Wider slots have zero unused high bits. + No operand role, K grouping, or numeric conversion is implied. + """ + + element_bits: int + slot_bits: int | None = None + + def __post_init__(self) -> None: + slot = self.element_bits if self.slot_bits is None else self.slot_bits + if not (1 <= self.element_bits <= slot <= 64): + raise ValueError("packing requires 1 <= element_bits <= slot_bits <= 64") + object.__setattr__(self, "slot_bits", slot) + + def group(self, carrier_bits: int) -> tuple[int, int]: + """Smallest whole-carrier group: (logical slots, carriers).""" + if carrier_bits not in (8, 16, 32, 64): + raise ValueError("carrier_bits must be 8, 16, 32, or 64") + common = gcd(self.slot_bits, carrier_bits) + return carrier_bits // common, self.slot_bits // common + + def bit_offset(self, index: int) -> int: + return _checked(_checked(index) * self.slot_bits) + + def byte_size(self, count: int, bit_offset: int = 0) -> int: + _checked(bit_offset) + bits = self.bit_offset(count) + if count == 0: + return 0 + return (_checked(bits + bit_offset) + 7) // 8 + + def pack(self, patterns: Sequence[int], *, bit_offset: int = 0) -> bytes: + """Pack encoded unsigned patterns, without interpreting their values.""" + out = bytearray(self.byte_size(len(patterns), bit_offset)) + for i, pattern in enumerate(patterns): + if not 0 <= pattern < (1 << self.element_bits): + raise ValueError("pattern does not fit element_bits") + pos = bit_offset + self.bit_offset(i) + # Byte-sized pieces avoid an implicit assumption about word boundaries. + remaining = self.element_bits + while remaining: + byte, shift = divmod(pos, 8) + take = min(remaining, 8 - shift) + out[byte] |= (pattern & ((1 << take) - 1)) << shift + pattern >>= take + remaining -= take + pos += take + return bytes(out) + + def unpack(self, data: bytes, count: int, *, bit_offset: int = 0) -> list[int]: + if len(data) < self.byte_size(count, bit_offset): + raise ValueError("packed buffer is too small") + result = [] + for i in range(count): + pos = bit_offset + self.bit_offset(i) + value = 0 + for bit in range(self.element_bits): + at = pos + bit + value |= ((data[at // 8] >> (at % 8)) & 1) << bit + result.append(value) + return result + + +@dataclass(frozen=True) +class FragmentPacking: + """A payload and its carrier capacity; all unused bits are zero.""" + + packing: BitPacking + count: int + carrier_bits: int + carrier_count: int + + def __post_init__(self) -> None: + self.packing.group(self.carrier_bits) + capacity = _checked(_checked(self.carrier_count) * self.carrier_bits) + if self.packing.bit_offset(self.count) > capacity: + raise ValueError("fragment payload exceeds carrier capacity") + + @property + def payload_bits(self) -> int: + return self.packing.bit_offset(self.count) + + @property + def live_carriers(self) -> int: + return (self.payload_bits + self.carrier_bits - 1) // self.carrier_bits + + @property + def padding_bits(self) -> int: + return self.carrier_count * self.carrier_bits - self.payload_bits + + def pack(self, patterns: Sequence[int]) -> tuple[int, ...]: + if len(patterns) != self.count: + raise ValueError("fragment logical count mismatch") + width = self.carrier_bits // 8 + data = self.packing.pack(patterns) + data += bytes(self.carrier_count * width - len(data)) + return tuple( + int.from_bytes(data[i : i + width], "little") + for i in range(0, len(data), width) + ) + + +@dataclass(frozen=True) +class MatrixFragmentLayout: + """Contiguous K chunks interleaved between lane groups. + + The atom selects this mapping. Storage and carrier widths do not determine it. + """ + + fragment: FragmentPacking + chunk_elements: int + lane_groups: int + lanes_per_group: int + + def __post_init__(self) -> None: + if ( + self.chunk_elements <= 0 + or self.lane_groups <= 0 + or self.lanes_per_group <= 0 + or self.fragment.count % self.chunk_elements + ): + raise ValueError("invalid matrix fragment chunk layout") + if self.fragment.packing.bit_offset(self.chunk_elements) % 8: + raise ValueError("matrix fragment chunks must occupy whole bytes") + + @property + def chunks_per_lane(self) -> int: + return self.fragment.count // self.chunk_elements + + @property + def chunk_bytes(self) -> int: + return self.fragment.packing.byte_size(self.chunk_elements) + + def coord(self, lane: int, slot: int) -> tuple[int, int]: + if not ( + 0 <= lane < self.lane_groups * self.lanes_per_group + and 0 <= slot < self.fragment.count + ): + raise ValueError("matrix fragment coordinate out of bounds") + chunk, element = divmod(slot, self.chunk_elements) + return ( + lane % self.lanes_per_group, + (chunk * self.lane_groups + lane // self.lanes_per_group) + * self.chunk_elements + + element, + ) + + +@dataclass(frozen=True) +class TensorStorage: + """Two-dimensional storage with a contiguous packed inner axis. + + Shape is logical. Row strides are bytes. Generic axis permutations and + negative strides are intentionally rejected by this initial interface. + """ + + dtype: str + shape: tuple[int, int] + row_stride_bytes: int | None = None + slot_bits: int | None = None + base_bit_offset: int = 0 + alignment_bytes: int = 1 + + def __post_init__(self) -> None: + info = dtype_info(self.dtype) + object.__setattr__(self, "dtype", info.name) + if len(self.shape) != 2: + raise ValueError("storage requires a two-dimensional logical shape") + for dim in self.shape: + _checked(dim) + _checked(self.base_bit_offset) + if self.alignment_bytes <= 0 or self.alignment_bytes & ( + self.alignment_bytes - 1 + ): + raise ValueError("alignment_bytes must be a positive power of two") + row_bytes = self.packing.byte_size(self.shape[1], self.base_bit_offset) + stride = row_bytes if self.row_stride_bytes is None else self.row_stride_bytes + if _checked(stride) < row_bytes: + raise ValueError("row stride is smaller than the packed row") + object.__setattr__(self, "row_stride_bytes", stride) + _checked(self.byte_size) + + @property + def info(self) -> DTypeInfo: + return dtype_info(self.dtype) + + @property + def packing(self) -> BitPacking: + return BitPacking(self.info.encoded_bits, self.slot_bits) + + @property + def byte_size(self) -> int: + rows, cols = self.shape + if rows == 0 or cols == 0: + return 0 + return _checked( + (rows - 1) * self.row_stride_bytes + + self.packing.byte_size(cols, self.base_bit_offset) + ) + + def address(self, row: int, col: int) -> tuple[int, int]: + if not (0 <= row < self.shape[0] and 0 <= col < self.shape[1]): + raise ValueError("storage coordinate out of bounds") + bits = _checked(self.base_bit_offset + self.packing.bit_offset(col)) + return _checked(row * self.row_stride_bytes + bits // 8), bits % 8 diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py new file mode 100644 index 000000000000..8d0bae29b5a2 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py @@ -0,0 +1,162 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Descriptor-driven packed transport, expanded before IR serialization.""" + +from __future__ import annotations + +from collections.abc import Callable +from math import gcd + +from ..core.dtypes import dtype_info +from ..core.ir import ( + I8, + I16, + I32, + I64, + IRBuilder, + Type, + Value, + VectorType, + dtype_to_ir_type, +) +from ..core.storage import FragmentPacking, MatrixFragmentLayout, TensorStorage + + +def storage_ir_type(dtype: str) -> Type: + """Addressable storage unit, distinct from the logical dtype resolver.""" + info = dtype_info(dtype) + if info.encoded_bits % 8 or info.name in ("e8m0", "e4m3", "e5m3"): + return I8 + return dtype_to_ir_type(info.name) + + +def load_matrix_fragment( + b: IRBuilder, + ptr: Value, + row_base: Value, + lane_group: Value, + k0: int, + *, + storage: TensorStorage, + layout: MatrixFragmentLayout, + carrier_type: Type = I32, +) -> Value: + """Load whole-byte chunks directly into the atom's carrier vector. + + row_base counts pointer storage units. lane_group is derived from the atom's + lane mapping by the caller. Partial chunks and nonzero bit origins reject. + """ + packing = layout.fragment + if not (0 < packing.count <= 0x7FFFFFFF and packing.carrier_count <= 0x7FFFFFFF): + raise ValueError("invalid matrix fragment chunk layout") + unit_type = storage_ir_type(storage.dtype) + unit_bytes = dtype_info(unit_type.name).encoded_bits // 8 + if ptr.type.pointee != unit_type: + raise ValueError("matrix pointer storage type mismatch") + if storage.packing != packing.packing or storage.base_bit_offset: + raise ValueError( + "matrix fragment requires matching packing and a zero bit origin" + ) + span = layout.chunk_elements * layout.lane_groups * layout.chunks_per_lane + if k0 < 0 or k0 + span > storage.shape[1]: + raise ValueError("matrix fragment exceeds the packed row") + origin_bits = storage.packing.bit_offset(k0) + if origin_bits % (8 * unit_bytes) or layout.chunk_bytes % unit_bytes: + raise ValueError("matrix fragment is not aligned to pointer storage units") + if dtype_info(carrier_type.name).encoded_bits != packing.carrier_bits: + raise ValueError("matrix carrier type width mismatch") + if packing.payload_bits % packing.carrier_bits: + raise ValueError("matrix fragment payload must occupy whole carriers") + chunk_units = layout.chunk_bytes // unit_bytes + origin_bytes = origin_bits // 8 + if ( + origin_bytes // unit_bytes > 0x7FFFFFFF + or chunk_units * layout.lane_groups > 0x7FFFFFFF + ): + raise ValueError("matrix fragment offset exceeds i32 range") + alignment = gcd( + storage.alignment_bytes, + storage.row_stride_bytes, + layout.chunk_bytes, + origin_bytes, + ) + lane_chunk = b.mul(lane_group, b.const_i32(chunk_units)) + step_base = b.add(row_base, b.const_i32(origin_bytes // unit_bytes)) + chunks = [] + for j in range(layout.chunks_per_lane): + offset = b.add( + b.add(step_base, b.const_i32(j * layout.lane_groups * chunk_units)), + lane_chunk, + ) + remaining = chunk_units + consumed = 0 + max_width = 8 if unit_bytes == 4 else 16 + while remaining: + width = min(max_width, 1 << (remaining.bit_length() - 1)) + at = b.add(offset, b.const_i32(consumed)) if consumed else offset + load_align = gcd(alignment, consumed * unit_bytes) + if width == 1: + value = b.vector_splat( + b.global_load(ptr, at, unit_type, align=load_align), 1 + ) + else: + value = b.global_load_vN(ptr, at, unit_type, width, align=load_align) + chunks.append(value) + consumed += width + remaining -= width + payload = chunks[0] + for chunk in chunks[1:]: + payload = b.vec_concat(payload, chunk) + payload_type = VectorType(carrier_type, packing.live_carriers) + if payload.type != payload_type: + payload = b.bitcast(payload, payload_type) + padding = packing.carrier_count - packing.live_carriers + if padding: + if carrier_type != I32: + raise ValueError("padded matrix fragments currently require i32 carriers") + payload = b.vec_concat(payload, b.vector_splat(b.const_i32(0), padding)) + return payload + + +def pack_fragment_bits( + b: IRBuilder, + load_bits: Callable[[int], Value], + fragment: FragmentPacking, +) -> list[Value]: + """Pack unsigned encoded patterns into integer carriers, including split fields. + + The loader must return canonical patterns with zero high bits. Tensor decoding + and numeric quantization are separate operations. + """ + if fragment.carrier_bits not in (32, 64): + raise ValueError("IR pattern packing currently requires i32 or i64 carriers") + word_type = I64 if fragment.carrier_bits == 64 else I32 + constant = b.const_i64 if fragment.carrier_bits == 64 else b.const_i32 + words = [constant(0) for _ in range(fragment.carrier_count)] + for j in range(fragment.count): + pattern = load_bits(j) + if pattern.type not in (I8, I16, I32, I64): + raise ValueError( + "pattern packing requires unsigned patterns in integer carriers" + ) + pattern_bits = dtype_info(pattern.type.name).encoded_bits + if pattern_bits < fragment.packing.element_bits: + raise ValueError("pattern carrier is smaller than encoded width") + if pattern_bits < fragment.carrier_bits: + pattern = b.zext(pattern, word_type) + elif pattern_bits > fragment.carrier_bits: + raise ValueError("pattern carrier is wider than output carrier") + start = fragment.packing.bit_offset(j) + remaining = fragment.packing.element_bits + consumed = 0 + while remaining: + word, shift = divmod(start, fragment.carrier_bits) + take = min(remaining, fragment.carrier_bits - shift) + part = b.lshr(pattern, constant(consumed)) if consumed else pattern + if take < remaining: + part = b.land(part, constant((1 << take) - 1)) + words[word] = b.lor(words[word], b.shl(part, constant(shift))) + start += take + consumed += take + remaining -= take + return words diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py index f8ead583eb19..1ae76ffb0d33 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py @@ -39,7 +39,18 @@ from typing import List, Literal -from ..core.ir import BF8E5M2, F32, FP8E4M3, I8, I32, I64, IRBuilder, Type, Value +from ..core.ir import ( + BF8E5M2, + F32, + FP8E4M3, + I8, + I32, + I64, + IRBuilder, + Type, + Value, + dtype_to_ir_type, +) __all__ = [ @@ -55,7 +66,7 @@ ] -QDType = Literal["i8", "fp8e4m3", "bf8e5m2"] +QDType = Literal["i8", "fp8e4m3", "bf8e5m2", "fp4e2m1", "fp6e2m3", "fp6e3m2"] # Per-dtype clamp magnitude (the largest representable absolute value). @@ -89,6 +100,9 @@ "i8": "i8", "fp8e4m3": "fp8e4m3", "bf8e5m2": "bf8e5m2", + "fp4e2m1": "fp4e2m1", + "fp6e2m3": "fp6e2m3", + "fp6e3m2": "fp6e3m2", } @@ -104,19 +118,14 @@ def _canon(qdtype: str) -> QDType: def quant_ir_type(qdtype: str) -> Type: """Map a quant-dtype alias string to the canonical IR :class:`Type`. - Accepts ``"i8"`` / ``"int8"`` / ``"fp8e4m3"`` / ``"fp8"`` / - ``"bf8e5m2"`` / ``"bf8"``. Raises :class:`ValueError` for anything - else; this is the single point of truth for the alias map so - everywhere in the codebase agrees on the canonical names. + Accepts the existing i8/FP8/BF8 aliases and the FP4/FP6 aliases from + the common dtype resolver. Type recognition does not enable scalar + quantization or dequantization for the low-bit formats. """ - canon = _canon(qdtype) - if canon == "i8": - return I8 - if canon == "fp8e4m3": - return FP8E4M3 - if canon == "bf8e5m2": - return BF8E5M2 - raise ValueError(f"unreachable: canon={canon!r}") + # Type recognition includes low-bit formats before scalar conversions exist. + if qdtype in ("fp4", "fp4e2m1", "fp6", "fp6e2m3", "bf6", "fp6e3m2"): + return dtype_to_ir_type(qdtype) + return dtype_to_ir_type(_canon(qdtype)) def quant_max_abs(qdtype: str) -> float: diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt b/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt index 8a27e62f01ef..4704a66de77a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt @@ -14,6 +14,15 @@ if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/core/smoke.cpp") endif() if(BUILD_TESTING) + add_executable(rocke_storage core/test_storage.cpp) + target_link_libraries(rocke_storage PRIVATE rocke_core) + add_test(NAME rocke_storage COMMAND rocke_storage) + set_tests_properties(rocke_storage PROPERTIES LABELS "ckc;unit;host") + rocke_cxx_test_installed_name(_rocke_installed_name rocke_storage) + set_target_properties(rocke_storage PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") + install(TARGETS rocke_storage RUNTIME DESTINATION ${ROCKE_CXX_TEST_INSTALL_DIR}) + set_property(GLOBAL APPEND PROPERTY ROCKE_INSTALL_TEST_TARGETS rocke_storage) + add_executable(rocke_dtypes core/test_dtypes.cpp) target_link_libraries(rocke_dtypes PRIVATE rocke_core) add_test(NAME rocke_dtypes COMMAND rocke_dtypes) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py index 47014e02e8a2..cee48c279c6e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py @@ -5,6 +5,55 @@ import pytest from rocke.core.dtypes import normalize_dtype +from rocke.core.dtypes import dtype_info +from rocke.core.ir import I8, IRBuilder, dtype_to_ir_type +from rocke.core.ir_serialize import parse, serialize +from rocke.helpers.quant import quant_ir_type, ir_to_qdtype, dequantize_scalar_to_f32 + + +@pytest.mark.parametrize( + "dtype,bits", + [ + ("fp4", 4), + ("fp6", 6), + ("bf6", 6), + ("fp8", 8), + ("bf8", 8), + ("f16", 16), + ("bf16", 16), + ("f32", 32), + ("e8m0", 8), + ("e4m3", 8), + ("e5m3", 8), + ], +) +def test_logical_dtype_and_serialization(dtype, bits): + info = dtype_info(dtype) + logical = dtype_to_ir_type(dtype) + assert info.encoded_bits == bits + assert logical != I8 + b = IRBuilder("dtype_transport") + b.param("code", logical) + assert serialize(parse(serialize(b.kernel))) == serialize(b.kernel) + + +@pytest.mark.parametrize("dtype", ["fp4", "fp6", "bf6"]) +def test_quant_type_recognition_does_not_enable_integer_conversion(dtype): + logical = quant_ir_type(dtype) + assert logical == dtype_to_ir_type(dtype) + assert ir_to_qdtype(logical) == dtype_info(dtype).name + b = IRBuilder("no_lowbit_conversion") + value = b.param("value", logical) + scale = b.const_f32(1) + with pytest.raises(ValueError, match="unsupported input type"): + dequantize_scalar_to_f32(b, value, scale=scale) + + +def test_unknown_dtype_and_unrepresented_integer_family(): + with pytest.raises(ValueError, match="unknown dtype"): + dtype_info("custom_format") + with pytest.raises(ValueError, match="no logical IR type"): + dtype_to_ir_type("iu4") @pytest.mark.parametrize( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py new file mode 100644 index 000000000000..79af622a235c --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py @@ -0,0 +1,147 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Mirrored authoring and bit-pattern placement, independent of MMA support.""" + +import os +import subprocess +from pathlib import Path + +import pytest + +from rocke.core.arch.wmma_scale import scaled_matrix_layout +from rocke.core.ir import I8, I32, I64, IRBuilder, PtrType +from rocke.core.ir_serialize import serialize +from rocke.core.lower_llvm import lower_kernel_to_llvm +from rocke.core.lower_hip import lower_kernel_to_hip +from rocke.core.storage import ( + BitPacking, + FragmentPacking, + MatrixFragmentLayout, + TensorStorage, +) +from rocke.helpers.mma_io import ( + load_matrix_fragment, + pack_fragment_bits, + storage_ir_type, +) + + +CASES = [ + "fp4", + "fp6", + "bf6", + "fp8", + "bf8", + "f16", + "bf16", + "pack_fp6_cross_word", + "pack_scale_bytes_i32", + "pack_scale_bytes_i64", +] + + +def build_transport(dtype): + b = IRBuilder("transport") + patterns = dtype.startswith("pack_") + unit = I8 if patterns else storage_ir_type(dtype) + typed = dtype in ("f16", "bf16") + carrier = unit if typed else I64 if dtype == "pack_scale_bytes_i64" else I32 + a = b.param("A", PtrType(unit, "global")) + o = b.param("O", PtrType(carrier, "global")) + if patterns: + bits = 6 if dtype == "pack_fp6_cross_word" else 8 + count = 16 if bits == 6 else 8 if dtype == "pack_scale_bytes_i64" else 4 + words = 3 if bits == 6 else 1 + fragment = FragmentPacking( + BitPacking(bits), count, 64 if carrier == I64 else 32, words + ) + values = pack_fragment_bits( + b, lambda j: b.global_load(a, b.const_i32(j), I8, align=1), fragment + ) + for j, value in enumerate(values): + b.global_store(o, b.const_i32(j), value, align=4) + else: + storage = TensorStorage(dtype, (16, 128), alignment_bytes=16) + layout = ( + MatrixFragmentLayout(FragmentPacking(BitPacking(16), 32, 16, 32), 16, 2, 16) + if typed + else scaled_matrix_layout(dtype, 16) + ) + base = b.const_i32(0) + lane = b.mod(b.thread_id_x(), b.const_i32(32)) + group = b.div(lane, b.const_i32(16)) + value = load_matrix_fragment( + b, a, base, group, 0, storage=storage, layout=layout, carrier_type=carrier + ) + for j in range(value.type.count): + index = b.const_i32(j) + element = b.vec_extract(value, j) + b.global_store(o, index, element, align=2 if typed else 4) + return b.kernel + + +@pytest.mark.parametrize("dtype", CASES) +def test_descriptor_authoring_lowers_in_both_backends(dtype): + kernel = build_transport(dtype) + assert "define" in lower_kernel_to_llvm( + kernel, arch="gfx1250", llvm_flavor="llvm23" + ) + assert "transport" in lower_kernel_to_hip(kernel, arch="gfx1250") + + +@pytest.mark.parametrize("dtype", CASES) +@pytest.mark.parametrize("route", ["ir", "hip"]) +def test_native_helper_emits_identical_ir(dtype, route): + executable = os.environ.get("ROCKE_STORAGE_TEST") + if not executable: + pytest.skip("set ROCKE_STORAGE_TEST to the built native storage test") + assert Path(executable).is_file() + actual = subprocess.run( + [executable, "--hip" if route == "hip" else "--emit", dtype], + check=True, + capture_output=True, + text=True, + ) + kernel = build_transport(dtype) + expected = ( + serialize(kernel) + if route == "ir" + else lower_kernel_to_hip(kernel, arch="gfx1250") + ) + assert actual.stdout == expected + + +def test_reject_partial_fragment_and_wrong_pointer(): + b = IRBuilder("invalid_transport") + ptr = b.param("A", PtrType(I8, "global")) + zero = b.const_i32(0) + with pytest.raises(ValueError, match="exceeds"): + load_matrix_fragment( + b, + ptr, + zero, + zero, + 1, + storage=TensorStorage("fp4", (16, 128)), + layout=scaled_matrix_layout("fp4", 16), + ) + with pytest.raises(ValueError, match="pointer storage type"): + load_matrix_fragment( + b, + ptr, + zero, + zero, + 0, + storage=TensorStorage("fp8", (16, 128)), + layout=scaled_matrix_layout("fp8", 16), + ) + + +def test_hip_declares_only_encountered_missing_vector_widths(): + source = lower_kernel_to_hip(build_transport("fp6"), arch="gfx1250") + for name in ("i8x24", "i8x40", "i32x12"): + assert source.count(f"using {name} =") == 1 + assert "using f16x32 =" not in source + source = lower_kernel_to_hip(build_transport("f16"), arch="gfx1250") + assert source.count("using f16x32 =") == 1 + assert "using i8x24 =" not in source diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp new file mode 100644 index 000000000000..ef2b578de597 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp @@ -0,0 +1,186 @@ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +/* Host packing fixtures and native authoring emission for test_mma_io.py. */ +#include "rocke/storage.h" + +#include +#include +#include +#include + +#include "rocke/helper_rocke.helpers.mma_io.h" +#include "rocke/helper_rocke.helpers.quant.h" +#include "rocke/ir_serialize.h" +#include "rocke/lower_hip.h" +#include "rocke/wmma_scale_internal.h" + +#define CHECK(x) \ + do \ + { \ + if(!(x)) \ + { \ + fprintf(stderr, "check failed at %d: %s\n", __LINE__, #x); \ + return 1; \ + } \ + } while(0) + +static rocke_value_t* load_bits(rocke_ir_builder_t* b, int j, void* ctx) +{ + return rocke_b_global_load(b, (rocke_value_t*)ctx, rocke_b_const_i32(b, j), rocke_i8(), 1); +} + +static int emit(const char* dtype, bool hip) +{ + rocke_ir_builder_t b; + CHECK(rocke_ir_builder_init(&b, "transport") == ROCKE_OK); + bool patterns = strncmp(dtype, "pack_", 5) == 0; + const auto* unit = patterns ? rocke_i8() : rocke_storage_ir_type(dtype); + CHECK(unit); + const bool typed = strcmp(dtype, "f16") == 0 || strcmp(dtype, "bf16") == 0; + const auto* carrier = typed ? unit : rocke_i32(); + if(strcmp(dtype, "pack_scale_bytes_i64") == 0) + carrier = rocke_i64(); + auto* a = rocke_b_param(&b, "A", rocke_ptr_type(&b, unit, "global"), NULL); + auto* o = rocke_b_param(&b, "O", rocke_ptr_type(&b, carrier, "global"), NULL); + if(patterns) + { + int bits = strcmp(dtype, "pack_fp6_cross_word") == 0 ? 6 : 8; + int count = bits == 6 ? 16 : (strcmp(dtype, "pack_scale_bytes_i64") == 0 ? 8 : 4); + rocke_bit_packing_t packing; + rocke_fragment_packing_t fragment; + CHECK(rocke_bit_packing_init(&packing, bits, 0)); + int words = bits == 6 ? 3 : 1; + CHECK(rocke_fragment_packing_init( + &fragment, &packing, count, carrier == rocke_i64() ? 64 : 32, words)); + rocke_value_t* values[3]; + CHECK(rocke_h_pack_fragment_bits(&b, load_bits, a, &fragment, values, words) == ROCKE_OK); + for(int j = 0; j < words; ++j) + rocke_b_global_store(&b, o, rocke_b_const_i32(&b, j), values[j], 4); + } + else + { + rocke_tensor_storage_t storage; + CHECK(rocke_tensor_storage_init(&storage, dtype, 16, 128, UINT64_MAX, 0, 0, 16)); + rocke_matrix_fragment_layout_t layout; + if(typed) + { + rocke_bit_packing_t packing; + rocke_fragment_packing_t fragment; + CHECK(rocke_bit_packing_init(&packing, 16, 0)); + CHECK(rocke_fragment_packing_init(&fragment, &packing, 32, 16, 32)); + CHECK(rocke_matrix_fragment_layout_init(&layout, &fragment, 16, 2, 16)); + } + else + layout = rocke_scaled_matrix_layout(dtype, 16); + auto* base = rocke_b_const_i32(&b, 0); + auto* thread = rocke_b_thread_id_x(&b); + auto* lane = rocke_b_mod(&b, thread, rocke_b_const_i32(&b, 32)); + auto* group = rocke_b_div(&b, lane, rocke_b_const_i32(&b, 16)); + auto* value + = rocke_h_load_matrix_fragment(&b, a, base, group, 0, &storage, &layout, carrier); + CHECK(value && rocke_ir_builder_ok(&b)); + for(int j = 0; j < value->type->count; ++j) + { + auto* index = rocke_b_const_i32(&b, j); + auto* element = rocke_b_vec_extract(&b, value, j); + rocke_b_global_store(&b, o, index, element, typed ? 2 : 4); + } + } + CHECK(rocke_ir_builder_ok(&b)); + if(hip) + { + rocke_strbuf_t text; + CHECK(rocke_strbuf_init(&text, 256) == 0); + rocke_lower_hip_opts_t opts = {}; + opts.arch = "gfx1250"; + CHECK(rocke_lower_kernel_to_hip(&b, b.kernel, &opts, &text) == ROCKE_OK); + fputs(rocke_strbuf_cstr(&text), stdout); + rocke_strbuf_free(&text); + } + else + { + char* text = NULL; + CHECK(rocke_ir_serialize(b.kernel, &text) == ROCKE_OK); + fputs(text, stdout); + free(text); + } + rocke_ir_builder_free(&b); + return 0; +} + +int main(int argc, char** argv) +{ + if(argc == 3 && strcmp(argv[1], "--emit") == 0) + return emit(argv[2], false); + if(argc == 3 && strcmp(argv[1], "--hip") == 0) + return emit(argv[2], true); + for(int bits : {4, 6, 8, 16, 32, 64}) + { + rocke_bit_packing_t packing; + CHECK(rocke_bit_packing_init(&packing, bits, 0)); + uint64_t patterns[256], decoded[256]; + size_t count = bits <= 8 ? size_t(1) << bits : 4; + for(size_t i = 0; i < count; ++i) + patterns[i] = bits <= 8 ? i : (i == 3 ? UINT64_MAX >> (64 - bits) : i); + for(uint64_t offset : {uint64_t(0), uint64_t(1), uint64_t(7), uint64_t(9)}) + { + uint64_t size; + CHECK(rocke_bit_packing_bytes(&packing, count, offset, &size)); + uint8_t data[2049]; + memset(data, 0xA5, sizeof(data)); + CHECK(rocke_bit_pack(&packing, patterns, count, offset, data, size)); + CHECK(data[size] == 0xA5); // No tail overrun. + CHECK(!rocke_bit_unpack(&packing, data, size - 1, count, offset, decoded)); + CHECK(rocke_bit_unpack(&packing, data, size, count, offset, decoded)); + CHECK(memcmp(patterns, decoded, count * sizeof(uint64_t)) == 0); + } + } + rocke_bit_packing_t six; + CHECK(rocke_bit_packing_init(&six, 6, 0)); + uint64_t values, carriers; + CHECK(rocke_bit_packing_group(&six, 32, &values, &carriers)); + CHECK(values == 16 && carriers == 3); + uint64_t patterns[16] = {}, words[3]; + patterns[5] = patterns[10] = 63; + rocke_fragment_packing_t fragment; + CHECK(rocke_fragment_packing_init(&fragment, &six, 16, 32, 3)); + CHECK(rocke_fragment_pack(&fragment, patterns, 16, words, 3)); + CHECK(words[0] == 0xC0000000 && words[1] == 0xF000000F && words[2] == 3); + CHECK(!rocke_bit_packing_bytes(&six, UINT64_MAX, 0, &values)); + CHECK(!rocke_fragment_packing_init(&fragment, &six, 16, 32, 2)); + rocke_tensor_storage_t storage; + CHECK(rocke_tensor_storage_init(&storage, "bf6", 3, 5, 8, 0, 3, 1)); + CHECK(rocke_tensor_storage_bytes(&storage, &values) && values == 21); + int shift; + CHECK(rocke_tensor_storage_address(&storage, 2, 4, &values, &shift)); + CHECK(values == 19 && shift == 3); + CHECK(!rocke_tensor_storage_address(&storage, 3, 0, &values, &shift)); + CHECK(!rocke_tensor_storage_init(&storage, "fp6", 3, 5, 3, 0, 0, 1)); + CHECK(!rocke_tensor_storage_init(&storage, "fp8", UINT64_MAX, 3, 3, 0, 0, 1)); + CHECK(rocke_tensor_storage_init(&storage, "fp4", 3, 0, 0, 0, 7, 1)); + CHECK(rocke_tensor_storage_bytes(&storage, &values) && values == 0); + for(const char* dtype : {"fp4", "fp6", "bf6", "e8m0", "e4m3", "e5m3"}) + { + const auto* logical = rocke_dtype_to_ir_type(dtype); + CHECK(logical && logical != rocke_i8()); + CHECK(rocke_scalar_by_name(logical->name) == logical); + if(dtype[0] != 'e') + CHECK(rocke_quant_ir_type(dtype) == logical); + rocke_ir_builder_t b, parsed; + CHECK(rocke_ir_builder_init(&b, "types") == ROCKE_OK); + CHECK(rocke_ir_builder_init(&parsed, "parsed") == ROCKE_OK); + rocke_b_param(&b, "pattern", logical, NULL); + char* text = NULL; + CHECK(rocke_ir_serialize(b.kernel, &text) == ROCKE_OK); + rocke_kernel_def_t* kernel = NULL; + CHECK(rocke_ir_parse(text, &parsed, &kernel) == ROCKE_OK); + char* again = NULL; + CHECK(rocke_ir_serialize(kernel, &again) == ROCKE_OK); + CHECK(strcmp(text, again) == 0); + free(text); + free(again); + rocke_ir_builder_free(&b); + rocke_ir_builder_free(&parsed); + } + return 0; +} diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py new file mode 100644 index 000000000000..515430599388 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py @@ -0,0 +1,94 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Independent bit-stream fixtures and storage bounds for matrices and scales.""" + +import pytest + +from rocke.core.arch.wmma_scale import ScalePacking, scaled_matrix_layout +from rocke.core.storage import BitPacking, FragmentPacking, TensorStorage + + +@pytest.mark.parametrize("bits", [4, 6, 8, 16, 32, 64]) +@pytest.mark.parametrize("offset", [0, 1, 7, 9]) +def test_dense_patterns_against_integer_oracle(bits, offset): + patterns = list(range(1 << bits)) if bits <= 8 else [0, 1, (1 << bits) - 1, 0x1234] + packing = BitPacking(bits) + encoded = packing.pack(patterns, bit_offset=offset) + expected = sum(v << (offset + i * bits) for i, v in enumerate(patterns)) + assert int.from_bytes(encoded, "little") == expected + assert packing.unpack(encoded, len(patterns), bit_offset=offset) == patterns + with pytest.raises(ValueError, match="too small"): + packing.unpack(encoded[:-1], len(patterns), bit_offset=offset) + + +def test_fp6_crosses_words_without_per_word_padding(): + packing = BitPacking(6) + assert packing.group(8) == (4, 3) + assert packing.group(32) == (16, 3) + patterns = [0] * 16 + patterns[5] = patterns[10] = 63 + # slot 5 occupies bits 30..35; slot 10 occupies bits 60..65. + expected = (0xC0000000, 0xF000000F, 0x00000003) + assert FragmentPacking(packing, 16, 32, 3).pack(patterns) == expected + assert BitPacking(6, 8).pack(patterns) == bytes(patterns) + assert packing.pack([1, 2, 3, 4]) == bytes([0x81, 0x30, 0x10]) + + +@pytest.mark.parametrize( + "dtype,live", [("fp8", 16), ("fp6", 12), ("bf6", 12), ("fp4", 8)] +) +def test_matrix_payload_and_padding(dtype, live): + layout = scaled_matrix_layout(dtype, 16) + assert layout.fragment.count == 64 + assert layout.fragment.live_carriers == live + assert layout.fragment.padding_bits == (16 - live) * 32 + patterns = [(1 << layout.fragment.packing.element_bits) - 1] * 64 + assert layout.fragment.pack(patterns) == (0xFFFFFFFF,) * live + (0,) * (16 - live) + for lane in range(32): + coords = [layout.coord(lane, slot) for slot in range(64)] + assert all(row == lane % 16 and 0 <= k < 128 for row, k in coords) + other = {layout.coord(lane ^ 16, slot)[1] for slot in range(64)} + assert {k for _, k in coords}.isdisjoint(other) + assert {k for _, k in coords} | other == set(range(128)) + + +@pytest.mark.parametrize( + "count,block_k,word", [(4, 32, 0x04030201), (8, 16, 0x0807060504030201)] +) +def test_scales_use_common_bit_packing(count, block_k, word): + scales = ScalePacking(count, block_k) + assert scales.association.block_k == block_k + assert scales.fragment.pack(list(range(1, count + 1))) == (word,) + assert scales.packing == BitPacking(8) + assert scales.word_bits == count * 8 + + +def test_storage_rows_offsets_and_empty_views(): + storage = TensorStorage("bf6", (3, 5), row_stride_bytes=8, base_bit_offset=3) + assert storage.dtype == "fp6e3m2" + assert storage.byte_size == 21 + assert storage.address(2, 4) == (19, 3) + assert TensorStorage("fp4", (0, 5)).byte_size == 0 + assert TensorStorage("fp4", (3, 0), base_bit_offset=7).byte_size == 0 + assert BitPacking(6).pack([], bit_offset=7) == b"" + assert TensorStorage("e8m0", (2, 8)).byte_size == 16 + with pytest.raises(ValueError, match="out of bounds"): + storage.address(3, 0) + with pytest.raises(ValueError, match="row stride"): + TensorStorage("fp6", (3, 5), row_stride_bytes=3) + with pytest.raises(ValueError, match="power of two"): + TensorStorage("fp4", (1, 4), alignment_bytes=3) + + +def test_invalid_packing_and_overflow(): + for bits, slots in [(0, None), (65, None), (6, 4)]: + with pytest.raises(ValueError): + BitPacking(bits, slots) + with pytest.raises(ValueError, match="capacity"): + FragmentPacking(BitPacking(6), 16, 32, 2) + with pytest.raises(ValueError, match="uint64"): + BitPacking(6).byte_size(1 << 63) + with pytest.raises(ValueError, match="uint64"): + TensorStorage("fp8", (1 << 63, 3)) + with pytest.raises(ValueError, match="fit"): + BitPacking(6).pack([64]) From 239e0c66f45d33a313cefe9f97ef638932590148 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Tue, 22 Sep 2026 21:22:35 -0400 Subject: [PATCH 02/32] fix(rocke): preserve HIP vector load alignment --- .../cpp/core/lower_hip/lower_hip_mem.cpp | 19 ++++++++++ .../platform/python/rocke/core/lower_hip.py | 13 +++++++ .../rocke/platform/tests/core/test_mma_io.py | 37 ++++++++++++++++++- .../platform/tests/core/test_storage.cpp | 6 ++- 4 files changed, 73 insertions(+), 2 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mem.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mem.cpp index 0b0584e37904..198da0fc0c2d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mem.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mem.cpp @@ -15,6 +15,7 @@ * rocke_h_name, rocke_h_type_to_hip, rocke_h_hip_scalar, rocke_h_vec_prefix, * rocke_h_smem_set_storage/_storage, rocke_h_fail, rocke_h_live) are NOT defined here. */ +#include "rocke/dtypes.h" #include "rocke/lower_hip_internal.h" #include @@ -530,6 +531,24 @@ static rocke_status_t _op_memref_global_load_vN(rocke_h_lowerer_t* lw, const roc elem_name = mem_attr_str(op, "elem_type", "f16"); prefix = rocke_h_vec_prefix_checked(lw, elem_name, /*full_map=*/true, "global_load_vN"); res = rocke_h_name(lw, op->results[0]); + const int64_t byte_count = vec * (rocke_dtype_info(elem_name)->encoded_bits / 8); + const int64_t align = mem_attr_int(op, "align", vec * 2); + if(align < byte_count) + { + /* Do not strengthen the IR's alignment through a vector-pointer cast. */ + rocke_h_emitf( + lw, + "%s%lld %s; __builtin_memcpy(&%s, __builtin_assume_aligned(%s + %s, %lld), %lld);", + prefix, + (long long)vec, + res, + res, + rocke_h_name(lw, ptr), + rocke_h_name(lw, idx), + (long long)align, + (long long)byte_count); + return lw->status; + } rocke_h_emitf(lw, "%s%lld %s = *reinterpret_cast(%s + %s);", prefix, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py index 22e38bca4581..cf601237d07f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py @@ -18,6 +18,7 @@ from typing import List, Optional from .arch.wmma_scale import gfx1250_scaled_wmma +from .dtypes import dtype_info from .ir import ( KernelDef, Op, @@ -486,6 +487,18 @@ def _op_memref_global_load_vN(self, op: Op) -> None: vec = int(op.attrs["vec"]) elem_name = op.attrs.get("elem_type", "f16") prefix = _vec_prefix(elem_name, "global_load_vN") + byte_count = vec * (dtype_info(elem_name).encoded_bits // 8) + align = int(op.attrs.get("align", vec * 2)) + if align < byte_count: + # A vector-pointer dereference would promise natural alignment. + # Copy the payload using only the alignment guaranteed by the IR. + self._emit( + f"{prefix}{vec} {_name(op.result)}; " + f"__builtin_memcpy(&{_name(op.result)}, " + f"__builtin_assume_aligned({_name(ptr)} + {_name(idx)}, {align}), " + f"{byte_count});" + ) + return self._emit( f"{prefix}{vec} {_name(op.result)} = " f"*reinterpret_cast({_name(ptr)} + {_name(idx)});" diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py index 79af622a235c..b711ecf45d26 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py @@ -3,6 +3,8 @@ """Mirrored authoring and bit-pattern placement, independent of MMA support.""" import os +import re +import shutil import subprocess from pathlib import Path @@ -29,6 +31,7 @@ CASES = [ "fp4", "fp6", + "fp6_padded", "bf6", "fp8", "bf8", @@ -42,6 +45,8 @@ def build_transport(dtype): b = IRBuilder("transport") + padded = dtype == "fp6_padded" + dtype = "fp6" if padded else dtype patterns = dtype.startswith("pack_") unit = I8 if patterns else storage_ir_type(dtype) typed = dtype in ("f16", "bf16") @@ -61,7 +66,12 @@ def build_transport(dtype): for j, value in enumerate(values): b.global_store(o, b.const_i32(j), value, align=4) else: - storage = TensorStorage(dtype, (16, 128), alignment_bytes=16) + storage = TensorStorage( + dtype, + (16, 128), + row_stride_bytes=97 if padded else None, + alignment_bytes=16, + ) layout = ( MatrixFragmentLayout(FragmentPacking(BitPacking(16), 32, 16, 32), 16, 2, 16) if typed @@ -145,3 +155,28 @@ def test_hip_declares_only_encountered_missing_vector_widths(): source = lower_kernel_to_hip(build_transport("f16"), arch="gfx1250") assert source.count("using f16x32 =") == 1 assert "using i8x24 =" not in source + + +@pytest.mark.parametrize( + "dtype,copy_bytes,alignment", + [("fp6", 16, 8), ("fp6_padded", 16, 1), ("f16", 32, 16)], +) +def test_hip_compiler_preserves_fragment_load_alignment(dtype, copy_bytes, alignment): + """Inspect compiler IR: source parity cannot detect a shared alignment bug.""" + if not shutil.which("hipcc"): + pytest.skip("hipcc not in PATH") + from rocke.helpers.compile import emit_device_llvm_ir_via_hipcc + + llvm = emit_device_llvm_ir_via_hipcc( + build_transport(dtype), arch="gfx950", extra_flags=["-O0"] + ) + source_alignments = [ + int(align) + for align, size in re.findall( + r"@llvm\.memcpy[^\n]*?\([^,]+,\s*ptr[^,]*\balign (\d+) [^,]+,\s*i64 (\d+)", + llvm, + ) + if int(size) == copy_bytes + ] + assert source_alignments, "expected unaligned-safe fragment payload copies" + assert all(value <= alignment for value in source_alignments), source_alignments diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp index ef2b578de597..47d4749920fb 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp @@ -33,6 +33,9 @@ static int emit(const char* dtype, bool hip) { rocke_ir_builder_t b; CHECK(rocke_ir_builder_init(&b, "transport") == ROCKE_OK); + const bool padded = strcmp(dtype, "fp6_padded") == 0; + if(padded) + dtype = "fp6"; bool patterns = strncmp(dtype, "pack_", 5) == 0; const auto* unit = patterns ? rocke_i8() : rocke_storage_ir_type(dtype); CHECK(unit); @@ -60,7 +63,8 @@ static int emit(const char* dtype, bool hip) else { rocke_tensor_storage_t storage; - CHECK(rocke_tensor_storage_init(&storage, dtype, 16, 128, UINT64_MAX, 0, 0, 16)); + CHECK(rocke_tensor_storage_init( + &storage, dtype, 16, 128, padded ? 97 : UINT64_MAX, 0, 0, 16)); rocke_matrix_fragment_layout_t layout; if(typed) { From 70ec174ef8559f90fca7092416a10071fc708810 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:37:50 -0400 Subject: [PATCH 03/32] fix(rocke): validate load alignment and fragment coordinates --- .../cpp/core/lower_hip/lower_hip_mem.cpp | 3 ++ .../rocke/platform/cpp/core/storage.cpp | 8 +++- .../platform/python/rocke/core/lower_hip.py | 4 ++ .../rocke/platform/tests/core/test_mma_io.py | 11 +++++ .../platform/tests/core/test_storage.cpp | 48 +++++++++++++++++++ 5 files changed, 72 insertions(+), 2 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mem.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mem.cpp index 198da0fc0c2d..cdfee832aad9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mem.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mem.cpp @@ -533,6 +533,9 @@ static rocke_status_t _op_memref_global_load_vN(rocke_h_lowerer_t* lw, const roc res = rocke_h_name(lw, op->results[0]); const int64_t byte_count = vec * (rocke_dtype_info(elem_name)->encoded_bits / 8); const int64_t align = mem_attr_int(op, "align", vec * 2); + if(align <= 0 || (align & (align - 1))) + return rocke_h_fail( + lw, ROCKE_ERR_VALUE, "global_load_vN: alignment must be a positive power of two"); if(align < byte_count) { /* Do not strengthen the IR's alignment through a vector-pointer cast. */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp index b1a2dda9cb6c..5ad91cf3287a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp @@ -253,8 +253,12 @@ bool rocke_matrix_fragment_layout_init(rocke_matrix_fragment_layout_t* out, bool rocke_matrix_fragment_coord( const rocke_matrix_fragment_layout_t* p, int lane, uint64_t slot, uint64_t* row, uint64_t* k) { - if(!p || !row || !k || lane < 0 || p->lanes_per_group <= 0 || p->lane_groups <= 0 - || p->chunk_elements <= 0 || uint64_t(lane) >= uint64_t(p->lane_groups) * p->lanes_per_group + // Public C descriptors can be constructed without going through init(). + rocke_matrix_fragment_layout_t checked; + if(!p || !row || !k || lane < 0 + || !rocke_matrix_fragment_layout_init( + &checked, &p->fragment, p->chunk_elements, p->lane_groups, p->lanes_per_group) + || uint64_t(lane) >= uint64_t(p->lane_groups) * p->lanes_per_group || slot >= p->fragment.count) return false; uint64_t chunk; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py index cf601237d07f..1352f07217ff 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py @@ -489,6 +489,10 @@ def _op_memref_global_load_vN(self, op: Op) -> None: prefix = _vec_prefix(elem_name, "global_load_vN") byte_count = vec * (dtype_info(elem_name).encoded_bits // 8) align = int(op.attrs.get("align", vec * 2)) + if align <= 0 or align & (align - 1): + raise ValueError( + "global_load_vN: alignment must be a positive power of two" + ) if align < byte_count: # A vector-pointer dereference would promise natural alignment. # Copy the payload using only the alignment guaranteed by the IR. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py index b711ecf45d26..5ac95d34e755 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py @@ -180,3 +180,14 @@ def test_hip_compiler_preserves_fragment_load_alignment(dtype, copy_bytes, align ] assert source_alignments, "expected unaligned-safe fragment payload copies" assert all(value <= alignment for value in source_alignments), source_alignments + + +@pytest.mark.parametrize("alignment", [0, -1, -16, 3, 24]) +def test_hip_rejects_invalid_vector_load_alignment(alignment): + b = IRBuilder("invalid_alignment") + ptr = b.param("A", PtrType(I8, "global")) + value = b.global_load_vN(ptr, b.const_i32(0), I8, 16) + # Serialized/custom IR can bypass the builder's default alignment handling. + value.op.attrs["align"] = alignment + with pytest.raises(ValueError, match="alignment must be a positive power of two"): + lower_kernel_to_hip(b.kernel, arch="gfx1250") diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp index 47d4749920fb..fd9b9ffbd7a3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp @@ -118,6 +118,54 @@ int main(int argc, char** argv) return emit(argv[2], false); if(argc == 3 && strcmp(argv[1], "--hip") == 0) return emit(argv[2], true); + for(int alignment : {0, -1, -16, 3, 24}) + { + rocke_ir_builder_t b; + CHECK(rocke_ir_builder_init(&b, "invalid_alignment") == ROCKE_OK); + auto* ptr = rocke_b_param(&b, "A", rocke_ptr_type(&b, rocke_i8(), "global"), NULL); + auto* value = rocke_b_global_load_vN(&b, ptr, rocke_b_const_i32(&b, 0), rocke_i8(), 16, 16); + CHECK(value); + // Exercise raw IR, including alignments normalized by the builder. + rocke_attr_set_int(&b, &value->op->attrs, "align", alignment); + rocke_strbuf_t text; + CHECK(rocke_strbuf_init(&text, 256) == 0); + rocke_lower_hip_opts_t opts = {}; + opts.arch = "gfx1250"; + CHECK(rocke_lower_kernel_to_hip(&b, b.kernel, &opts, &text) == ROCKE_ERR_VALUE); + rocke_strbuf_free(&text); + rocke_ir_builder_free(&b); + } + const auto layout = rocke_scaled_matrix_layout("fp6", 16); + uint64_t row, k; + CHECK(rocke_matrix_fragment_coord(&layout, 31, 63, &row, &k)); + CHECK(row == 15 && k == 127); + // C callers can construct or mutate public descriptors without init(). + for(int defect = 0; defect < 6; ++defect) + { + auto invalid = layout; + switch(defect) + { + case 0: + invalid.fragment.packing.element_bits = 0; + break; + case 1: + invalid.fragment.packing.slot_bits = 4; + break; + case 2: + invalid.fragment.carrier_bits = 7; + break; + case 3: + invalid.fragment.carrier_count = 1; + break; + case 4: + invalid.chunk_elements = 3; + break; + case 5: + invalid.chunk_elements = 2; + break; // 12 bits: not byte aligned. + } + CHECK(!rocke_matrix_fragment_coord(&invalid, 0, 0, &row, &k)); + } for(int bits : {4, 6, 8, 16, 32, 64}) { rocke_bit_packing_t packing; From 054d1b3fe1083245025c8f0672b60dab41d5b113 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:26:48 -0400 Subject: [PATCH 04/32] feat(rocke): migrate homogeneous FP6 GEMM onto shared storage --- .../rocke/platform/cpp/core/arch/data.cpp | 92 ++++++++ .../cpp/include/rocke/wmma_scale_internal.h | 4 + .../architecture/dtype_storage_packing.md | 14 +- .../rocke/core/arch/data/arch_specs.json | 4 + .../platform/python/rocke/core/arch/target.py | 55 ++++- .../python/rocke/core/arch/wmma_scale.py | 4 +- .../python/rocke/examples/gfx1250/gemm/FP6.md | 91 ++++++++ .../rocke/examples/gfx1250/gemm/README.md | 55 +++-- .../gfx1250/gemm/block_scaled_gemm_verify.py | 132 +++++++++-- .../rocke/examples/gfx1250/gemm/mxfp6_gemm.py | 44 ++++ .../python/rocke/instances/SUPPORT_MATRIX.md | 6 + .../instances/gfx1250/block_scaled_gemm.py | 138 +++++------ .../core/gfx1250_scaled_wmma_llvm23.json | 8 + .../platform/tests/core/test_arch_mma_ssot.py | 16 +- .../core/test_gfx1250_fp6_scaled_wmma.py | 214 ++++++++++++++++++ .../tests/core/test_gfx1250_mxfp6_example.py | 48 ++++ .../tests/core/test_gfx1250_scaled_wmma.py | 4 +- .../tests/core/test_mma_catalog_lookup.cpp | 4 +- .../instances/parity/gfx1250_lowering_emit.c | 15 +- .../instances/parity/gfx1250_lowering_emit.py | 9 + .../instances/test_gfx1250_mxfp6_numeric.py | 55 +++++ .../instances/test_gfx1250_mxfp_examples.py | 2 +- 22 files changed, 896 insertions(+), 118 deletions(-) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/FP6.md create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/mxfp6_gemm.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scaled_wmma_llvm23.json create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_fp6_scaled_wmma.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mxfp6_example.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp6_numeric.py diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp index 3774425da6ac..4e99590858b6 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp @@ -932,8 +932,12 @@ static const rocke_ati_mma_frag_row_t rocke_ati_mma_frag[] = { {"wmma_gfx1250_f32_16x16x64_bf8_fp8", 8}, {"wmma_gfx1250_f32_16x16x64_bf8_bf8", 8}, {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k32", 8}, {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k32", 8}, {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k16", 8}, {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16", 8}, }; @@ -1738,6 +1742,50 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { 4, &lm_wmma_scale_k32_a, &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, {"wmma_scaled", "bf8e5m2", "bf8e5m2", @@ -1782,6 +1830,50 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { 8, &lm_wmma_scale_k16_a, &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, {"wmma_scaled", "bf8e5m2", "bf8e5m2", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h index 08585dd6b734..2c5622aaf359 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h @@ -112,6 +112,10 @@ static inline rocke_scaled_wmma_op_t rocke_scaled_wmma_contract(const rocke_mma_ spec.matrix_formats[i] = 0; else if(strcmp(dtypes[i], "bf8e5m2") == 0) spec.matrix_formats[i] = 1; + else if(strcmp(dtypes[i], "fp6e2m3") == 0) + spec.matrix_formats[i] = 2; + else if(strcmp(dtypes[i], "fp6e3m2") == 0) + spec.matrix_formats[i] = 3; else ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA matrix format"); if(!scale_dtypes[i] || strcmp(scale_dtypes[i], "e8m0") != 0) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md index 37d8d51b91f0..efc477f8c14c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md @@ -92,8 +92,9 @@ A/B layouts. Existing `a_frag_len`/`b_frag_len` retain their ABI-vector meaning. The current gfx1250 scaled matrix layouts have 64 elements per lane and sixteen i32 carriers. FP8 occupies all sixteen words. FP4 occupies eight and pads eight. -The FP6 transport descriptor occupies twelve and pads four, without enabling -FP6 catalog entries or numerical conversions. +FP6 occupies twelve and pads four. The FP6 consumer adds homogeneous E2M3 and +E3M2 catalog entries with E8M0 scales; the packing descriptor alone does not +enable an instruction or numerical conversion. `ScaleAssociation` records `block_k`: how many source K elements share one scale. This is separate from the bit layout. Existing `ScalePacking(count, @@ -113,10 +114,11 @@ FP4/FP6/FP8 matrix payloads, and scale word packing. Existing GEMM signatures remain compatible. HIP emits declarations for any encountered vector widths absent from its fixed compatibility prologue. -The shared descriptor/helper changes can be consumed by both FP4 and FP6 -branches. FP4/FP8 builder migration can use them directly. Future FP6 numerical -integration should consume the six-bit storage and fragment descriptors and -retain separate target and conversion validation. +The [FP6 GEMM builder](../../python/rocke/instances/gfx1250/block_scaled_gemm.py) +consumes the shared tensor storage, matrix-fragment loader, and scale bit packer. +Its E2M3/E3M2 numerical support is specific to the selected gfx1250 atoms. +FP4 is an independent consumer of the same foundation; it is not a prerequisite +for FP6. Target-independent storage does not establish gfx950 numerical support. First-class tensor-view/fragment IR nodes, arbitrary packed axes, masked partial tiles, concurrent packed stores, and scalar low-bit conversions are separate diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json index 3d12a94fae34..1681f42927f3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json @@ -178,8 +178,12 @@ {"family": "wmma", "a": "bf8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_fp8"}, {"family": "wmma", "a": "bf8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_bf8"}, {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k32"}, {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k32"}, {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k16"}, {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16"} ] }, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py index fe94ac88f845..34b6a449ea3d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py @@ -821,7 +821,8 @@ class _FragInfo: None, _wmma_gfx12_acc_16x16, ), - # Native gfx1250 scaled WMMA. FP8/BF8 use 64 bytes per lane as <16 x i32>. + # Native gfx1250 scaled WMMA. FP8 uses 64 bytes per lane; FP6 uses + # 48 packed bytes padded with four zero words to the same <16 x i32> ABI. # SCALE packs four K=32 E8M0 bytes in i32 and SCALE16 packs eight K=16 # bytes in i64. Both share the gfx12 column-distributed accumulator. "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32": _FragInfo( @@ -837,6 +838,32 @@ class _FragInfo: a_scale_fn=_wmma_gfx1250_a_scale, b_scale_fn=_wmma_gfx1250_b_scale, ), + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k32": _FragInfo( + 16, + 16, + 8, + 32, + None, + None, + _wmma_gfx12_acc_16x16, + a_scale_frag_len=4, + b_scale_frag_len=4, + a_scale_fn=_wmma_gfx1250_a_scale, + b_scale_fn=_wmma_gfx1250_b_scale, + ), + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k32": _FragInfo( + 16, + 16, + 8, + 32, + None, + None, + _wmma_gfx12_acc_16x16, + a_scale_frag_len=4, + b_scale_frag_len=4, + a_scale_fn=_wmma_gfx1250_a_scale, + b_scale_fn=_wmma_gfx1250_b_scale, + ), "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k32": _FragInfo( 16, 16, @@ -863,6 +890,32 @@ class _FragInfo: a_scale_fn=_wmma_gfx1250_a_scale, b_scale_fn=_wmma_gfx1250_b_scale, ), + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k16": _FragInfo( + 16, + 16, + 8, + 32, + None, + None, + _wmma_gfx12_acc_16x16, + a_scale_frag_len=8, + b_scale_frag_len=8, + a_scale_fn=_wmma_gfx1250_a_scale, + b_scale_fn=_wmma_gfx1250_b_scale, + ), + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k16": _FragInfo( + 16, + 16, + 8, + 32, + None, + None, + _wmma_gfx12_acc_16x16, + a_scale_frag_len=8, + b_scale_frag_len=8, + a_scale_fn=_wmma_gfx1250_a_scale, + b_scale_fn=_wmma_gfx1250_b_scale, + ), "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16": _FragInfo( 16, 16, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py index 7f0284ed983c..1634d633cbbb 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py @@ -25,7 +25,7 @@ def __post_init__(self) -> None: def scaled_matrix_layout(dtype: str, abi_words: int) -> MatrixFragmentLayout: """gfx1250 scaled operand layout, independent of atom availability. - FP6 describes the transport contract for future catalog integration. + Matrix formats share storage mechanics but retain their atom-selected layout. """ info = dtype_info(dtype) chunks = {"fp8e4m3": 16, "bf8e5m2": 16, "fp6e2m3": 32, "fp6e3m2": 32, "fp4e2m1": 32} @@ -143,7 +143,7 @@ def gfx1250_scaled_wmma(op_id: str) -> ScaledWmmaOp | None: atom = ArchTarget.from_gfx("gfx1250").mma.by_op_id(op_id.removeprefix("tile.")) if atom is None or atom.family != "wmma_scaled": return None - formats = {"fp8e4m3": 0, "bf8e5m2": 1} + formats = {"fp8e4m3": 0, "bf8e5m2": 1, "fp6e2m3": 2, "fp6e3m2": 3} # The current backend supports E8M0 for both inputs and a shared K-group size. # Keep these restrictions here, independently of the catalog query model. if ( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/FP6.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/FP6.md new file mode 100644 index 000000000000..ecce7492f0c3 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/FP6.md @@ -0,0 +1,91 @@ +# Packed FP6 GEMM + +The [block-scaled GEMM builder](../../../instances/gfx1250/block_scaled_gemm.py) +supports both six-bit formats on gfx1250: `fp6` / `fp6e2m3` means E2M3, +and `bf6` / `fp6e3m2` means E3M2. Both operands use the same six-bit format. Mixed A/B formats are a separate extension. Native paths use K=128 atoms; +`wmma_scale` groups scales by K=32 and `wmma_scale16` by K=16. + +## Contents + +- [Packed storage](#packed-storage) +- [E8M0 scales](#e8m0-scales) +- [ISA and validation references](#isa-and-validation-references) + +## Packed storage + +FP6 buffers are byte arrays with shapes `[M, 3*K/4]` and `[N, 3*K/4]`. +Four consecutive six-bit codes occupy three bytes, low bits first. Codes +cross byte and word boundaries; they are not stored one per byte. Both formats +have a sign at bit 5, finite exponent encodings, subnormals, and signed zero. +E2M3 has exponent bias 1; E3M2 has bias 3. + +A lane with half index `h = lane // 16` consumes K ranges +`[32*h, 32*h+32)` and `[64+32*h, 96+32*h)` for each atom. Each range occupies +24 bytes. The resulting twelve i32 words are padded with four zero words for +the sixteen-word compiler builtin interface. A/B use independent storage descriptors with the same dense format. + +The builder uses [`TensorStorage`](../../../core/storage.py), +[`ScaledWmmaOp.matrix_layout`](../../../core/arch/wmma_scale.py), and +[`load_matrix_fragment`](../../../helpers/mma_io.py). Dense six-bit packing +places sixteen elements across three i32 words. The shared loader derives +alignment and assembles each 24-byte chunk from 16-byte and 8-byte loads. +Numerical encoders and the independent reference decoder remain separate from +these unsigned bit-pattern operations. + +## E8M0 scales + +Native paths use packed E8M0 scale bytes for every matrix pair. The shared +`scale_dtype="e8m0"` selects this contract; `i8` remains its storage alias. + +```python +from rocke.instances.gfx1250.block_scaled_gemm import ( + BlockScaledGemmSpec, build_block_scaled_gemm, +) + +spec = BlockScaledGemmSpec( + name="mxfp6", M=32, N=48, K=256, + dtype_a="fp6e2m3", dtype_b="fp6e2m3", + matrix_path="wmma_scale16", block_k=16, scale_dtype="e8m0", +) +kernel = build_block_scaled_gemm(spec) +``` + +## ISA and validation references + +The [AMD machine-readable ISA](https://gpuopen.com/download/machine-readable-isa/latest/) snapshot +dated 2026-08-06 contains `amdgpu_isa_cdna5.xml`, which defines `FMT_NUM_FP6`, +`FMT_NUM_BF6`, their packed fields, and SCALE/SCALE16 operand widths. The atom +ABI uses sixteen i32 words for each matrix input. The scale operands are B32 +for SCALE and B64 for SCALE16. + +CPU tests compare all 64 encodings of each FP6 format with an independent +numeric dtype and check packed-byte boundaries. GPU tests exercise all code +products at lane/K-group boundaries, neutral and independent A/B scales, +isolated K groups, K=256, with E8M0 scales. These are +bounded correctness fixtures; they do not establish arbitrary-input rounding, +E8M0 NaN behavior, or performance. + +```sh +ROCKE_LLVM_FLAVOR=llvm23 ROCKE_BACKEND=both ROCKE_CPP_STRICT=1 \ +ROCKE_REQUIRE_GFX1250=1 python -m pytest -q -s \ + tests/instances/test_gfx1250_mxfp6_numeric.py +``` + +Use a matching gfx1250 device, LLVM 23 COMGR/HIP, NumPy/ml_dtypes, and a freshly +built C++ extension. COMGR checks Python/C++ LLVM identity; the HIP verifier +compiles the Python HIP lowerer output. Matrix quantization and conversion, +partial tiles, scalar six-bit arithmetic/conversions, and a dynamic K loop remain +outside this packed-input builder. Logical FP6 types already exist, but the +storage descriptors expand into byte loads and integer carrier vectors before +IR serialization; packed TensorView integration remains separate work. + +Run the focused example with: + +```sh +python -m rocke.examples.gfx1250.gemm.mxfp6_gemm --dtype fp6 +python -m rocke.examples.gfx1250.gemm.mxfp6_gemm --dtype bf6 --matrix-path wmma_scale16 --compile-route hip +``` + +`mxfp6_gemm` runs both `fp6` (E2M3) and `bf6` (E3M2) by default, as +separate homogeneous cases. Use `--dtype fp6`, `--dtype bf6`, or explicit +`--dtype both` to select the encodings. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md index 1be444bf4917..880e8ce6a94c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md @@ -1,24 +1,40 @@ # gfx1250 block-scaled GEMM examples -`mxfp8_gemm` runs homogeneous FP8 E4M3 and BF8 E5M2 separately by default. -Use `--dtype fp8` or `--dtype bf8` to run one encoding; `--dtype both` is -the explicit default. Canonical names are `fp8e4m3` and `bf8e5m2`. +Matrix dtype names normalize through the architecture catalog: `fp8e4m3`, +`bf8e5m2`, `fp6e2m3`, `fp6e3m2`, and `fp4e2m1`. The short spellings `fp8`, +`bf8`, `fp6`, `bf6`, and `fp4` remain accepted aliases wherever that matrix +format is supported. Example argument parsing normalizes aliases before +checking the family's supported choices. The FP8 and FP6 examples default to +`both`, which runs their two homogeneous encodings separately. + +These matrix names are distinct from scale formats such as `e8m0`. Scales use +byte storage and packed integer instruction operands; a scale-format name does +not introduce a scalar IR type or general conversion support. Existing operation +IDs retain their instruction-format tokens. + +Each example selects one homogeneous input family and reuses the shared +spec-driven builder, compiler, launcher, and numerical verifier. + +| Module | Matrix inputs | Block scales | +| --- | --- | --- | +| `mxfp8_gemm` | FP8 E4M3 (`fp8`) or BF8 E5M2 (`bf8`) on A and B | E8M0 | +| `mxfp6_gemm` | FP6 E2M3 or E3M2 on A and B | E8M0 | + +Run from an environment with rocKE installed and a visible gfx1250 device: ```bash ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.mxfp8_gemm -ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.mxfp8_gemm --dtype bf8 --matrix-path wmma_scale16 +ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.mxfp6_gemm --dtype fp6 +ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.mxfp6_gemm --dtype bf6 ``` -SCALE uses E8M0 blocks of 32 K elements; SCALE16 uses blocks of 16. -Use `--compile-route hip` for HIP compilation and `--case all` for neutral, -one-sided, combined, and isolated scale-group fixtures. The `mixed` fixture -means scale variation, not different A/B matrix formats. +Defaults use SCALE with K=32 scale groups. Add `--matrix-path wmma_scale16` +for K=16 groups, `--compile-route hip` for HIP compilation, or `--case all` +for neutral, one-sided, combined, and isolated scale-group fixtures. +The `mixed` fixture name means combined A/B scale variation, not mixed dtypes. +See [FP6.md](FP6.md) for the packed FP6 input contract. FP6 does not require FP4. -The examples share argument parsing, the spec-driven builder, compiler, -launcher, and numerical verifier. Numerical tests use opt-in fixtures in -`tests/instances/conftest.py` and are independently runnable by family. - -The target-independent aliases in `core/dtypes.py` normalize logical format +The target-independent aliases in [core/dtypes.py](../../../core/dtypes.py) normalize logical format names. Recognition does not imply that a target supports an atom; use its catalog to query supported operand formats and shapes. Each example defines its accepted matrix and scale formats. @@ -33,10 +49,21 @@ The native loader uses the atom's `a_scale_layout()` and `b_scale_layout()` for lane ownership. Global scale tensors have shapes `[M, K/block_k]` and `[K/block_k, N]`, respectively. K32 packs four E8M0 bytes per lane into i32; K16 packs eight into i64, first K group in the low byte. Both half-waves carry -the same scales. Matrix A/B lane maps for these scaled atoms are not yet exposed. +the same scales. Matrix fragments use the shared storage descriptors and +`ScaledWmmaOp.matrix_layout()`; scale addressing remains a separate contract. CPU tests independently check every scale coordinate, multi-tile addresses and packed byte order. The opt-in numerical suite exercises both encodings and scale group sizes, one-sided/group-isolated inputs, multiple tiles and K steps, and HIP/COMGR compilation. These cases bound the validation; dtype recognition alone does not establish support for other operand contracts. + +`mxfp8_gemm` runs both FP8 E4M3 and BF8 E5M2 by default, as separate +homogeneous cases. Use `--dtype fp8` or `--dtype bf8` to run one encoding, +or `--dtype both` explicitly. The names follow the FP6/BF6 convention. +Both operands use the selected format with E8M0 scales; mixed matrix formats +have a separate example. + +`mxfp6_gemm` runs both `fp6` (E2M3) and `bf6` (E3M2) by default, as +separate homogeneous cases. Use `--dtype fp6`, `--dtype bf6`, or explicit +`--dtype both` to select the encodings. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py index 86252f0c3c9f..03fb53e39d3d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py @@ -3,8 +3,8 @@ """Launch gfx1250 block-scaled GEMM and compare with an independent reference. The default invocation keeps the K=64 FP8/BF8 WMMA + FP32-scale verifier. -Native ``--matrix-path wmma_scale`` / ``wmma_scale16`` use FP8/BF8 and E8M0 -scales with K=32 / K=16 groups. Native fixtures cover K=128 or 256 and use +Native ``--matrix-path wmma_scale`` / ``wmma_scale16`` use homogeneous FP8 or FP6 and +E8M0 scales with K=32 / K=16 groups. Native fixtures cover K=128 or 256 and use bounded dyadic values, permitting exact comparison after BF16 rounding. Run on visible HIP device 0 (must be gfx1250), for example:: @@ -30,9 +30,11 @@ from ....helpers import compile_kernel from ....helpers.compile import compile_kernel_via_hipcc from ....core.dtypes import normalize_dtype +from ....core.storage import BitPacking from ....instances.gfx1250.block_scaled_gemm import ( _LOWBIT_DTYPES, BlockScaledGemmSpec, + _canon_lowbit, block_scaled_gemm_grid, build_block_scaled_gemm, is_valid_spec, @@ -47,6 +49,43 @@ def decode_e8m0(encoded: np.ndarray) -> np.ndarray: return np.ldexp(np.ones(encoded.shape), encoded.astype(np.int32) - 127) +def pack_fp6_codes(codes: np.ndarray) -> np.ndarray: + """Pack four consecutive six-bit codes into three little-endian bytes.""" + if ( + codes.dtype != np.uint8 + or codes.ndim != 2 + or codes.shape[1] % 4 + or np.any(codes > 63) + ): + raise ValueError( + "expected rank-2 uint8 FP6 codes in [0, 63] and K divisible by 4" + ) + packed = b"".join(BitPacking(6).pack(row.tolist()) for row in codes) + return np.frombuffer(packed, dtype=np.uint8).reshape(codes.shape[0], -1) + + +def decode_fp6(packed: np.ndarray, dtype: str = "fp6") -> np.ndarray: + """Decode packed E2M3 or E3M2, including finite extrema and signed zero.""" + if packed.dtype != np.uint8 or packed.ndim != 2 or packed.shape[1] % 3: + raise ValueError("expected rank-2 uint8 FP6 bytes with row size divisible by 3") + if dtype not in ("fp6", "fp6e2m3", "bf6", "fp6e3m2"): + raise ValueError("FP6 dtype must be fp6/E2M3 or bf6/E3M2") + # Bit addressing is independent of the packer's four-code groups. + bits = np.unpackbits(packed, axis=1, bitorder="little").reshape( + packed.shape[0], -1, 6 + ) + codes = (bits * (1 << np.arange(6))).sum(axis=-1) + mantissa_bits, bias = (3, 1) if _canon_lowbit(dtype) == "fp6" else (2, 3) + fraction = (codes & ((1 << mantissa_bits) - 1)) / (1 << mantissa_bits) + exponent = (codes & 31) >> mantissa_bits + magnitude = np.where( + exponent == 0, + np.ldexp(fraction, 1 - bias), + np.ldexp(1 + fraction, exponent - bias), + ) + return np.copysign(magnitude, np.where(codes & 32, -1.0, 1.0)) + + def reference_result( a: np.ndarray, b: np.ndarray, @@ -55,6 +94,8 @@ def reference_result( block_k: int, *, native: bool, + dtype_a: str | None = None, + dtype_b: str | None = None, ) -> np.ndarray: """Expand scales onto logical A/B elements, multiply, then round to BF16. @@ -62,13 +103,22 @@ def reference_result( scales 2**[-2,3]. At K<=256, even the sum of absolute products fits in 2**22 units of 2**-8, so every FP32 partial sum is exact. Float64 host arithmetic and a single BF16 rounding provide an independent oracle. + FP6 all-code fixtures isolate one K element, avoiding accumulation error. """ import ml_dtypes sa = decode_e8m0(a_scale) if native else a_scale.astype(np.float64) sb = decode_e8m0(b_scale) if native else b_scale.astype(np.float64) - scaled_a = a.astype(np.float64) * np.repeat(sa, block_k, axis=1) - scaled_b = b.astype(np.float64) * np.repeat(sb.T, block_k, axis=1) + + def matrix_values(data, dtype): + if dtype in ("fp6", "fp6e2m3", "bf6", "fp6e3m2"): + return decode_fp6(data, dtype) + return data.astype(np.float64) + + a_values = matrix_values(a, dtype_a) + b_values = matrix_values(b, dtype_b) + scaled_a = a_values * np.repeat(sa, block_k, axis=1) + scaled_b = b_values * np.repeat(sb.T, block_k, axis=1) ref = scaled_a @ scaled_b.T if native: ref = ref.astype(ml_dtypes.bfloat16) @@ -92,26 +142,58 @@ def make_case_inputs( "bf8e5m2": ml_dtypes.float8_e5m2, } rng = np.random.default_rng(0xB10C) - magnitude = 0.25 if native else 0.5 - a = (rng.integers(-4, 5, size=(spec.M, spec.K)) * magnitude).astype( - lowbit_types[spec.dtype_a] - ) - b = (rng.integers(-4, 5, size=(spec.N, spec.K)) * magnitude).astype( - lowbit_types[spec.dtype_b] - ) + + def operand(dtype, rows): + kind = _canon_lowbit(dtype) + if kind in ("fp6", "bf6"): + # Small dyadic values ensure exact FP32 partial sums in these tests. + codes = rng.integers( + 0, 9 if kind == "fp6" else 13, size=(rows, spec.K), dtype=np.uint8 + ) + return codes | (rng.integers(0, 2, size=codes.shape, dtype=np.uint8) << 5) + magnitude = 0.25 if native else 0.5 + return (rng.integers(-4, 5, size=(rows, spec.K)) * magnitude).astype( + lowbit_types[dtype] + ) + + a, b = operand(spec.dtype_a, spec.M), operand(spec.dtype_b, spec.N) if native: - sa = rng.integers(125, 131, size=(spec.M, groups), dtype=np.uint8) - sb = rng.integers(125, 131, size=(groups, spec.N), dtype=np.uint8) - neutral = 127 + small = any( + _canon_lowbit(d) in ("fp6", "bf6") for d in (spec.dtype_a, spec.dtype_b) + ) + + sa = rng.integers( + 125, 129 if small else 131, size=(spec.M, groups), dtype=np.uint8 + ) + sb = rng.integers( + 125, 129 if small else 131, size=(groups, spec.N), dtype=np.uint8 + ) + neutral_a = neutral_b = 127 else: sa = rng.uniform(0.5, 1.5, size=(spec.M, groups)).astype(np.float32) sb = rng.uniform(0.5, 1.5, size=(groups, spec.N)).astype(np.float32) - neutral = 1.0 + neutral_a = neutral_b = 1.0 if case in ("neutral", "b-only"): - sa.fill(neutral) + sa.fill(neutral_a) if case in ("neutral", "a-only"): - sb.fill(neutral) - if case.startswith("group-"): + sb.fill(neutral_b) + if case.startswith("codes-"): + if ( + _canon_lowbit(spec.dtype_a) not in ("fp6", "bf6") + or _canon_lowbit(spec.dtype_b) not in ("fp6", "bf6") + or min(spec.M, spec.N) < 64 + ): + raise ValueError("codes-N requires FP6/BF6 operands and M/N >= 64") + k = int(case.removeprefix("codes-")) + if not 0 <= k < spec.K: + raise ValueError("codes-N K index outside the matrix") + a.fill(0) + b.fill(0) + a[:, k] = np.arange(spec.M, dtype=np.uint8) % 64 + b[:, k] = np.arange(spec.N, dtype=np.uint8) % 64 + sa.fill(neutral_a) + sb.fill(neutral_b) + elif case.startswith("group-"): group = int(case.removeprefix("group-")) if not 0 <= group < groups: raise ValueError(f"scale group {group} outside [0, {groups})") @@ -120,6 +202,12 @@ def make_case_inputs( b[:, ~active] = 0 elif case not in ("neutral", "a-only", "b-only", "mixed"): raise ValueError(f"unknown verification case {case!r}") + + def pack(data, dtype): + kind = _canon_lowbit(dtype) + return pack_fp6_codes(data) if kind in ("fp6", "bf6") else data + + a, b = pack(a, spec.dtype_a), pack(b, spec.dtype_b) return a, b, sa, sb @@ -211,7 +299,13 @@ def run_cases( fn = module.get_function(art.kernel_name) for case in cases: inputs = make_case_inputs(spec, case) - expected = reference_result(*inputs, spec.block_k, native=native) + expected = reference_result( + *inputs, + spec.block_k, + native=native, + dtype_a=spec.dtype_a, + dtype_b=spec.dtype_b, + ) label = ( f"{spec.resolved_matrix_path()}/{spec.dtype_a}/{compile_route}/{case} " f"{spec.M}x{spec.N}x{spec.K} bk{spec.block_k}" diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/mxfp6_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/mxfp6_gemm.py new file mode 100644 index 000000000000..2868ca75e9db --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/mxfp6_gemm.py @@ -0,0 +1,44 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Homogeneous packed FP6 E2M3 or E3M2 GEMM with packed E8M0 block scales on gfx1250.""" + +from __future__ import annotations + +from ....core.arch.target import normalize_dtype +from ....instances.gfx1250.block_scaled_gemm import BlockScaledGemmSpec +from ._scaled_gemm_example import argument_parser, verify + + +def make_spec(args) -> BlockScaledGemmSpec: + return BlockScaledGemmSpec( + name="mxfp6_gemm", + M=args.m, + N=args.n, + K=args.k, + dtype_a=args.dtype, + dtype_b=args.dtype, + dtype_c="bf16", + scale_dtype="e8m0", + matrix_path=args.matrix_path, + block_k=16 if args.matrix_path == "wmma_scale16" else 32, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argument_parser(__doc__) + parser.add_argument( + "--dtype", + type=normalize_dtype, + choices=("fp6e2m3", "fp6e3m2", "both"), + default="both", + ) + args = parser.parse_args(argv) + dtypes = ("fp6e2m3", "fp6e3m2") if args.dtype == "both" else (args.dtype,) + for dtype in dtypes: + args.dtype = dtype + verify(make_spec(args), args) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md index 64e980cb7e07..cc7736217a60 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md @@ -42,6 +42,12 @@ These emit generic AMDGPU IR; arch only sets the comgr target triple. --- +## gfx1250 native scaled GEMM + +`block_scaled_gemm` supports homogeneous FP8 E4M3, BF8 E5M2, FP6 E2M3, and FP6 E3M2 +through SCALE and SCALE16 with LLVM 23 and E8M0 scales. +See the [packed FP6 input contract](../examples/gfx1250/gemm/FP6.md). + ## GEMM family | Instance | gfx942 | gfx950 | gfx1151 | Notes | diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py index 9ee88cbdeb4a..6ee051a36702 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py @@ -17,22 +17,26 @@ from ...core.arch import ArchTarget from ...core.arch.wmma_scale import gfx1250_scaled_wmma from ...core.ir import ( - BF16, F16, F32, I8, I32, - I64, IRBuilder, KernelDef, PtrType, Type, VectorType, ) -from ...helpers.quant import quant_ir_type +from ...helpers.mma_io import load_matrix_fragment, pack_fragment_bits, storage_ir_type +from ...core.storage import TensorStorage from ...helpers.spec import SignatureBuilder, ceil_div_grid, kernel_name_join -_LOWBIT_FORMATS = {"fp8e4m3": "fp8", "bf8e5m2": "bf8"} +_LOWBIT_FORMATS = { + "fp8e4m3": "fp8", + "bf8e5m2": "bf8", + "fp6e2m3": "fp6", + "fp6e3m2": "bf6", +} _LOWBIT_DTYPES = frozenset(_LOWBIT_FORMATS) _OUTPUT_DTYPES = {"fp16", "f16", "bf16"} _SCALE_DTYPES = {"fp16", "f16", "fp32", "f32"} @@ -169,10 +173,12 @@ def is_valid_spec(spec: BlockScaledGemmSpec, arch: str = "gfx1250") -> Tuple[boo or normalize_dtype(spec.dtype_b) not in _LOWBIT_DTYPES ): return False, ( - f"A/B must be fp8 or bf8 (got A={spec.dtype_a!r}, B={spec.dtype_b!r})" + f"A/B must be fp8, bf8, fp6, or bf6 (got A={spec.dtype_a!r}, B={spec.dtype_b!r})" ) matrix_path = spec.resolved_matrix_path() native_scale = matrix_path in ("wmma_scale", "wmma_scale16") + if native_scale and _canon_lowbit(spec.dtype_a) != _canon_lowbit(spec.dtype_b): + return False, "native gfx1250 SCALE/SCALE16 requires matching matrix formats" family = matrix_path if native_scale else "wmma" atom_k = _WMMA_SCALE_K if native_scale else _WMMA_K if not native_scale and not target.mma.has_shape( @@ -198,7 +204,7 @@ def is_valid_spec(spec: BlockScaledGemmSpec, arch: str = "gfx1250") -> Tuple[boo required_block_k = 16 if matrix_path == "wmma_scale16" else 32 if spec.block_k != required_block_k: return False, ( - f"{matrix_path} requires block_k={required_block_k} E8M0 groups " + f"{matrix_path} requires block_k={required_block_k} scale groups " f"(got {spec.block_k})" ) try: @@ -244,8 +250,22 @@ def block_scaled_gemm_signature(spec: BlockScaledGemmSpec) -> List[dict]: ) return ( SignatureBuilder() - .ptr("A", _canon_lowbit(spec.dtype_a)) - .ptr("B", _canon_lowbit(spec.dtype_b)) + .ptr( + "A", + ( + "i8" + if storage_ir_type(spec.dtype_a) == I8 + else _canon_lowbit(spec.dtype_a) + ), + ) + .ptr( + "B", + ( + "i8" + if storage_ir_type(spec.dtype_b) == I8 + else _canon_lowbit(spec.dtype_b) + ), + ) .ptr("A_scale", scale) .ptr("B_scale", scale) .ptr("C", spec.dtype_c) @@ -260,15 +280,6 @@ def block_scaled_gemm_grid(spec: BlockScaledGemmSpec) -> Tuple[int, int, int]: return ceil_div_grid((spec.N, spec.tile_n), (spec.M, spec.tile_m)) -def _storage_type(dtype: str) -> Type: - dtype = normalize_dtype(dtype) - if dtype in ("fp16", "f16"): - return F16 - if dtype == "bf16": - return BF16 - return quant_ir_type(dtype) - - def _scale_type(dtype: str) -> Type: return F16 if _wire_scale_dtype(dtype) == "f16" else F32 @@ -293,16 +304,18 @@ def build_block_scaled_gemm( low-bit bytes per lane as ``<8 x i32>``. For FP8/BF8, native fragments carry 64 bytes as ``<16 x i32>`` as four 16-byte K chunks, alternating chunks between lane halves. Both paths use the gfx12 column-distributed ``<8 x f32>`` - accumulator layout. + accumulator layout. FP6 uses prepacked E2M3 or E3M2 bytes: A is + [M, 3*K/4], B is [N, 3*K/4], four codes per three little-endian bytes. + Each lane pads twelve packed i32 words to the sixteen-word builtin ABI. Scale arrays remain A_scale[M, K/block_k] and B_scale[K/block_k, N]. """ ok, reason = is_valid_spec(spec, arch=arch) if not ok: raise ValueError(f"invalid block_scaled_gemm spec for {arch}: {reason}") - a_ty = _storage_type(spec.dtype_a) - b_ty = _storage_type(spec.dtype_b) - c_ty = _storage_type(spec.dtype_c) + a_ty = storage_ir_type(spec.dtype_a) + b_ty = storage_ir_type(spec.dtype_b) + c_ty = storage_ir_type(spec.dtype_c) matrix_path = spec.resolved_matrix_path() native_scale = matrix_path in ("wmma_scale", "wmma_scale16") scale_ty = I8 if native_scale else _scale_type(spec.scale_dtype) @@ -311,7 +324,6 @@ def build_block_scaled_gemm( ) op_id = atom.op_id if atom is not None else _wmma_op_id(spec.dtype_a, spec.dtype_b) scale_op = gfx1250_scaled_wmma(op_id) - atom_k = _WMMA_SCALE_K if native_scale else _WMMA_K frag_words = atom.a_frag_len if atom is not None else _ACC a_frag_ty = VectorType(I32, frag_words) @@ -349,10 +361,28 @@ def build_block_scaled_gemm( n0 = ir.mul(ir.block_id_x(), c16) a_row = ir.add(m0, frag) # this lane's A row b_row = ir.add(n0, frag) # this lane's B row (= output col n) - a_base = ir.mul(a_row, cK) - b_base = ir.mul(b_row, cK) + a_storage = TensorStorage(spec.dtype_a, (spec.M, spec.K), alignment_bytes=16) + b_storage = TensorStorage(spec.dtype_b, (spec.N, spec.K), alignment_bytes=16) + + def _row_stride(storage): + # All current pointer types use their natural byte size; low-bit inputs + # use an explicit byte carrier. A/B storage contracts stay independent. + if storage.info.encoded_bits >= 8: + return cK + values, packed_bytes = storage.packing.group(8) + groups = ir.div(cK, ir.const_i32(values)) + return ( + groups if packed_bytes == 1 else ir.mul(groups, ir.const_i32(packed_bytes)) + ) + + a_stride = _row_stride(a_storage) + b_stride = ( + a_stride if a_storage.packing == b_storage.packing else _row_stride(b_storage) + ) + a_base = ir.mul(a_row, a_stride) + b_base = ir.mul(b_row, b_stride) - def _load_frag(ptr, base, storage_ty, k0): + def _load_frag(ptr, base, storage_ty, k0, storage, operand): if not native_scale: off0 = ir.add(ir.add(base, ir.const_i32(k0)), half_k) off1 = ir.add(off0, ir.const_i32(16)) @@ -360,40 +390,23 @@ def _load_frag(ptr, base, storage_ty, k0): hi = ir.global_load_vN(ptr, off1, storage_ty, 16, align=16) return ir.bitcast(ir.vec_concat(lo, hi), a_frag_ty) - # CK's gfx1250 SCALE and SCALE16 wrappers use the same four 16-byte - # matrix chunks. The instruction variants differ in scale grouping, - # not in their <16 x i32> A/B fragment layout. - chunk_width = 16 - lane_chunk = ir.mul(half, ir.const_i32(chunk_width)) - step_base = ir.add(base, ir.const_i32(k0)) - chunks = [ - ir.global_load_vN( - ptr, - ir.add( - ir.add(step_base, ir.const_i32(offset)), - lane_chunk, - ), - storage_ty, - chunk_width, - align=chunk_width, - ) - for offset in range(0, atom_k, 2 * chunk_width) - ] - packed = chunks[0] - for chunk in chunks[1:]: - packed = ir.vec_concat(packed, chunk) - return ir.bitcast(packed, a_frag_ty) + return load_matrix_fragment( + ir, + ptr, + base, + half, + k0, + storage=storage, + layout=scale_op.matrix_layout(operand), + ) def _pack_strided_scales(ptr, call_idx, *, for_b): assert scale_op is not None and atom is not None layout = atom.b_scale_layout() if for_b else atom.a_scale_layout() - packing = scale_op.scales - count = packing.count - word_ty = I64 if packing.word_bits == 64 else I32 - word_const = ir.const_i64 if packing.word_bits == 64 else ir.const_i32 - packed = word_const(0) + packing = scale_op.scale_packing("b" if for_b else "a") scale_groups = spec.K // spec.block_k - for j in range(count): + + def _load_bits(j): coord0, coord1 = layout.coord(ir, lane, j) group_offset = ir.const_i32(call_idx * (atom.k // spec.block_k)) if for_b: @@ -404,19 +417,16 @@ def _pack_strided_scales(ptr, call_idx, *, for_b): row = ir.add(m0, coord0) group = ir.add(group_offset, coord1) idx = ir.add(ir.mul(row, ir.const_i32(scale_groups)), group) - byte = ir.global_load(ptr, idx, I8, align=1) - widened = ir.zext(byte, word_ty) - shift = word_const(j * packing.element_bits) - shifted = ir.shl(widened, shift) - packed = ir.lor(packed, shifted) - return packed + return ir.global_load(ptr, idx, I8, align=1) + + return pack_fragment_bits(ir, _load_bits, packing.fragment)[0] if native_scale: acc = ir.zero_vec_f32(_ACC) for step in range(spec.K // _WMMA_SCALE_K): k0 = step * _WMMA_SCALE_K - a_frag = _load_frag(A, a_base, a_ty, k0) - b_frag = _load_frag(B, b_base, b_ty, k0) + a_frag = _load_frag(A, a_base, a_ty, k0, a_storage, "a") + b_frag = _load_frag(B, b_base, b_ty, k0, b_storage, "b") a_scale = _pack_strided_scales(AScale, step, for_b=False) b_scale = _pack_strided_scales(BScale, step, for_b=True) acc = ir.mma(op_id, a_frag, b_frag, acc, a_scale, b_scale) @@ -438,8 +448,8 @@ def _pack_strided_scales(ptr, call_idx, *, for_b): acc = ir.zero_vec_f32(_ACC) for step in range(steps_per_group): k0 = kg * spec.block_k + step * _WMMA_K - a_frag = _load_frag(A, a_base, a_ty, k0) - b_frag = _load_frag(B, b_base, b_ty, k0) + a_frag = _load_frag(A, a_base, a_ty, k0, a_storage, "a") + b_frag = _load_frag(B, b_base, b_ty, k0, b_storage, "b") acc = ir.mma(op_id, a_frag, b_frag, acc) # b_scale[kg, n] (col = n0 + frag), shared across this lane's 8 slots. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scaled_wmma_llvm23.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scaled_wmma_llvm23.json new file mode 100644 index 000000000000..1b1aebd55c63 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scaled_wmma_llvm23.json @@ -0,0 +1,8 @@ +{ + "wmma_scale/bf6/bf6/e8m0/e8m0": "5f74da097ed7f569e8973473a8bfffbef4d4ee72e5612cfde09fe285e2e876e3", + "wmma_scale/fp6/fp6/e8m0/e8m0": "5497e2e9065964d988527ec2cfc625a3d50c220b8aa20856b802d1a20a38b047", + "wmma_scale/fp8/fp8/e8m0/e8m0": "da7c4863a5e6cd9518ef538bd078f9033ae3a345bb8f5e41c90fd8d8cdada713", + "wmma_scale16/bf6/bf6/e8m0/e8m0": "514004319c740421f2548a38e10229887e854037651db85b1011ce23a32531f6", + "wmma_scale16/fp6/fp6/e8m0/e8m0": "24c1923138c955b7a134b1a4e7dc77da03a3c308e7afd6b7d223d8edb1c967c4", + "wmma_scale16/fp8/fp8/e8m0/e8m0": "943ca6687c183bb7ec1cd449edb1bf7163a0e94af4e3482ad7b56c0181462b6f" +} diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py index 34b7a8576513..15344cfc952b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py @@ -223,10 +223,15 @@ def test_e5m2_is_not_a_scale_dtype_alias(dtype, field): def test_scaled_catalog_identity_and_backend_contract(): catalog = ArchTarget.from_gfx("gfx1250").mma rows = [row for row in catalog.ops if row.family == "wmma_scaled"] - assert len(rows) == 4 - assert len({row.op_id for row in rows}) == 4 + assert len(rows) == 8 + assert len({row.op_id for row in rows}) == 8 for row in rows: - dtype = {"fp8e4m3": "fp8", "bf8e5m2": "bf8"}[row.a_dtype] + dtype = { + "fp8e4m3": "fp8", + "bf8e5m2": "bf8", + "fp6e2m3": "fp6", + "fp6e3m2": "bf6", + }[row.a_dtype] assert row.op_id == ( f"wmma_gfx1250_f32_16x16x128_{dtype}_{dtype}" f"_scale_e8m0_e8m0_k{row.scale_block_k}" @@ -236,9 +241,8 @@ def test_scaled_catalog_identity_and_backend_contract(): assert isinstance(row.scale_block_k, MmaScaleBlockK) packing = gfx1250_scaled_wmma(row.op_id) assert packing.atom is row - assert packing.matrix_formats == ( - (0, 0) if row.a_dtype == "fp8e4m3" else (1, 1) - ) + selector = {"fp8e4m3": 0, "bf8e5m2": 1, "fp6e2m3": 2, "fp6e3m2": 3}[row.a_dtype] + assert packing.matrix_formats == (selector, selector) assert packing.scales.count * packing.scales.block_k == row.k assert packing.matrix_llvm_types == ("<16 x i32>", "<16 x i32>") for family in ("wmma_scale", "wmma_scale16"): diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_fp6_scaled_wmma.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_fp6_scaled_wmma.py new file mode 100644 index 000000000000..8fd659bcd6ff --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_fp6_scaled_wmma.py @@ -0,0 +1,214 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Homogeneous FP6 packing and native WMMA with E8M0 scales.""" + +import hashlib +import json +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pytest + +from rocke.core.arch import ArchTarget +from rocke.core.backend import resolve_backend +from rocke.core.lower_hip import lower_kernel_to_hip +from rocke.core.lower_llvm import lower_kernel_to_llvm +from rocke.examples.gfx1250.gemm.block_scaled_gemm_verify import ( + decode_fp6, + pack_fp6_codes, +) +from rocke.instances.gfx1250.block_scaled_gemm import ( + BlockScaledGemmSpec, + block_scaled_gemm_signature, + build_block_scaled_gemm, + is_valid_spec, +) + +FORMATS = ("fp8", "bf8", "fp6", "bf6", "fp4") + + +@pytest.mark.parametrize( + "case,expected_sha", + json.loads( + Path(__file__).with_name("gfx1250_scaled_wmma_llvm23.json").read_text() + ).items(), +) +def test_scaled_wmma_llvm23_golden(case, expected_sha): + mode, a, b, sa, sb = case.split("/") + assert sa == sb == "e8m0" + spec = spec_for(a, b, mode) + llvm = lower_kernel_to_llvm( + build_block_scaled_gemm(spec), arch="gfx1250", llvm_flavor="llvm23" + ) + assert hashlib.sha256(llvm.encode()).hexdigest() == expected_sha + + +@pytest.mark.parametrize("alias,canonical", [("fp6", "fp6e2m3"), ("bf6", "fp6e3m2")]) +def test_fp6_catalog_aliases(alias, canonical): + from rocke.core.arch.target import normalize_dtype + + assert normalize_dtype(alias) == canonical + spec = spec_for(alias, alias) + assert is_valid_spec(spec)[0] + + +def spec_for(a="fp6", b="fp6", mode="wmma_scale", **kwargs): + return BlockScaledGemmSpec( + name="fp6_test", + M=32, + N=48, + K=256, + dtype_a=a, + dtype_b=b, + scale_dtype="e8m0", + matrix_path=mode, + block_k=16 if mode == "wmma_scale16" else 32, + **kwargs, + ) + + +@pytest.mark.parametrize( + "dtype,ml_name", [("fp6", "float6_e2m3fn"), ("bf6", "float6_e3m2fn")] +) +def test_all_fp6_codes_against_independent_dtype(dtype, ml_name): + ml = pytest.importorskip("ml_dtypes") + codes = np.arange(64, dtype=np.uint8).reshape(1, -1) + actual = decode_fp6(pack_fp6_codes(codes), dtype) + expected = codes.view(getattr(ml, ml_name)).astype(np.float64) + np.testing.assert_array_equal(actual, expected) + np.testing.assert_array_equal(np.signbit(actual), np.signbit(expected)) + assert actual[0, 31] == (7.5 if dtype == "fp6" else 28.0) + + +def test_fp6_cross_byte_packing(): + codes = np.array([[1, 2, 3, 4, 63, 0, 32, 1]], dtype=np.uint8) + np.testing.assert_array_equal( + pack_fp6_codes(codes), [[0x81, 0x30, 0x10, 0x3F, 0, 6]] + ) + + +@pytest.mark.parametrize( + "codes", + [ + np.array([[64, 0, 0, 0]], dtype=np.uint8), + np.zeros((1, 3), dtype=np.uint8), + np.zeros((1, 4), dtype=np.int32), + ], +) +def test_invalid_fp6_codes(codes): + with pytest.raises(ValueError, match="FP6 codes"): + pack_fp6_codes(codes) + + +@pytest.mark.parametrize( + "a,b,mode", + [(d, d, m) for d in ("fp6", "bf6") for m in ("wmma_scale", "wmma_scale16")], +) +def test_all_native_matrix_pairs(a, b, mode): + spec = spec_for(a, b, mode) + assert is_valid_spec(spec)[0] + atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( + family="wmma_scaled", + a_dtype=a, + b_dtype=b, + c_dtype="fp32", + m=16, + n=16, + k=128, + scales=("e8m0", "e8m0", spec.block_k), + ) + assert (atom.a_frag_len, atom.b_frag_len, atom.c_frag_len) == (16, 16, 8) + kernel = build_block_scaled_gemm(spec) + llvm = lower_kernel_to_llvm(kernel, arch="gfx1250", llvm_flavor="llvm23") + call = next( + l for l in llvm.splitlines() if "call <8 x float> @llvm.amdgcn.wmma.scale" in l + ) + assert f"i32 {FORMATS.index(a)}, <16 x i32>" in call + assert f"i32 {FORMATS.index(b)}, <16 x i32>" in call + hip = lower_kernel_to_hip(kernel, arch="gfx1250") + assert f"__builtin_amdgcn_{mode}_f32_16x16x128_f8f6f4({FORMATS.index(a)}," in hip + for flavor in ("llvm20", "llvm22"): + error_type = RuntimeError if resolve_backend() == "cpp" else NotImplementedError + with pytest.raises(error_type, match="requires llvm23"): + lower_kernel_to_llvm(kernel, arch="gfx1250", llvm_flavor=flavor) + if a in ("fp6", "bf6"): + assert block_scaled_gemm_signature(spec)[0]["type"] == "ptr" + + +@pytest.mark.parametrize( + "changes", + [ + {"matrix_path": "wmma"}, + {"K": 192}, + {"scale_dtype": "fp32"}, + ], +) +def test_reject_invalid_fp6_contract(changes): + spec = replace(spec_for(), **changes) + assert not is_valid_spec(spec)[0] + with pytest.raises(ValueError): + build_block_scaled_gemm(spec) + + +def test_fp6_aliases(): + original = spec_for() + alias = replace(original, dtype_a="fp6e2m3", dtype_b="fp6e2m3") + assert alias.kernel_name() == original.kernel_name() + assert is_valid_spec(alias)[0] + + +@pytest.mark.parametrize("a,b", [("fp6", "bf6"), ("fp8", "fp4"), ("fp4", "fp4")]) +def test_other_matrix_contracts_are_not_admitted(a, b): + assert not is_valid_spec(spec_for(a, b))[0] + + +@pytest.mark.parametrize("mode", ["wmma_scale", "wmma_scale16"]) +@pytest.mark.parametrize("a", FORMATS) +@pytest.mark.parametrize("b", FORMATS) +def test_homogeneous_catalog_boundary(a, b, mode): + spec = spec_for(a, b, mode) + accepted = a == b and a in ("fp8", "bf8", "fp6", "bf6") + assert is_valid_spec(spec)[0] == accepted + atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( + family="wmma_scaled", + a_dtype=a, + b_dtype=b, + c_dtype="fp32", + m=16, + n=16, + k=128, + scales=("e8m0", "e8m0", spec.block_k), + ) + assert (atom is not None) == accepted + + +@pytest.mark.parametrize("dtype", ["fp6", "bf6"]) +@pytest.mark.parametrize("block_k", [16, 32]) +def test_fp6_atom_storage_contract(dtype, block_k): + from rocke.core.arch.wmma_scale import gfx1250_scaled_wmma + + atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( + family="wmma_scaled", + a_dtype=dtype, + b_dtype=dtype, + c_dtype="fp32", + m=16, + n=16, + k=128, + scales=("e8m0", "e8m0", block_k), + ) + contract = gfx1250_scaled_wmma(atom.op_id) + for operand in ("a", "b"): + layout = contract.matrix_layout(operand) + assert layout.fragment.packing.group(32) == (16, 3) + assert layout.chunk_elements == 32 and layout.chunk_bytes == 24 + assert layout.fragment.live_carriers == 12 + assert layout.fragment.padding_bits == 128 + assert contract.scale_packing(operand).association.block_k == block_k + # Literal bit-stream oracle, independent of the shared packer. + patterns = list(range(64)) + expected = sum(value << (6 * i) for i, value in enumerate(patterns)) + assert layout.fragment.pack(patterns) == tuple( + (expected >> (32 * i)) & 0xFFFFFFFF for i in range(16) + ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mxfp6_example.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mxfp6_example.py new file mode 100644 index 000000000000..c862dd217310 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mxfp6_example.py @@ -0,0 +1,48 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""The six-bit example defaults to both homogeneous encodings.""" + +import pytest +from rocke.examples.gfx1250.gemm import _scaled_gemm_example, mxfp6_gemm + + +@pytest.mark.parametrize( + "argv,dtypes", + [ + (["--dtype", "fp6e2m3"], ("fp6e2m3",)), + (["--dtype", " FP6 "], ("fp6e2m3",)), + (["--dtype", "fp6e3m2"], ("fp6e3m2",)), + (["--dtype", " BF6 "], ("fp6e3m2",)), + ([], ("fp6e2m3", "fp6e3m2")), + (["--dtype", "both"], ("fp6e2m3", "fp6e3m2")), + (["--dtype", "fp6"], ("fp6e2m3",)), + (["--dtype", "bf6"], ("fp6e3m2",)), + ], +) +def test_mxfp6_cli_selects_formats(monkeypatch, argv, dtypes): + calls = [] + + def run(spec, cases, **kwargs): + calls.append((spec, cases, kwargs)) + return len(cases) + + monkeypatch.setattr(_scaled_gemm_example, "run_cases", run) + assert ( + mxfp6_gemm.main( + argv + + [ + "--matrix-path", + "wmma_scale16", + "--compile-route", + "hip", + "--case", + "all", + ] + ) + == 0 + ) + assert tuple(spec.dtype_a for spec, _, _ in calls) == dtypes + for spec, cases, kwargs in calls: + assert spec.dtype_a == spec.dtype_b + assert spec.block_k == 16 and len(cases) == 20 + assert kwargs == {"compile_route": "hip"} diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py index 5e66d0728d75..167deaedad14 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py @@ -335,14 +335,14 @@ def test_native_slice_rejects_unsupported_contracts(self): M=16, N=16, K=128, - dtype_b="bf8", + dtype_b="fp16", scale_dtype="e8m0", block_k=32, matrix_path="wmma_scale", ) ok, why = is_valid_spec(bad_dtype) self.assertFalse(ok) - self.assertIn("requested operand and scale contract", why) + self.assertIn("A/B must be", why) bad_block = BlockScaledGemmSpec( name="bad_block", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp index 0ca6d1c65b9e..4a5a140b336c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp @@ -307,7 +307,7 @@ static int test_scale_contracts() ++scaled_rows; } } - CHECK(packed_rows == 4 && scaled_rows == 4); + CHECK(packed_rows == 4 && scaled_rows == 8); return 0; } @@ -376,7 +376,7 @@ static int test_scale_layouts_and_families() } } } - CHECK(scaled == 4); + CHECK(scaled == 8); return 0; } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c index 77993e4bd0dc..485679a5a480 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c @@ -383,6 +383,8 @@ typedef struct config { build_fn_t build; const char* arch; + const char* dtype; + bool scale16; } config_t; /* Each gfx1250 config that tests a *choice* of encoding is followed by its @@ -411,6 +413,10 @@ static const config_t CONFIGS[] = { {build_global_tr16_bf16, "gfx1250"}, {build_global_tr16_i16, "gfx1250"}, {build_tensor_transfers, "gfx1250"}, + {NULL, "gfx1250", "fp6", false}, + {NULL, "gfx1250", "bf6", false}, + {NULL, "gfx1250", "fp6", true}, + {NULL, "gfx1250", "bf6", true}, {build_wmma_scale_bf8, "gfx1250"}, {build_wmma_scale16_bf8, "gfx1250"}, {build_scale_coordinates_k32, "gfx1250"}, @@ -449,7 +455,14 @@ int main(int argc, char** argv) } /* Python: b.kernel.attrs["max_workgroup_size"] = 64 */ rocke_attr_set_int(&b, &b.kernel->attrs, "max_workgroup_size", 64); - CONFIGS[idx].build(&b); + if(CONFIGS[idx].build) + { + CONFIGS[idx].build(&b); + } + else + { + wmma_scaled(&b, CONFIGS[idx].scale16, CONFIGS[idx].dtype); + } if(!rocke_ir_builder_ok(&b)) { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py index 7dd837947469..9f177887bd52 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py @@ -267,6 +267,15 @@ def build_tensor_transfers(b: IRBuilder) -> None: ] +# Matrix pairs with E8M0 scales; retain the original config indices above. +_SCALED_PAIRS = [("fp6", "fp6"), ("bf6", "bf6")] +CONFIGS.extend( + (_wmma_scaled(a, b, mode), "gfx1250") + for mode in ("scale", "scale16") + for a, b in _SCALED_PAIRS +) + + # Homogeneous BF8 belongs to the eight-bit example contract. CONFIGS.extend( (_wmma_scaled("bf8", "bf8", mode), "gfx1250") for mode in ("scale", "scale16") diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp6_numeric.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp6_numeric.py new file mode 100644 index 000000000000..d1fdc8de05ad --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp6_numeric.py @@ -0,0 +1,55 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""gfx1250 MXFP6 numerical cases.""" + +import subprocess +import sys + +import pytest + + +@pytest.mark.parametrize( + "dtype,matrix_path,route,m,n,k,case,count", + [ + (dtype, *case) + for dtype in ("fp6", "bf6") + for case in [ + ("wmma_scale", "comgr", 16, 16, 128, "all", 8), + ("wmma_scale16", "comgr", 16, 16, 128, "all", 12), + ("wmma_scale", "comgr", 32, 48, 256, "mixed", 1), + ("wmma_scale16", "comgr", 32, 48, 256, "mixed", 1), + ("wmma_scale", "hip", 32, 48, 256, "mixed", 1), + ("wmma_scale16", "hip", 32, 48, 256, "mixed", 1), + ] + ], +) +def test_numeric(numeric_case, dtype, matrix_path, route, m, n, k, case, count): + numeric_case(dtype, matrix_path, route, m, n, k, case, count) + + +@pytest.mark.parametrize("dtype", ["fp6", "bf6"]) +@pytest.mark.parametrize("matrix_path", ["wmma_scale", "wmma_scale16"]) +@pytest.mark.parametrize("route", ["comgr", "hip"]) +def test_all_fp6_codes_numeric(gpu_env, dtype, matrix_path, route): + # All 64 x 64 code products at both sides of the lane-half/K-group boundaries. + script = """ +from rocke.instances.gfx1250.block_scaled_gemm import BlockScaledGemmSpec +from rocke.examples.gfx1250.gemm.block_scaled_gemm_verify import run_cases +import sys +dtype, path, route = sys.argv[1:] +spec = BlockScaledGemmSpec(name='fp6_codes', M=64, N=64, K=128, + dtype_a=dtype, dtype_b=dtype, matrix_path=path, scale_dtype='e8m0', + block_k=16 if path == 'wmma_scale16' else 32) +assert run_cases(spec, tuple(f'codes-{k}' for k in (0,31,32,63,64,95,96,127)), compile_route=route) == 8 +""" + result = subprocess.run( + [sys.executable, "-c", script, dtype, matrix_path, route], + env=gpu_env, + capture_output=True, + text=True, + timeout=300, + ) + output = result.stdout + result.stderr + assert result.returncode == 0, output + assert output.count("bad=0") == 8, output + print(output, end="") diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp_examples.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp_examples.py index 91e6513daf8c..4f9cccc8b34c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp_examples.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp_examples.py @@ -12,7 +12,7 @@ from rocke.instances.gfx1250.block_scaled_gemm import build_block_scaled_gemm -@pytest.mark.parametrize("family,dtype", [("mxfp8", "fp8")]) +@pytest.mark.parametrize("family,dtype", [("mxfp8", "fp8"), ("mxfp6", "fp6")]) @pytest.mark.parametrize("path,block_k", [("wmma_scale", 32), ("wmma_scale16", 16)]) def test_example_contract_and_lowering(family, dtype, path, block_k): example = importlib.import_module(f"rocke.examples.gfx1250.gemm.{family}_gemm") From 29b5454ee2a7434ec8236d1cc7f2bf8b91ed1ae0 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Thu, 24 Sep 2026 14:08:31 -0400 Subject: [PATCH 05/32] fix(rocke): validate storage descriptors and typed row strides --- .../rocke/platform/cpp/core/storage.cpp | 18 ++- .../rocke/platform/cpp/helpers/mma_io.cpp | 6 +- .../platform/cpp/include/rocke/storage.h | 1 + .../architecture/dtype_storage_packing.md | 7 + .../platform/python/rocke/helpers/mma_io.py | 9 +- .../rocke/platform/tests/core/test_mma_io.py | 61 ++++++++- .../platform/tests/core/test_storage.cpp | 125 +++++++++++++++++- .../rocke/platform/tests/core/test_storage.py | 7 + 8 files changed, 217 insertions(+), 17 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp index 5ad91cf3287a..131c00e3989d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp @@ -196,7 +196,7 @@ bool rocke_tensor_storage_init(rocke_tensor_storage_t* out, p.row_stride_bytes = row_stride_bytes == UINT64_MAX ? row_bytes : row_stride_bytes; p.base_bit_offset = base_bit_offset; p.alignment_bytes = alignment_bytes; - if(p.row_stride_bytes < row_bytes || !rocke_tensor_storage_bytes(&p, &bytes)) + if(!rocke_tensor_storage_bytes(&p, &bytes)) return false; *out = p; return true; @@ -204,22 +204,28 @@ bool rocke_tensor_storage_init(rocke_tensor_storage_t* out, bool rocke_tensor_storage_bytes(const rocke_tensor_storage_t* p, uint64_t* bytes) { - if(!p || !bytes || !valid(&p->packing)) + uint64_t row_bytes, prefix; + if(!p || !bytes || !p->dtype || !valid(&p->packing) + || rocke_dtype_info(p->dtype->name) != p->dtype + || p->packing.element_bits != p->dtype->encoded_bits || !p->alignment_bytes + || (p->alignment_bytes & (p->alignment_bytes - 1)) + || !rocke_bit_packing_bytes(&p->packing, p->cols, p->base_bit_offset, &row_bytes) + || p->row_stride_bytes < row_bytes) return false; if(!p->rows || !p->cols) { *bytes = 0; return true; } - uint64_t row_bytes, prefix; - return rocke_bit_packing_bytes(&p->packing, p->cols, p->base_bit_offset, &row_bytes) - && mul(p->rows - 1, p->row_stride_bytes, &prefix) && add(prefix, row_bytes, bytes); + return mul(p->rows - 1, p->row_stride_bytes, &prefix) && add(prefix, row_bytes, bytes); } bool rocke_tensor_storage_address( const rocke_tensor_storage_t* p, uint64_t row, uint64_t col, uint64_t* byte, int* shift) { - if(!p || !byte || !shift || row >= p->rows || col >= p->cols) + uint64_t bytes; + if(!byte || !shift || !rocke_tensor_storage_bytes(p, &bytes) || row >= p->rows + || col >= p->cols) return false; uint64_t bits, prefix; if(!rocke_bit_packing_offset(&p->packing, col, &bits) || !add(bits, p->base_bit_offset, &bits) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp index ecb70efaefb2..c7d01678a04e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp @@ -45,6 +45,9 @@ rocke_value_t* rocke_h_load_matrix_fragment(rocke_ir_builder_t* b, if(!storage || !storage->dtype || !layout || !carrier_type || !ptr || !row_base || !lane_group) ckc::raise_status(ROCKE_ERR_VALUE, "null matrix fragment argument"); + uint64_t storage_bytes; + if(!rocke_tensor_storage_bytes(storage, &storage_bytes)) + ckc::raise_status(ROCKE_ERR_VALUE, "invalid tensor storage descriptor"); const auto& packing = layout->fragment; rocke_matrix_fragment_layout_t checked; if(!rocke_matrix_fragment_layout_init(&checked, @@ -69,7 +72,8 @@ rocke_value_t* rocke_h_load_matrix_fragment(rocke_ir_builder_t* b, uint64_t origin_bits, chunk_bytes; if(!rocke_bit_packing_offset(&storage->packing, k0, &origin_bits) || !rocke_bit_packing_bytes(&packing.packing, layout->chunk_elements, 0, &chunk_bytes) - || origin_bits % (8 * unit_bytes) || chunk_bytes % unit_bytes) + || origin_bits % (8 * unit_bytes) || chunk_bytes % unit_bytes + || storage->row_stride_bytes % unit_bytes) ckc::raise_status(ROCKE_ERR_VALUE, "matrix fragment is not aligned to pointer storage units"); const rocke_dtype_info_t* carrier_info = rocke_dtype_info(carrier_type->name); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h index 3ea1794d6369..133c1310c9ac 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h @@ -104,6 +104,7 @@ bool rocke_tensor_storage_init(rocke_tensor_storage_t* out, int slot_bits, uint64_t base_bit_offset, uint64_t alignment_bytes); +/* Size and address queries validate public descriptors; invalid input returns false. */ bool rocke_tensor_storage_bytes(const rocke_tensor_storage_t* p, uint64_t* bytes); bool rocke_tensor_storage_address( const rocke_tensor_storage_t* p, uint64_t row, uint64_t col, uint64_t* byte, int* shift); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md index 37d8d51b91f0..ecab80f8458e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md @@ -71,15 +71,22 @@ For example, dense FP6 uses four elements in three bytes; `slot_bits=8` describe one six-bit pattern in each byte instead. Both have the same logical dtype. Address calculation includes the bit offset, and allocation bounds include the partial tail byte. Empty views require no accesses. +Native size queries, address queries, and fragment loads revalidate the public +storage descriptor, including dtype/packing consistency, stride, alignment, and +allocation bounds, because C callers can construct or mutate these structs. [`storage_ir_type`](../../python/rocke/helpers/mma_io.py) chooses an addressable unit. FP4/FP6 use I8 units while retaining their logical identities in the storage descriptor. This function is deliberately separate from `dtype_to_ir_type`. +FP8/BF8 retain nominal FP8/BF8 pointer types despite occupying one byte per value. The fragment loader accepts complete, byte-aligned chunks. It derives load alignment from base alignment, row stride, chunk spacing, and origin. A 24-byte FP6 chunk becomes 16-byte and 8-byte loads without reading past the chunk. Partial fragments and nonzero bit origins are rejected by this initial loader. +Row strides must also be divisible by the pointer storage unit: an odd-byte +FP16/BF16 stride is valid generic storage but cannot be loaded through a typed +two-byte pointer. `row_base` is an element index in those storage units. The host bit packer writes a fresh, exclusively owned output buffer; concurrent packed stores are not provided. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py index 8d0bae29b5a2..38bcbe29e7e1 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py @@ -43,7 +43,8 @@ def load_matrix_fragment( ) -> Value: """Load whole-byte chunks directly into the atom's carrier vector. - row_base counts pointer storage units. lane_group is derived from the atom's + row_base counts pointer storage units, which must divide the row stride. + lane_group is derived from the atom's lane mapping by the caller. Partial chunks and nonzero bit origins reject. """ packing = layout.fragment @@ -61,7 +62,11 @@ def load_matrix_fragment( if k0 < 0 or k0 + span > storage.shape[1]: raise ValueError("matrix fragment exceeds the packed row") origin_bits = storage.packing.bit_offset(k0) - if origin_bits % (8 * unit_bytes) or layout.chunk_bytes % unit_bytes: + if ( + origin_bits % (8 * unit_bytes) + or layout.chunk_bytes % unit_bytes + or storage.row_stride_bytes % unit_bytes + ): raise ValueError("matrix fragment is not aligned to pointer storage units") if dtype_info(carrier_type.name).encoded_bits != packing.carrier_bits: raise ValueError("matrix carrier type width mismatch") diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py index 5ac95d34e755..190beabfe0e9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py @@ -37,6 +37,8 @@ "bf8", "f16", "bf16", + "f16_padded", + "bf16_padded", "pack_fp6_cross_word", "pack_scale_bytes_i32", "pack_scale_bytes_i64", @@ -45,8 +47,8 @@ def build_transport(dtype): b = IRBuilder("transport") - padded = dtype == "fp6_padded" - dtype = "fp6" if padded else dtype + padded = dtype.endswith("_padded") + dtype = dtype.removesuffix("_padded") patterns = dtype.startswith("pack_") unit = I8 if patterns else storage_ir_type(dtype) typed = dtype in ("f16", "bf16") @@ -69,7 +71,7 @@ def build_transport(dtype): storage = TensorStorage( dtype, (16, 128), - row_stride_bytes=97 if padded else None, + row_stride_bytes=(258 if typed else 97) if padded else None, alignment_bytes=16, ) layout = ( @@ -77,7 +79,7 @@ def build_transport(dtype): if typed else scaled_matrix_layout(dtype, 16) ) - base = b.const_i32(0) + base = b.const_i32((129 if typed else 97) if padded else 0) lane = b.mod(b.thread_id_x(), b.const_i32(32)) group = b.div(lane, b.const_i32(16)) value = load_matrix_fragment( @@ -157,6 +159,57 @@ def test_hip_declares_only_encountered_missing_vector_widths(): assert "using i8x24 =" not in source +@pytest.mark.parametrize("dtype", ["f16", "bf16"]) +def test_reject_row_stride_not_in_pointer_units(dtype): + b = IRBuilder("odd_stride") + unit = storage_ir_type(dtype) + ptr = b.param("A", PtrType(unit, "global")) + zero = b.const_i32(0) + before = serialize(b.kernel) + with pytest.raises(ValueError, match="aligned to pointer storage units"): + load_matrix_fragment( + b, + ptr, + zero, + zero, + 0, + storage=TensorStorage(dtype, (2, 64), row_stride_bytes=129), + layout=MatrixFragmentLayout( + FragmentPacking(BitPacking(16), 32, 16, 32), 16, 2, 16 + ), + carrier_type=unit, + ) + assert serialize(b.kernel) == before + + +@pytest.mark.parametrize("dtype", ["fp8", "bf8"]) +def test_byte_width_does_not_erase_pointer_identity(dtype): + assert storage_ir_type(dtype) != I8 + b = IRBuilder("nominal_pointer") + ptr = b.param("A", PtrType(I8, "global")) + zero = b.const_i32(0) + with pytest.raises(ValueError, match="pointer storage type mismatch"): + load_matrix_fragment( + b, + ptr, + zero, + zero, + 0, + storage=TensorStorage(dtype, (16, 128)), + layout=scaled_matrix_layout(dtype, 16), + ) + + +@pytest.mark.parametrize("elem_type", ["unknown", "fp4e2m1"]) +def test_hip_rejects_unsupported_vector_element(elem_type): + b = IRBuilder("invalid_element") + ptr = b.param("A", PtrType(I8, "global")) + value = b.global_load_vN(ptr, b.const_i32(0), I8, 16) + value.op.attrs["elem_type"] = elem_type + with pytest.raises(NotImplementedError, match="unsupported element type"): + lower_kernel_to_hip(b.kernel, arch="gfx1250") + + @pytest.mark.parametrize( "dtype,copy_bytes,alignment", [("fp6", 16, 8), ("fp6_padded", 16, 1), ("f16", 32, 16)], diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp index fd9b9ffbd7a3..cd210412cf9b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp @@ -33,9 +33,11 @@ static int emit(const char* dtype, bool hip) { rocke_ir_builder_t b; CHECK(rocke_ir_builder_init(&b, "transport") == ROCKE_OK); - const bool padded = strcmp(dtype, "fp6_padded") == 0; + const bool padded = strstr(dtype, "_padded") != NULL; if(padded) - dtype = "fp6"; + dtype = strcmp(dtype, "f16_padded") == 0 ? "f16" + : strcmp(dtype, "bf16_padded") == 0 ? "bf16" + : "fp6"; bool patterns = strncmp(dtype, "pack_", 5) == 0; const auto* unit = patterns ? rocke_i8() : rocke_storage_ir_type(dtype); CHECK(unit); @@ -64,7 +66,7 @@ static int emit(const char* dtype, bool hip) { rocke_tensor_storage_t storage; CHECK(rocke_tensor_storage_init( - &storage, dtype, 16, 128, padded ? 97 : UINT64_MAX, 0, 0, 16)); + &storage, dtype, 16, 128, padded ? (typed ? 258 : 97) : UINT64_MAX, 0, 0, 16)); rocke_matrix_fragment_layout_t layout; if(typed) { @@ -76,7 +78,7 @@ static int emit(const char* dtype, bool hip) } else layout = rocke_scaled_matrix_layout(dtype, 16); - auto* base = rocke_b_const_i32(&b, 0); + auto* base = rocke_b_const_i32(&b, padded ? (typed ? 129 : 97) : 0); auto* thread = rocke_b_thread_id_x(&b); auto* lane = rocke_b_mod(&b, thread, rocke_b_const_i32(&b, 32)); auto* group = rocke_b_div(&b, lane, rocke_b_const_i32(&b, 16)); @@ -112,12 +114,127 @@ static int emit(const char* dtype, bool hip) return 0; } +static int test_storage_validation() +{ + rocke_tensor_storage_t storage; + CHECK(rocke_tensor_storage_init(&storage, "fp6", 16, 128, 97, 0, 0, 16)); + const auto layout = rocke_scaled_matrix_layout("fp6", 16); + for(int defect = 0; defect < 10; ++defect) + { + auto invalid = storage; + switch(defect) + { + case 0: + invalid.dtype = NULL; + break; + case 1: + invalid.dtype = rocke_dtype_info("fp8"); + break; + case 2: + invalid.packing.element_bits = 0; + break; + case 3: + invalid.packing.slot_bits = 0; + break; + case 4: + invalid.packing.slot_bits = 4; + break; + case 5: + invalid.row_stride_bytes = 95; + break; + case 6: + invalid.alignment_bytes = 0; + break; + case 7: + invalid.alignment_bytes = 3; + break; + case 8: + invalid.rows = UINT64_MAX; + break; + case 9: + invalid.base_bit_offset = UINT64_MAX; + break; + } + uint64_t bytes; + int shift; + CHECK(!rocke_tensor_storage_bytes(&invalid, &bytes)); + CHECK(!rocke_tensor_storage_address(&invalid, 0, 0, &bytes, &shift)); + rocke_ir_builder_t b; + CHECK(rocke_ir_builder_init(&b, "invalid_storage") == ROCKE_OK); + auto* ptr = rocke_b_param(&b, "A", rocke_ptr_type(&b, rocke_i8(), "global"), NULL); + auto* zero = rocke_b_const_i32(&b, 0); + CHECK( + !rocke_h_load_matrix_fragment(&b, ptr, zero, zero, 0, &invalid, &layout, rocke_i32())); + CHECK(!rocke_ir_builder_ok(&b)); + rocke_ir_builder_free(&b); + if(defect < 8) + { + invalid.rows = 0; + CHECK(!rocke_tensor_storage_bytes(&invalid, &bytes)); + } + } + for(const char* dtype : {"f16", "bf16", "fp8", "bf8"}) + { + const auto* unit = rocke_storage_ir_type(dtype); + CHECK(unit && !rocke_type_eq(unit, rocke_i8())); + const bool typed = strcmp(dtype, "f16") == 0 || strcmp(dtype, "bf16") == 0; + rocke_tensor_storage_t direct = {rocke_dtype_info(dtype), + 2, + 128, + typed ? 257u : 129u, + 0, + 1, + {typed ? 16 : 8, typed ? 16 : 8}}; + uint64_t bytes; + int shift; + CHECK(rocke_tensor_storage_bytes(&direct, &bytes)); + CHECK(rocke_tensor_storage_address(&direct, 1, 0, &bytes, &shift)); + CHECK(bytes == direct.row_stride_bytes && shift == 0); + rocke_matrix_fragment_layout_t fragment; + if(typed) + { + rocke_fragment_packing_t packing; + CHECK(rocke_fragment_packing_init(&packing, &direct.packing, 32, 16, 32)); + CHECK(rocke_matrix_fragment_layout_init(&fragment, &packing, 16, 2, 16)); + } + else + fragment = rocke_scaled_matrix_layout(dtype, 16); + rocke_ir_builder_t b; + CHECK(rocke_ir_builder_init(&b, "invalid_pointer_units") == ROCKE_OK); + auto* ptr + = rocke_b_param(&b, "A", rocke_ptr_type(&b, typed ? unit : rocke_i8(), "global"), NULL); + auto* zero = rocke_b_const_i32(&b, 0); + CHECK(!rocke_h_load_matrix_fragment( + &b, ptr, zero, zero, 0, &direct, &fragment, typed ? unit : rocke_i32())); + CHECK(!rocke_ir_builder_ok(&b)); + rocke_ir_builder_free(&b); + } + return 0; +} + int main(int argc, char** argv) { if(argc == 3 && strcmp(argv[1], "--emit") == 0) return emit(argv[2], false); if(argc == 3 && strcmp(argv[1], "--hip") == 0) return emit(argv[2], true); + CHECK(test_storage_validation() == 0); + for(const char* elem_type : {"unknown", "fp4e2m1"}) + { + rocke_ir_builder_t b; + CHECK(rocke_ir_builder_init(&b, "invalid_element") == ROCKE_OK); + auto* ptr = rocke_b_param(&b, "A", rocke_ptr_type(&b, rocke_i8(), "global"), NULL); + auto* value = rocke_b_global_load_vN(&b, ptr, rocke_b_const_i32(&b, 0), rocke_i8(), 16, 16); + CHECK(value); + rocke_attr_set_str(&b, &value->op->attrs, "elem_type", elem_type); + rocke_strbuf_t text; + CHECK(rocke_strbuf_init(&text, 256) == 0); + rocke_lower_hip_opts_t opts = {}; + opts.arch = "gfx1250"; + CHECK(rocke_lower_kernel_to_hip(&b, b.kernel, &opts, &text) == ROCKE_ERR_KEY); + rocke_strbuf_free(&text); + rocke_ir_builder_free(&b); + } for(int alignment : {0, -1, -16, 3, 24}) { rocke_ir_builder_t b; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py index 515430599388..6824621d8f9c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py @@ -80,6 +80,13 @@ def test_storage_rows_offsets_and_empty_views(): TensorStorage("fp4", (1, 4), alignment_bytes=3) +@pytest.mark.parametrize("dtype", ["f16", "bf16"]) +def test_storage_allows_byte_strides(dtype): + storage = TensorStorage(dtype, (2, 64), row_stride_bytes=129) + assert storage.address(1, 0) == (129, 0) + assert storage.byte_size == 257 + + def test_invalid_packing_and_overflow(): for bits, slots in [(0, None), (65, None), (6, 4)]: with pytest.raises(ValueError): From 40b9b825ab57695a79808a538eb68e7ec54aca8c Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:01:58 -0400 Subject: [PATCH 06/32] fix(rocke): unify E4M3 dtype and storage identity --- .../rocke/platform/cpp/core/dtypes.cpp | 27 ++++++++++++------ .../rocke/platform/cpp/core/ir/core_types.cpp | 5 +--- .../rocke/platform/cpp/helpers/mma_io.cpp | 3 +- .../rocke/platform/cpp/helpers/quant.cpp | 6 ++-- .../rocke/helper_rocke.helpers.quant.h | 2 +- .../rocke/platform/cpp/include/rocke/ir.h | 2 -- .../architecture/dtype_storage_packing.md | 12 ++++++-- .../platform/python/rocke/core/__init__.py | 2 -- .../platform/python/rocke/core/dtypes.py | 4 +-- .../rocke/platform/python/rocke/core/ir.py | 2 -- .../python/rocke/core/ir_serialize.py | 2 +- .../platform/python/rocke/helpers/mma_io.py | 2 +- .../platform/python/rocke/helpers/quant.py | 1 + .../rocke/platform/tests/core/test_dtypes.py | 28 ++++++++++++++++++- .../rocke/platform/tests/core/test_mma_io.py | 22 +++++++++++++-- .../platform/tests/core/test_storage.cpp | 21 +++++++++++++- 16 files changed, 105 insertions(+), 36 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/dtypes.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/dtypes.cpp index b5ab7d5507ae..dcc32275b741 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/dtypes.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/dtypes.cpp @@ -44,6 +44,7 @@ static const rocke_dtype_alias_t k_dtype_aliases[] = { {"int32", ROCKE_DTYPE_I32}, {"fp8", "fp8e4m3"}, {"fp8e4m3", "fp8e4m3"}, + {"e4m3", "fp8e4m3"}, {"bf8", "bf8e5m2"}, {"bf8e5m2", "bf8e5m2"}, {"fp6", "fp6e2m3"}, @@ -139,15 +140,23 @@ const char* rocke_normalize_dtype(const char* name, char* lowered, size_t cap) } static const rocke_dtype_info_t k_dtype_info[] = { - {"fp32", ROCKE_DTYPE_FLOAT, 32}, {"fp16", ROCKE_DTYPE_FLOAT, 16}, - {"bf16", ROCKE_DTYPE_FLOAT, 16}, {"fp8e4m3", ROCKE_DTYPE_FLOAT, 8}, - {"bf8e5m2", ROCKE_DTYPE_FLOAT, 8}, {"fp6e2m3", ROCKE_DTYPE_FLOAT, 6}, - {"fp6e3m2", ROCKE_DTYPE_FLOAT, 6}, {"fp4e2m1", ROCKE_DTYPE_FLOAT, 4}, - {"e8m0", ROCKE_DTYPE_FLOAT, 8}, {"e4m3", ROCKE_DTYPE_FLOAT, 8}, - {"e5m3", ROCKE_DTYPE_FLOAT, 8}, {"i4", ROCKE_DTYPE_INTEGER, 4}, - {"iu4", ROCKE_DTYPE_INTEGER, 4}, {"i8", ROCKE_DTYPE_INTEGER, 8}, - {"iu8", ROCKE_DTYPE_INTEGER, 8}, {"i16", ROCKE_DTYPE_INTEGER, 16}, - {"i32", ROCKE_DTYPE_INTEGER, 32}, {"i64", ROCKE_DTYPE_INTEGER, 64}, + {"fp32", ROCKE_DTYPE_FLOAT, 32}, + {"fp16", ROCKE_DTYPE_FLOAT, 16}, + {"bf16", ROCKE_DTYPE_FLOAT, 16}, + {"fp8e4m3", ROCKE_DTYPE_FLOAT, 8}, + {"bf8e5m2", ROCKE_DTYPE_FLOAT, 8}, + {"fp6e2m3", ROCKE_DTYPE_FLOAT, 6}, + {"fp6e3m2", ROCKE_DTYPE_FLOAT, 6}, + {"fp4e2m1", ROCKE_DTYPE_FLOAT, 4}, + {"e8m0", ROCKE_DTYPE_FLOAT, 8}, + {"e5m3", ROCKE_DTYPE_FLOAT, 8}, + {"i4", ROCKE_DTYPE_INTEGER, 4}, + {"iu4", ROCKE_DTYPE_INTEGER, 4}, + {"i8", ROCKE_DTYPE_INTEGER, 8}, + {"iu8", ROCKE_DTYPE_INTEGER, 8}, + {"i16", ROCKE_DTYPE_INTEGER, 16}, + {"i32", ROCKE_DTYPE_INTEGER, 32}, + {"i64", ROCKE_DTYPE_INTEGER, 64}, {"i1", ROCKE_DTYPE_PREDICATE, 1}, }; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp index d5e5bb48ee58..6062cadb30c0 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp @@ -60,7 +60,6 @@ ROCKE_SCALAR_SINGLETON(rocke_fp4e2m1, ROCKE_SCALAR_FP4E2M1, "fp4e2m1") ROCKE_SCALAR_SINGLETON(rocke_fp6e2m3, ROCKE_SCALAR_FP6E2M3, "fp6e2m3") ROCKE_SCALAR_SINGLETON(rocke_fp6e3m2, ROCKE_SCALAR_FP6E3M2, "fp6e3m2") ROCKE_SCALAR_SINGLETON(rocke_e8m0, ROCKE_SCALAR_E8M0, "e8m0") -ROCKE_SCALAR_SINGLETON(rocke_e4m3, ROCKE_SCALAR_E4M3, "e4m3") ROCKE_SCALAR_SINGLETON(rocke_e5m3, ROCKE_SCALAR_E5M3, "e5m3") #undef ROCKE_SCALAR_SINGLETON @@ -85,7 +84,7 @@ const rocke_type_t* rocke_scalar_by_name(const char* name) return rocke_f16(); if(strcmp(name, "f32") == 0) return rocke_f32(); - if(strcmp(name, "fp8e4m3") == 0) + if(strcmp(name, "fp8e4m3") == 0 || strcmp(name, "e4m3") == 0) return rocke_fp8e4m3(); if(strcmp(name, "bf8e5m2") == 0) return rocke_bf8e5m2(); @@ -97,8 +96,6 @@ const rocke_type_t* rocke_scalar_by_name(const char* name) return rocke_fp6e3m2(); if(strcmp(name, "e8m0") == 0) return rocke_e8m0(); - if(strcmp(name, "e4m3") == 0) - return rocke_e4m3(); if(strcmp(name, "e5m3") == 0) return rocke_e5m3(); return NULL; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp index c7d01678a04e..521ccfa0c2fc 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp @@ -13,8 +13,7 @@ const rocke_type_t* rocke_storage_ir_type(const char* dtype) const rocke_dtype_info_t* info = rocke_dtype_info(dtype); if(!info) return NULL; - if(info->encoded_bits % 8 || strcmp(info->name, "e8m0") == 0 || strcmp(info->name, "e4m3") == 0 - || strcmp(info->name, "e5m3") == 0) + if(info->encoded_bits % 8 || strcmp(info->name, "e8m0") == 0 || strcmp(info->name, "e5m3") == 0) return rocke_i8(); return rocke_dtype_to_ir_type(info->name); } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/quant.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/quant.cpp index fcb13a085464..89a7c90df38c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/quant.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/quant.cpp @@ -22,9 +22,9 @@ const rocke_type_t* rocke_quant_ir_type(const char* qdtype) { return rocke_dtype_to_ir_type("i8"); } - /* `_canon` -> "fp8e4m3" : keys "fp8e4m3", "fp8", "fp8_e4m3". */ + /* `_canon` -> "fp8e4m3" : keys "fp8e4m3", "fp8", "fp8_e4m3", "e4m3". */ if(strcmp(qdtype, "fp8e4m3") == 0 || strcmp(qdtype, "fp8") == 0 - || strcmp(qdtype, "fp8_e4m3") == 0) + || strcmp(qdtype, "fp8_e4m3") == 0 || strcmp(qdtype, "e4m3") == 0) { return rocke_dtype_to_ir_type("fp8e4m3"); } @@ -63,7 +63,7 @@ const rocke_type_t* rocke_b_quant_ir_type(rocke_ir_builder_t* b, const char* qdt b, ROCKE_ERR_VALUE, "unsupported quant dtype %s%s%s; expected one of " - "['bf8', 'bf8e5m2', 'fp8', 'fp8_e4m3', 'fp8_e5m2', 'fp8e4m3', " + "['bf8', 'bf8e5m2', 'e4m3', 'fp8', 'fp8_e4m3', 'fp8_e5m2', 'fp8e4m3', " "'i8', 'int8']", qdtype ? "'" : "", qdtype ? qdtype : "None", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.quant.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.quant.h index 2b24e6d2e172..12b21e1a2ee0 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.quant.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.quant.h @@ -22,7 +22,7 @@ extern "C" { * * Accepts (matching Python `_QDTYPE_ALIAS`): * "i8", "int8" -> rocke_i8() - * "fp8e4m3", "fp8", "fp8_e4m3" -> rocke_fp8e4m3() + * "fp8e4m3", "fp8", "fp8_e4m3", "e4m3" -> rocke_fp8e4m3() * "bf8e5m2", "bf8", "fp8_e5m2" -> rocke_bf8e5m2() * "fp4", "fp4e2m1" -> rocke_fp4e2m1() * "fp6", "fp6e2m3" -> rocke_fp6e2m3() diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h index 131ea637e341..a6fcf74b8ab5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h @@ -109,7 +109,6 @@ typedef enum rocke_scalar_kind ROCKE_SCALAR_FP6E2M3, ROCKE_SCALAR_FP6E3M2, ROCKE_SCALAR_E8M0, - ROCKE_SCALAR_E4M3, ROCKE_SCALAR_E5M3, ROCKE_SCALAR__COUNT } rocke_scalar_kind_t; @@ -584,7 +583,6 @@ const rocke_type_t* rocke_fp4e2m1(void); const rocke_type_t* rocke_fp6e2m3(void); const rocke_type_t* rocke_fp6e3m2(void); const rocke_type_t* rocke_e8m0(void); -const rocke_type_t* rocke_e4m3(void); const rocke_type_t* rocke_e5m3(void); /* Logical dtype resolver; NULL for unknown or unrepresented encodings. */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md index ecab80f8458e..0d68276de961 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md @@ -22,15 +22,21 @@ selectors. An integer carrier does not identify the numerical format. [`dtype_info`](../../python/rocke/core/dtypes.py) and [`rocke_dtype_info`](../../cpp/include/rocke/dtypes.h) return target-independent encoding descriptors. `dtype_to_ir_type` and `rocke_dtype_to_ir_type` resolve -logical types, including distinct FP4 E2M1, FP6 E2M3, FP6 E3M2, and scale types. +logical types, including distinct FP4 E2M1, FP6 E2M3, FP6 E3M2, E8M0, and E5M3. `quant_ir_type` delegates type resolution to this common resolver. Recognition and serialization do not enable scalar conversion, arithmetic, or -an instruction on a target. In this implementation, FP4/FP6 and the nominal scale -types are used in authoring metadata; storage helpers expand their transport into +an instruction on a target. In this implementation, FP4/FP6, E8M0, and E5M3 +are used in authoring metadata; storage helpers expand their transport into existing integer operations before serialization. Raw scalar lowering of these types remains unsupported. Existing FP16/BF16 and FP8/BF8 lowering is unchanged. +`e4m3` is an alias for the canonical `fp8e4m3` encoding, logical IR type, and +nominal FP8 storage pointer. Both spellings serialize as `fp8e4m3`. The +architecture scale selector `MmaScaleDType.E4M3` describes its operand role; it +does not introduce a second scalar type. Scale words can still be assembled +from raw byte patterns with the common bit packer. + Scale-format validation remains separate from matrix alias normalization: E5M3 is not BF8 E5M2. Registering a scale dtype does not enable a backend selector. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/__init__.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/__init__.py index 57596e6ba255..c6b12bbdb49e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/__init__.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/__init__.py @@ -55,7 +55,6 @@ FP6E2M3, FP6E3M2, E8M0, - E4M3, E5M3, dtype_to_ir_type, I1, @@ -111,7 +110,6 @@ "FP6E2M3", "FP6E3M2", "E8M0", - "E4M3", "E5M3", "CodegenPolicy", "SchedulerStrategy", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/dtypes.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/dtypes.py index 3808f87f3ad0..b0745c226610 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/dtypes.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/dtypes.py @@ -25,6 +25,7 @@ "fp32": "fp32", "fp8": "fp8e4m3", "fp8e4m3": "fp8e4m3", + "e4m3": "fp8e4m3", "bf8": "bf8e5m2", "bf8e5m2": "bf8e5m2", "fp6": "fp6e2m3", @@ -83,7 +84,6 @@ class DTypeInfo: ("fp6e3m2", 6), ("fp4e2m1", 4), ("e8m0", 8), - ("e4m3", 8), ("e5m3", 8), ), ), @@ -108,7 +108,7 @@ class DTypeInfo: def dtype_info(name: str) -> DTypeInfo: """Resolve an encoding; recognition alone does not enable an operation. - Scale names remain distinct keys. In particular e5m3 is not bf8e5m2. + E4M3 shares the fp8e4m3 encoding; e5m3 remains distinct from bf8e5m2. Operand-role validation belongs to the selected atom. """ key = normalize_dtype(name) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py index 6dc0175c467d..73f01b156cfc 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py @@ -57,7 +57,6 @@ def __repr__(self) -> str: FP6E2M3 = Type("fp6e2m3") FP6E3M2 = Type("fp6e3m2") E8M0 = Type("e8m0") -E4M3 = Type("e4m3") E5M3 = Type("e5m3") @@ -82,7 +81,6 @@ def dtype_to_ir_type(dtype: str) -> Type: "fp6e2m3": FP6E2M3, "fp6e3m2": FP6E3M2, "e8m0": E8M0, - "e4m3": E4M3, "e5m3": E5M3, } if info.name not in types: diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir_serialize.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir_serialize.py index 18c9c5cdbc5a..542a14d5c580 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir_serialize.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir_serialize.py @@ -136,7 +136,7 @@ def _parse_type(s: str) -> Type: shape_str = inner[lb + 1 : rb].strip() shape = tuple(int(x) for x in shape_str.split("x")) if shape_str else () return SmemType(elem, shape) - return Type(s) + return Type("fp8e4m3" if s == "e4m3" else s) # -------------------------------------------------------------------------- diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py index 38bcbe29e7e1..1e0745da422a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py @@ -25,7 +25,7 @@ def storage_ir_type(dtype: str) -> Type: """Addressable storage unit, distinct from the logical dtype resolver.""" info = dtype_info(dtype) - if info.encoded_bits % 8 or info.name in ("e8m0", "e4m3", "e5m3"): + if info.encoded_bits % 8 or info.name in ("e8m0", "e5m3"): return I8 return dtype_to_ir_type(info.name) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py index 1ae76ffb0d33..07e7cc1158a9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py @@ -90,6 +90,7 @@ "fp8e4m3": "fp8e4m3", "fp8": "fp8e4m3", "fp8_e4m3": "fp8e4m3", + "e4m3": "fp8e4m3", "bf8e5m2": "bf8e5m2", "bf8": "bf8e5m2", "fp8_e5m2": "bf8e5m2", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py index cee48c279c6e..ca6de16b1fe8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py @@ -6,7 +6,10 @@ from rocke.core.dtypes import normalize_dtype from rocke.core.dtypes import dtype_info -from rocke.core.ir import I8, IRBuilder, dtype_to_ir_type +from rocke.core.arch.target import MmaScaleDType +from rocke.core.ir import FP8E4M3, I8, IRBuilder, dtype_to_ir_type +from rocke.core.storage import TensorStorage +from rocke.helpers.mma_io import storage_ir_type from rocke.core.ir_serialize import parse, serialize from rocke.helpers.quant import quant_ir_type, ir_to_qdtype, dequantize_scalar_to_f32 @@ -63,6 +66,7 @@ def test_unknown_dtype_and_unrepresented_integer_family(): ("bfloat16", "bf16"), (" Float\t", "fp32"), ("FP8", "fp8e4m3"), + (" E4M3 ", "fp8e4m3"), ("BF8", "bf8e5m2"), ("FP6", "fp6e2m3"), ("BF6", "fp6e3m2"), @@ -84,3 +88,25 @@ def test_architecture_entry_points_reexport_core_normalization(): assert normalize_dtype.__module__ == "rocke.core.dtypes" assert core_normalize is arch_normalize is target_normalize is normalize_dtype + + +@pytest.mark.parametrize("spelling", ["e4m3", "fp8e4m3", "fp8", MmaScaleDType.E4M3]) +def test_e4m3_shares_encoding_type_and_storage(spelling): + assert dtype_info(spelling) is dtype_info("fp8e4m3") + assert dtype_to_ir_type(spelling) is FP8E4M3 + assert storage_ir_type(spelling) is FP8E4M3 + assert TensorStorage(spelling, (2, 128)) == TensorStorage("fp8e4m3", (2, 128)) + assert quant_ir_type(spelling) is FP8E4M3 + b = IRBuilder("e4m3_conversion") + value = b.param("value", dtype_to_ir_type(spelling)) + b.cvt_fp8_to_f32(value) + assert serialize(b.kernel) == serialize(parse(serialize(b.kernel))) + + +def test_scale_role_remains_independent_of_encoding_aliases(): + assert MmaScaleDType("fp8e4m3") is MmaScaleDType.E4M3 + assert str(MmaScaleDType.E4M3) == "e4m3" + assert dtype_info("e5m3") is not dtype_info("bf8e5m2") + assert dtype_to_ir_type("e5m3") != dtype_to_ir_type("bf8e5m2") + with pytest.raises(ValueError): + MmaScaleDType("bf8e5m2") diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py index 190beabfe0e9..e1e1e3d74ece 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py @@ -11,8 +11,8 @@ import pytest from rocke.core.arch.wmma_scale import scaled_matrix_layout -from rocke.core.ir import I8, I32, I64, IRBuilder, PtrType -from rocke.core.ir_serialize import serialize +from rocke.core.ir import FP8E4M3, I8, I32, I64, IRBuilder, PtrType, VectorType +from rocke.core.ir_serialize import parse, serialize from rocke.core.lower_llvm import lower_kernel_to_llvm from rocke.core.lower_hip import lower_kernel_to_hip from rocke.core.storage import ( @@ -34,6 +34,7 @@ "fp6_padded", "bf6", "fp8", + "e4m3", "bf8", "f16", "bf16", @@ -244,3 +245,20 @@ def test_hip_rejects_invalid_vector_load_alignment(alignment): value.op.attrs["align"] = alignment with pytest.raises(ValueError, match="alignment must be a positive power of two"): lower_kernel_to_hip(b.kernel, arch="gfx1250") + + +def test_e4m3_alias_serialization_matches_native(): + executable = os.environ.get("ROCKE_STORAGE_TEST") + if not executable: + pytest.skip("set ROCKE_STORAGE_TEST to the built native storage test") + b = IRBuilder("canonical_e4m3") + b.param("scalar", FP8E4M3) + b.param("pointer", PtrType(FP8E4M3, "global")) + b.param("vector", VectorType(FP8E4M3, 4)) + canonical = serialize(b.kernel) + alias = canonical.replace("fp8e4m3", "e4m3") + assert serialize(parse(alias)) == canonical + result = subprocess.run( + [executable, "--parse", alias], check=True, capture_output=True, text=True + ) + assert result.stdout == canonical diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp index cd210412cf9b..0d4ce1fcf0f4 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp @@ -173,7 +173,7 @@ static int test_storage_validation() CHECK(!rocke_tensor_storage_bytes(&invalid, &bytes)); } } - for(const char* dtype : {"f16", "bf16", "fp8", "bf8"}) + for(const char* dtype : {"f16", "bf16", "fp8", "e4m3", "bf8"}) { const auto* unit = rocke_storage_ir_type(dtype); CHECK(unit && !rocke_type_eq(unit, rocke_i8())); @@ -218,7 +218,26 @@ int main(int argc, char** argv) return emit(argv[2], false); if(argc == 3 && strcmp(argv[1], "--hip") == 0) return emit(argv[2], true); + if(argc == 3 && strcmp(argv[1], "--parse") == 0) + { + rocke_ir_builder_t b; + CHECK(rocke_ir_builder_init(&b, "parse") == ROCKE_OK); + rocke_kernel_def_t* kernel = NULL; + CHECK(rocke_ir_parse(argv[2], &b, &kernel) == ROCKE_OK); + char* text = NULL; + CHECK(rocke_ir_serialize(kernel, &text) == ROCKE_OK); + fputs(text, stdout); + free(text); + rocke_ir_builder_free(&b); + return 0; + } CHECK(test_storage_validation() == 0); + CHECK(rocke_dtype_info("e4m3") == rocke_dtype_info("fp8e4m3")); + CHECK(rocke_dtype_to_ir_type("e4m3") == rocke_fp8e4m3()); + CHECK(rocke_storage_ir_type("e4m3") == rocke_fp8e4m3()); + CHECK(rocke_quant_ir_type("e4m3") == rocke_fp8e4m3()); + CHECK(rocke_scalar_by_name("e4m3") == rocke_fp8e4m3()); + CHECK(rocke_dtype_info("e5m3") != rocke_dtype_info("bf8e5m2")); for(const char* elem_type : {"unknown", "fp4e2m1"}) { rocke_ir_builder_t b; From d21f2833df3b17e72fc77ce8c12399b178f4ab8d Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Thu, 24 Sep 2026 16:03:26 -0400 Subject: [PATCH 07/32] fix(rocke): wire native parity into the standard runner Report complete quant type aliases while preserving scalar quantization restrictions. --- .../rocke/platform/cpp/helpers/quant.cpp | 8 +- .../platform/python/rocke/helpers/quant.py | 21 +++- .../rocke/platform/tests/README.md | 13 ++ .../rocke/platform/tests/core/test_dtypes.py | 43 ++++++- .../rocke/platform/tests/core/test_mma_io.py | 17 +++ .../platform/tests/core/test_storage.cpp | 18 +++ .../rocke/platform/tests/run_all.py | 75 ++++++++++- .../platform/tests/tools/test_run_all.py | 118 ++++++++++++++++++ 8 files changed, 298 insertions(+), 15 deletions(-) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/quant.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/quant.cpp index 89a7c90df38c..e7ff6241752c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/quant.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/quant.cpp @@ -55,16 +55,16 @@ const rocke_type_t* rocke_b_quant_ir_type(rocke_ir_builder_t* b, const char* qdt ty = rocke_quant_ir_type(qdtype); if(ty == NULL) { - /* Mirror the Python `_canon` ValueError, including the {qdtype!r} + /* Mirror the Python `quant_ir_type` ValueError, including the {qdtype!r} * single-quote repr for the (non-NULL) string case and the sorted - * key list `sorted(_QDTYPE_ALIAS)`. NULL is reported as "None" to + * key list `sorted(_QUANT_TYPE_ALIAS)`. NULL is reported as "None" to * match Python's repr(None). */ return (const rocke_type_t*)rocke_i_set_err( b, ROCKE_ERR_VALUE, "unsupported quant dtype %s%s%s; expected one of " - "['bf8', 'bf8e5m2', 'e4m3', 'fp8', 'fp8_e4m3', 'fp8_e5m2', 'fp8e4m3', " - "'i8', 'int8']", + "['bf6', 'bf8', 'bf8e5m2', 'e4m3', 'fp4', 'fp4e2m1', 'fp6', 'fp6e2m3', " + "'fp6e3m2', 'fp8', 'fp8_e4m3', 'fp8_e5m2', 'fp8e4m3', 'i8', 'int8']", qdtype ? "'" : "", qdtype ? qdtype : "None", qdtype ? "'" : ""); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py index 07e7cc1158a9..eb4604c48809 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py @@ -97,6 +97,17 @@ } +# Type recognition includes formats without scalar quantization support. +_QUANT_TYPE_ALIAS = _QDTYPE_ALIAS | { + "fp4": "fp4e2m1", + "fp4e2m1": "fp4e2m1", + "fp6": "fp6e2m3", + "fp6e2m3": "fp6e2m3", + "bf6": "fp6e3m2", + "fp6e3m2": "fp6e3m2", +} + + _IR_TO_QDTYPE = { "i8": "i8", "fp8e4m3": "fp8e4m3", @@ -123,10 +134,12 @@ def quant_ir_type(qdtype: str) -> Type: the common dtype resolver. Type recognition does not enable scalar quantization or dequantization for the low-bit formats. """ - # Type recognition includes low-bit formats before scalar conversions exist. - if qdtype in ("fp4", "fp4e2m1", "fp6", "fp6e2m3", "bf6", "fp6e3m2"): - return dtype_to_ir_type(qdtype) - return dtype_to_ir_type(_canon(qdtype)) + if qdtype not in _QUANT_TYPE_ALIAS: + raise ValueError( + f"unsupported quant dtype {qdtype!r}; expected one of " + f"{sorted(_QUANT_TYPE_ALIAS)}" + ) + return dtype_to_ir_type(_QUANT_TYPE_ALIAS[qdtype]) def quant_max_abs(qdtype: str) -> float: diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md b/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md index 28c98405ffe1..38da77e7b588 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md @@ -104,3 +104,16 @@ divergence - the Python builder correctly rejects wave32 WMMA on gfx942.) - EXCLUDED from rocKE: `test_gen_instances.py` (imports `ck4inductor`, a separate package) and `test_rocke_examples.py` (drives the external `example/ck_tile/dsl` tree, not part of rocKE) stay in `composablekernel/python/test`. + +### Native storage parity in the standard runner + +`run_all.py --build-root ` builds the `rocke_storage` target in a configured +build and obtains its executable path from CTest. Both pytest passes receive that +path, so storage IR/HIP parity and serialization tests run automatically. +`--config` selects the native test configuration (default `Release`). A build or +fixture-discovery failure stops the runner instead of silently skipping coverage. + +An explicit `ROCKE_STORAGE_TEST` overrides discovery and must name an existing +executable. With no configured build or explicit override, the runner reports +native storage parity as skipped; direct pytest invocations can use the same +override. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py index ca6de16b1fe8..2222d6ed4727 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py @@ -2,6 +2,8 @@ # SPDX-License-Identifier: MIT """Core dtype names are independent of target support and retain public aliases.""" +import ast + import pytest from rocke.core.dtypes import normalize_dtype @@ -11,7 +13,12 @@ from rocke.core.storage import TensorStorage from rocke.helpers.mma_io import storage_ir_type from rocke.core.ir_serialize import parse, serialize -from rocke.helpers.quant import quant_ir_type, ir_to_qdtype, dequantize_scalar_to_f32 +from rocke.helpers.quant import ( + quant_ir_type, + ir_to_qdtype, + dequantize_scalar_to_f32, + quant_max_abs, +) @pytest.mark.parametrize( @@ -110,3 +117,37 @@ def test_scale_role_remains_independent_of_encoding_aliases(): assert dtype_to_ir_type("e5m3") != dtype_to_ir_type("bf8e5m2") with pytest.raises(ValueError): MmaScaleDType("bf8e5m2") + + +def test_quant_type_diagnostic_lists_every_accepted_spelling(): + expected = { + "i8", + "int8", + "fp8", + "fp8e4m3", + "fp8_e4m3", + "e4m3", + "bf8", + "bf8e5m2", + "fp8_e5m2", + "fp4", + "fp4e2m1", + "fp6", + "fp6e2m3", + "bf6", + "fp6e3m2", + } + with pytest.raises(ValueError) as error: + quant_ir_type("invalid") + accepted = ast.literal_eval(str(error.value).split("expected one of ")[1]) + assert accepted == sorted(expected) + assert all(quant_ir_type(name) is not None for name in accepted) + + +@pytest.mark.parametrize( + "dtype", ["fp4", "fp4e2m1", "fp6", "fp6e2m3", "bf6", "fp6e3m2"] +) +def test_type_alias_does_not_enable_scalar_quantization(dtype): + assert quant_ir_type(dtype) is not None + with pytest.raises(ValueError, match="unsupported quant dtype"): + quant_max_abs(dtype) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py index e1e1e3d74ece..558b3ff0ddae 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py @@ -262,3 +262,20 @@ def test_e4m3_alias_serialization_matches_native(): [executable, "--parse", alias], check=True, capture_output=True, text=True ) assert result.stdout == canonical + + +def test_quant_type_diagnostic_matches_native(): + from rocke.helpers.quant import quant_ir_type + + executable = os.environ.get("ROCKE_STORAGE_TEST") + if not executable: + pytest.skip("set ROCKE_STORAGE_TEST to the built native storage test") + with pytest.raises(ValueError) as error: + quant_ir_type("invalid") + result = subprocess.run( + [executable, "--quant-error", "invalid"], + check=True, + capture_output=True, + text=True, + ) + assert result.stdout == str(error.value) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp index 0d4ce1fcf0f4..edc54e0e51d7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp @@ -8,6 +8,7 @@ #include #include +#include "rocke/error.hpp" #include "rocke/helper_rocke.helpers.mma_io.h" #include "rocke/helper_rocke.helpers.quant.h" #include "rocke/ir_serialize.h" @@ -231,6 +232,23 @@ int main(int argc, char** argv) rocke_ir_builder_free(&b); return 0; } + if(argc == 3 && strcmp(argv[1], "--quant-error") == 0) + { + rocke_ir_builder_t b; + CHECK(rocke_ir_builder_init(&b, "quant_error") == ROCKE_OK); + try + { + rocke_b_quant_ir_type(&b, argv[2]); + CHECK(false); + } + catch(const ckc::Error& error) + { + CHECK(error.code() == ROCKE_ERR_VALUE); + fputs(error.what(), stdout); + } + rocke_ir_builder_free(&b); + return 0; + } CHECK(test_storage_validation() == 0); CHECK(rocke_dtype_info("e4m3") == rocke_dtype_info("fp8e4m3")); CHECK(rocke_dtype_to_ir_type("e4m3") == rocke_fp8e4m3()); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py index a888b1b7ffb8..7f60753b8175 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py @@ -11,11 +11,12 @@ # # Usage: # python rocke/platform/tests/run_all.py [--no-guard] [--no-gate] [--no-pytest] -# [--no-both] [--only SUBSTR] [--build-root DIR] +# [--no-both] [--only SUBSTR] [--build-root DIR] [--config CONFIG] from __future__ import annotations import argparse +import json import os import re import subprocess @@ -86,7 +87,59 @@ def relative_path_guard() -> int: return 0 -def differential_pytest_pass() -> int: +def native_pytest_env(build_root: Path, config: str) -> dict[str, str]: + """Make the native storage comparisons available to both pytest passes.""" + env = dict(os.environ) + executable = env.get("ROCKE_STORAGE_TEST") + if not executable and (build_root / "CMakeCache.txt").is_file(): + subprocess.run( + [ + "cmake", + "--build", + str(build_root), + "--config", + config, + "--target", + "rocke_storage", + ], + check=True, + ) + listing = subprocess.run( + [ + "ctest", + "--test-dir", + str(build_root), + "-C", + config, + "--show-only=json-v1", + "-R", + "^rocke_storage$", + ], + check=True, + capture_output=True, + text=True, + ) + tests = json.loads(listing.stdout)["tests"] + commands = [ + test.get("command", []) for test in tests if test["name"] == "rocke_storage" + ] + if len(commands) != 1 or not commands[0]: + raise ValueError("CTest did not resolve the built rocke_storage executable") + executable = commands[0][0] + if executable: + path = Path(executable).resolve() + if not path.is_file(): + raise ValueError(f"native storage test executable does not exist: {path}") + env["ROCKE_STORAGE_TEST"] = str(path) + print(f"\n== native storage parity: {path} ==") + else: + print( + "\n== native storage parity: SKIPPED (no configured build or ROCKE_STORAGE_TEST) ==" + ) + return env + + +def differential_pytest_pass(env: dict[str, str]) -> int: """Re-run pytest with ``ROCKE_BACKEND=both`` (the cross-engine gate). The default pass exercises one engine per assertion, so two engines that @@ -110,6 +163,7 @@ def differential_pytest_pass() -> int: [sys.executable, "-c", "import rocke_engine"], capture_output=True, cwd=str(TESTS), + env=env, ) if probe.returncode != 0: print( @@ -133,7 +187,7 @@ def differential_pytest_pass() -> int: "either compared byte-for-byte or fails; remaining skips are " "environmental (torch / GPU)" ) - env = dict(os.environ, ROCKE_BACKEND="both") + env = dict(env, ROCKE_BACKEND="both") return subprocess.run( [sys.executable, "-m", "pytest", str(TESTS), "-rs"], cwd=str(TESTS), env=env ).returncode @@ -157,6 +211,9 @@ def main() -> int: ap.add_argument( "--build-root", default=str(Path(tempfile.gettempdir()) / "rocke_verify") ) + ap.add_argument( + "--config", default="Release", help="native test build/CTest configuration" + ) args = ap.parse_args() status = 0 @@ -177,13 +234,18 @@ def main() -> int: status |= subprocess.run(gate).returncode if not args.no_pytest: + try: + pytest_env = native_pytest_env(Path(args.build_root).resolve(), args.config) + except (OSError, ValueError, subprocess.CalledProcessError) as exc: + print(f"native storage parity setup failed: {exc}", file=sys.stderr) + return 1 print("\n== pytest ==") status |= subprocess.run( - [sys.executable, "-m", "pytest", str(TESTS)], cwd=str(TESTS) + [sys.executable, "-m", "pytest", str(TESTS)], cwd=str(TESTS), env=pytest_env ).returncode if not args.no_pytest and not args.no_both: - status |= differential_pytest_pass() + status |= differential_pytest_pass(pytest_env) build_root = Path(args.build_root) # Only ctest when the CTest-registered binaries were actually built (the @@ -200,7 +262,8 @@ def main() -> int: ): print("\n== ctest ==") status |= subprocess.run( - ["ctest", "--output-on-failure", "--no-tests=ignore"], cwd=str(build_root) + ["ctest", "-C", args.config, "--output-on-failure", "--no-tests=ignore"], + cwd=str(build_root), ).returncode print("\nRESULT:", "GREEN" if status == 0 else "RED") diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py new file mode 100644 index 000000000000..28e06b7b5c46 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py @@ -0,0 +1,118 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""The standard runner must supply native fixture coverage to both pytest passes.""" + +import importlib.util +import json +from pathlib import Path +import subprocess + +import pytest + + +@pytest.fixture +def runner(monkeypatch): + path = Path(__file__).resolve().parents[1] / "run_all.py" + spec = importlib.util.spec_from_file_location("rocke_run_all", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.delenv("ROCKE_STORAGE_TEST", raising=False) + monkeypatch.delenv("ROCKE_BACKEND", raising=False) + return module + + +@pytest.mark.parametrize( + "filename", ["rocke_storage", "provider_rocke_storage_test.exe"] +) +def test_runner_passes_registered_fixture_to_both_backends( + runner, monkeypatch, tmp_path, filename +): + (tmp_path / "CMakeCache.txt").touch() + executable = tmp_path / "Debug" / filename + executable.parent.mkdir() + executable.touch() + calls = [] + + def run(command, **kwargs): + calls.append((command, kwargs)) + listing = {"tests": [{"name": "rocke_storage", "command": [str(executable)]}]} + return subprocess.CompletedProcess(command, 0, stdout=json.dumps(listing)) + + monkeypatch.setattr(runner.subprocess, "run", run) + monkeypatch.setattr( + runner.sys, + "argv", + [ + "run_all.py", + "--no-guard", + "--no-gate", + "--build-root", + str(tmp_path), + "--config", + "Debug", + ], + ) + assert runner.main() == 0 + children = [ + (cmd, kw["env"]) for cmd, kw in calls if cmd[0] == runner.sys.executable + ] + assert len(children) == 3 # default pytest, extension import probe, both pytest + assert all(env["ROCKE_STORAGE_TEST"] == str(executable) for _, env in children) + assert "ROCKE_BACKEND" not in children[0][1] + assert children[-1][1]["ROCKE_BACKEND"] == "both" + assert calls[0][0] == [ + "cmake", + "--build", + str(tmp_path), + "--config", + "Debug", + "--target", + "rocke_storage", + ] + + +def test_explicit_fixture_override_is_resolved_before_pytest_changes_directory( + runner, monkeypatch, tmp_path +): + executable = tmp_path / "fixture" + executable.touch() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("ROCKE_STORAGE_TEST", "fixture") + env = runner.native_pytest_env(tmp_path, "Release") + assert env["ROCKE_STORAGE_TEST"] == str(executable) + + +def test_missing_build_reports_native_coverage_skip(runner, tmp_path, capsys): + assert "ROCKE_STORAGE_TEST" not in runner.native_pytest_env(tmp_path, "Release") + assert "native storage parity: SKIPPED" in capsys.readouterr().out + + +def test_invalid_override_is_an_error(runner, monkeypatch, tmp_path): + monkeypatch.setenv("ROCKE_STORAGE_TEST", str(tmp_path / "missing")) + with pytest.raises(ValueError, match="does not exist"): + runner.native_pytest_env(tmp_path, "Release") + + +@pytest.mark.parametrize("failure", ["build", "registration", "executable"]) +def test_native_setup_failure_prevents_silently_skipped_pytest( + runner, monkeypatch, tmp_path, failure +): + (tmp_path / "CMakeCache.txt").touch() + calls = [] + + def run(command, **kwargs): + calls.append(command) + if command[0] == "cmake" and failure == "build": + raise subprocess.CalledProcessError(1, command) + command_path = [str(tmp_path / "missing")] if failure == "executable" else [] + listing = {"tests": [{"name": "rocke_storage", "command": command_path}]} + return subprocess.CompletedProcess(command, 0, stdout=json.dumps(listing)) + + monkeypatch.setattr(runner.subprocess, "run", run) + monkeypatch.setattr( + runner.sys, + "argv", + ["run_all.py", "--no-guard", "--no-gate", "--build-root", str(tmp_path)], + ) + assert runner.main() == 1 + assert all(command[0] != runner.sys.executable for command in calls) From 1956a3402e58f2aebb2ee06ffd4d35ec7dc5bc1d Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Thu, 24 Sep 2026 17:54:50 -0400 Subject: [PATCH 08/32] fix(rocke): reject empty storage in fragment loaders Keep empty descriptors valid while rejecting fragment loads before IR emission in both engines. Cover empty shapes and nonempty controls in Python and native tests. --- .../rocke/platform/cpp/helpers/mma_io.cpp | 2 + .../platform/python/rocke/helpers/mma_io.py | 2 + .../rocke/platform/tests/core/test_mma_io.py | 39 +++++++++++++++ .../platform/tests/core/test_storage.cpp | 49 +++++++++++++++++++ 4 files changed, 92 insertions(+) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp index 521ccfa0c2fc..d34f32a589b9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp @@ -47,6 +47,8 @@ rocke_value_t* rocke_h_load_matrix_fragment(rocke_ir_builder_t* b, uint64_t storage_bytes; if(!rocke_tensor_storage_bytes(storage, &storage_bytes)) ckc::raise_status(ROCKE_ERR_VALUE, "invalid tensor storage descriptor"); + if(storage_bytes == 0) + ckc::raise_status(ROCKE_ERR_VALUE, "matrix fragment requires nonempty tensor storage"); const auto& packing = layout->fragment; rocke_matrix_fragment_layout_t checked; if(!rocke_matrix_fragment_layout_init(&checked, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py index 1e0745da422a..15e7893550cc 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py @@ -47,6 +47,8 @@ def load_matrix_fragment( lane_group is derived from the atom's lane mapping by the caller. Partial chunks and nonzero bit origins reject. """ + if storage.byte_size == 0: + raise ValueError("matrix fragment requires nonempty tensor storage") packing = layout.fragment if not (0 < packing.count <= 0x7FFFFFFF and packing.carrier_count <= 0x7FFFFFFF): raise ValueError("invalid matrix fragment chunk layout") diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py index 558b3ff0ddae..caa74ce0cc67 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py @@ -150,6 +150,45 @@ def test_reject_partial_fragment_and_wrong_pointer(): ) +@pytest.mark.parametrize("dtype", ["fp4", "fp6", "fp8", "f16", "bf16"]) +@pytest.mark.parametrize("shape", [(0, 128), (16, 0), (0, 0), (16, 128)]) +def test_fragment_load_requires_nonempty_storage(dtype, shape): + storage = TensorStorage(dtype, shape) + typed = dtype in ("f16", "bf16") + unit = storage_ir_type(dtype) + layout = ( + MatrixFragmentLayout(FragmentPacking(BitPacking(16), 32, 16, 32), 16, 2, 16) + if typed + else scaled_matrix_layout(dtype, 16) + ) + b = IRBuilder("empty_storage") + ptr = b.param("A", PtrType(unit, "global")) + zero = b.const_i32(0) + before = serialize(b.kernel) + + def load(): + return load_matrix_fragment( + b, + ptr, + zero, + zero, + 0, + storage=storage, + layout=layout, + carrier_type=unit if typed else I32, + ) + + if 0 in shape: + assert storage.byte_size == 0 + with pytest.raises(ValueError, match="nonempty tensor storage"): + load() + assert serialize(b.kernel) == before + else: + assert storage.byte_size > 0 + assert load() is not None + assert "memref.global_load" in serialize(b.kernel) + + def test_hip_declares_only_encountered_missing_vector_widths(): source = lower_kernel_to_hip(build_transport("fp6"), arch="gfx1250") for name in ("i8x24", "i8x40", "i32x12"): diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp index edc54e0e51d7..4114583b8320 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp @@ -213,6 +213,54 @@ static int test_storage_validation() return 0; } +static int test_empty_storage_load() +{ + for(const char* dtype : {"fp4", "fp6", "fp8", "f16", "bf16"}) + { + const auto* unit = rocke_storage_ir_type(dtype); + const bool typed = strcmp(dtype, "f16") == 0 || strcmp(dtype, "bf16") == 0; + for(int shape = 0; shape < 4; ++shape) + { + const uint64_t rows = shape == 0 || shape == 2 ? 0 : 16; + const uint64_t cols = shape == 1 || shape == 2 ? 0 : 128; + rocke_tensor_storage_t storage; + CHECK(rocke_tensor_storage_init(&storage, dtype, rows, cols, UINT64_MAX, 0, 0, 1)); + uint64_t bytes; + CHECK(rocke_tensor_storage_bytes(&storage, &bytes)); + CHECK((bytes == 0) == (shape != 3)); + rocke_matrix_fragment_layout_t layout; + if(typed) + { + rocke_fragment_packing_t fragment; + CHECK(rocke_fragment_packing_init(&fragment, &storage.packing, 32, 16, 32)); + CHECK(rocke_matrix_fragment_layout_init(&layout, &fragment, 16, 2, 16)); + } + else + layout = rocke_scaled_matrix_layout(dtype, 16); + rocke_ir_builder_t b; + CHECK(rocke_ir_builder_init(&b, "empty_storage") == ROCKE_OK); + auto* ptr = rocke_b_param(&b, "A", rocke_ptr_type(&b, unit, "global"), NULL); + auto* zero = rocke_b_const_i32(&b, 0); + const int before = b.kernel->body->num_ops; + auto* value = rocke_h_load_matrix_fragment( + &b, ptr, zero, zero, 0, &storage, &layout, typed ? unit : rocke_i32()); + if(bytes == 0) + { + CHECK(!value && rocke_ir_builder_status(&b) == ROCKE_ERR_VALUE); + CHECK(strstr(rocke_ir_builder_error(&b), "nonempty tensor storage")); + CHECK(b.kernel->body->num_ops == before); + } + else + { + CHECK(value && rocke_ir_builder_ok(&b)); + CHECK(b.kernel->body->num_ops > before); + } + rocke_ir_builder_free(&b); + } + } + return 0; +} + int main(int argc, char** argv) { if(argc == 3 && strcmp(argv[1], "--emit") == 0) @@ -250,6 +298,7 @@ int main(int argc, char** argv) return 0; } CHECK(test_storage_validation() == 0); + CHECK(test_empty_storage_load() == 0); CHECK(rocke_dtype_info("e4m3") == rocke_dtype_info("fp8e4m3")); CHECK(rocke_dtype_to_ir_type("e4m3") == rocke_fp8e4m3()); CHECK(rocke_storage_ir_type("e4m3") == rocke_fp8e4m3()); From 699e6b8b34b7e09971f4c28d37970c2d302c7c27 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Thu, 24 Sep 2026 18:47:22 -0400 Subject: [PATCH 09/32] fix(rocke): bound fragment offsets and discover configured CTests Validate complete static fragment displacements before emitting i32 offsets in both engines. Resolve CTest readiness from registered commands and cover multi-config execution and failure propagation. --- .../rocke/platform/cpp/helpers/mma_io.cpp | 7 +- .../platform/python/rocke/helpers/mma_io.py | 3 + .../rocke/platform/tests/README.md | 8 ++- .../rocke/platform/tests/core/test_mma_io.py | 41 +++++++++++ .../platform/tests/core/test_storage.cpp | 56 +++++++++++++++ .../rocke/platform/tests/run_all.py | 65 +++++++++-------- .../platform/tests/tools/test_run_all.py | 72 +++++++++++++++++++ 7 files changed, 220 insertions(+), 32 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp index d34f32a589b9..c4baddbb2557 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp @@ -91,7 +91,12 @@ rocke_value_t* rocke_h_load_matrix_fragment(rocke_ir_builder_t* b, "padded matrix fragments currently require i32 carriers"); const uint64_t chunk_units = chunk_bytes / unit_bytes; const uint64_t origin_bytes = origin_bits / 8; - if(origin_bytes / unit_bytes > INT_MAX || chunk_units * layout->lane_groups > INT_MAX) + // Bound the last loaded unit without overflowing intermediate products. + if(origin_bytes / unit_bytes > INT_MAX + || chunk_units > uint64_t(INT_MAX) / layout->lane_groups + || packing.count / layout->chunk_elements + > (uint64_t(INT_MAX) + 1 - origin_bytes / unit_bytes) / chunk_units + / layout->lane_groups) ckc::raise_status(ROCKE_ERR_VALUE, "matrix fragment offset exceeds i32 range"); int alignment = gcd(gcd(storage->alignment_bytes, storage->row_stride_bytes), gcd(chunk_bytes, origin_bytes)); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py index 15e7893550cc..b47f120a0beb 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py @@ -76,9 +76,12 @@ def load_matrix_fragment( raise ValueError("matrix fragment payload must occupy whole carriers") chunk_units = layout.chunk_bytes // unit_bytes origin_bytes = origin_bits // 8 + # Bound the last loaded unit across all chunks and lane groups, including k0. if ( origin_bytes // unit_bytes > 0x7FFFFFFF or chunk_units * layout.lane_groups > 0x7FFFFFFF + or layout.chunks_per_lane + > (0x80000000 - origin_bytes // unit_bytes) // chunk_units // layout.lane_groups ): raise ValueError("matrix fragment offset exceeds i32 range") alignment = gcd( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md b/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md index 38da77e7b588..870e008c00f7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md @@ -45,9 +45,11 @@ Four distinct things run here; don't conflate them: 1. **Default runner** (`python tests/run_all.py`): relative-path guard -> byte-identity gate (`tools/check_byte_identity.py`) -> `pytest` (the `test_*.py` modules - above) -> `ctest` **iff** the registered binaries (`rocke_ir_serialize_roundtrip`, - `rocke_tiled_attention_2d_reentrancy`) are present in `--build-root`. This is the - only set that is gated. + above) -> `ctest` when any registered test executable is built in `--build-root` + for the selected `--config`. CTest supplies the executable paths, including + configuration directories and platform suffixes. The entire registered suite + runs, so a partial build exposes missing tests as failures. No configured test + build or no built registered tests is reported explicitly as a skipped stage. 2. **Diagnostics** (opt-in, not in the gate): `run_diff.py --ir` (IR-canonical diff), `fuzz_diff.py`, `ir_artifact_diff.py`. 3. **GPU / manual numeric lanes** (need a HIP device; skipped/not-collected diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py index caa74ce0cc67..23515da8a3cd 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py @@ -189,6 +189,47 @@ def load(): assert "memref.global_load" in serialize(b.kernel) +@pytest.mark.parametrize("dtype", ["e8m0", "fp6", "f16"]) +@pytest.mark.parametrize( + "count,groups,k0,valid", + [ + (64, 100_000_000, 0, False), + (64, 1 << 25, 0, True), + (64, 1 << 25, 1, False), + (64, 2, (1 << 31) - 128, True), + (64, 2, (1 << 31) - 127, False), + (16, 100_000_000, 1_500_000_000, False), + (64, 0x7FFFFFFF, 0, False), + ], +) +def test_fragment_offset_bounds(dtype, count, groups, k0, valid): + # Padded FP6 and E8M0 use byte units; FP16 uses two-byte pointer units. + bits = 16 if dtype == "f16" else 6 if dtype == "fp6" else 8 + packing = BitPacking(bits, 16 if dtype == "f16" else 8) + storage = TensorStorage( + dtype, (1, k0 + count * groups), slot_bits=packing.slot_bits + ) + layout = MatrixFragmentLayout( + FragmentPacking(packing, count, 32, count * packing.slot_bits // 32), + 16, + groups, + 1, + ) + b = IRBuilder("offset_bounds") + ptr = b.param("A", PtrType(storage_ir_type(dtype), "global")) + zero = b.const_i32(0) + before = serialize(b.kernel) + if valid: + assert ( + load_matrix_fragment(b, ptr, zero, zero, k0, storage=storage, layout=layout) + is not None + ) + else: + with pytest.raises(ValueError, match="offset exceeds i32 range"): + load_matrix_fragment(b, ptr, zero, zero, k0, storage=storage, layout=layout) + assert serialize(b.kernel) == before + + def test_hip_declares_only_encountered_missing_vector_widths(): source = lower_kernel_to_hip(build_transport("fp6"), arch="gfx1250") for name in ("i8x24", "i8x40", "i32x12"): diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp index 4114583b8320..981597c52560 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp @@ -261,6 +261,61 @@ static int test_empty_storage_load() return 0; } +static int test_fragment_offset_bounds() +{ + const struct + { + int count, groups; + uint64_t k0; + bool valid; + } cases[] = {{64, 100000000, 0, false}, + {64, 1 << 25, 0, true}, + {64, 1 << 25, 1, false}, + {64, 2, (uint64_t(1) << 31) - 128, true}, + {64, 2, (uint64_t(1) << 31) - 127, false}, + {16, 100000000, 1500000000, false}, + {64, INT32_MAX, 0, false}}; + for(const char* dtype : {"e8m0", "fp6", "f16"}) + { + const int slot_bits = strcmp(dtype, "f16") == 0 ? 16 : 8; + for(const auto& c : cases) + { + rocke_tensor_storage_t storage; + CHECK(rocke_tensor_storage_init(&storage, + dtype, + 1, + c.k0 + uint64_t(c.count) * c.groups, + UINT64_MAX, + slot_bits, + 0, + 1)); + rocke_fragment_packing_t fragment; + CHECK(rocke_fragment_packing_init( + &fragment, &storage.packing, c.count, 32, c.count * slot_bits / 32)); + rocke_matrix_fragment_layout_t layout; + CHECK(rocke_matrix_fragment_layout_init(&layout, &fragment, 16, c.groups, 1)); + rocke_ir_builder_t b; + CHECK(rocke_ir_builder_init(&b, "offset_bounds") == ROCKE_OK); + auto* ptr = rocke_b_param( + &b, "A", rocke_ptr_type(&b, rocke_storage_ir_type(dtype), "global"), NULL); + auto* zero = rocke_b_const_i32(&b, 0); + const int before = b.kernel->body->num_ops; + auto* value = rocke_h_load_matrix_fragment( + &b, ptr, zero, zero, c.k0, &storage, &layout, rocke_i32()); + if(c.valid) + CHECK(value && rocke_ir_builder_ok(&b)); + else + { + CHECK(!value && rocke_ir_builder_status(&b) == ROCKE_ERR_VALUE); + CHECK(strstr(rocke_ir_builder_error(&b), "offset exceeds i32 range")); + CHECK(b.kernel->body->num_ops == before); + } + rocke_ir_builder_free(&b); + } + } + return 0; +} + int main(int argc, char** argv) { if(argc == 3 && strcmp(argv[1], "--emit") == 0) @@ -299,6 +354,7 @@ int main(int argc, char** argv) } CHECK(test_storage_validation() == 0); CHECK(test_empty_storage_load() == 0); + CHECK(test_fragment_offset_bounds() == 0); CHECK(rocke_dtype_info("e4m3") == rocke_dtype_info("fp8e4m3")); CHECK(rocke_dtype_to_ir_type("e4m3") == rocke_fp8e4m3()); CHECK(rocke_storage_ir_type("e4m3") == rocke_fp8e4m3()); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py index 7f60753b8175..7f4b90c6124e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py @@ -87,6 +87,34 @@ def relative_path_guard() -> int: return 0 +def ctest_tests( + build_root: Path, config: str, pattern: str | None = None +) -> list[dict]: + command = [ + "ctest", + "--test-dir", + str(build_root), + "-C", + config, + "--show-only=json-v1", + ] + if pattern: + command += ["-R", pattern] + listing = subprocess.run(command, check=True, capture_output=True, text=True) + return json.loads(listing.stdout)["tests"] + + +def ctest_ready(build_root: Path, config: str) -> bool: + if not (build_root / "CTestTestfile.cmake").is_file(): + print("\n== ctest: SKIPPED (no configured test build) ==") + return False + commands = [test.get("command", []) for test in ctest_tests(build_root, config)] + if any(command and Path(command[0]).is_file() for command in commands): + return True + print(f"\n== ctest: SKIPPED (no built registered tests for {config}) ==") + return False + + def native_pytest_env(build_root: Path, config: str) -> dict[str, str]: """Make the native storage comparisons available to both pytest passes.""" env = dict(os.environ) @@ -104,22 +132,7 @@ def native_pytest_env(build_root: Path, config: str) -> dict[str, str]: ], check=True, ) - listing = subprocess.run( - [ - "ctest", - "--test-dir", - str(build_root), - "-C", - config, - "--show-only=json-v1", - "-R", - "^rocke_storage$", - ], - check=True, - capture_output=True, - text=True, - ) - tests = json.loads(listing.stdout)["tests"] + tests = ctest_tests(build_root, config, "^rocke_storage$") commands = [ test.get("command", []) for test in tests if test["name"] == "rocke_storage" ] @@ -248,18 +261,14 @@ def main() -> int: status |= differential_pytest_pass(pytest_env) build_root = Path(args.build_root) - # Only ctest when the CTest-registered binaries were actually built (the - # byte-identity gate builds just `rocke_core`, so a gate-only build dir has the - # registration file but no test executables -> running ctest there would - # spuriously fail). Gate on the registered tests only; `rocke_smoke` is an - # optional build-only target (not an add_test target) so it is not a signal. - test_bins = [ - build_root / "tests" / b - for b in ("rocke_ir_serialize_roundtrip", "rocke_tiled_attention_2d_reentrancy") - ] - if (build_root / "CTestTestfile.cmake").exists() and any( - b.exists() for b in test_bins - ): + # Run the entire registered suite once any executable is built; partial + # builds must expose their missing tests rather than silently lose coverage. + try: + ready = ctest_ready(build_root, args.config) + except (OSError, ValueError, subprocess.CalledProcessError) as exc: + print(f"CTest discovery failed: {exc}", file=sys.stderr) + return 1 + if ready: print("\n== ctest ==") status |= subprocess.run( ["ctest", "-C", args.config, "--output-on-failure", "--no-tests=ignore"], diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py index 28e06b7b5c46..d9dbd4f08bdd 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py @@ -28,6 +28,7 @@ def test_runner_passes_registered_fixture_to_both_backends( runner, monkeypatch, tmp_path, filename ): (tmp_path / "CMakeCache.txt").touch() + (tmp_path / "CTestTestfile.cmake").touch() executable = tmp_path / "Debug" / filename executable.parent.mkdir() executable.touch() @@ -69,6 +70,77 @@ def run(command, **kwargs): "--target", "rocke_storage", ] + assert any(cmd[:3] == ["ctest", "-C", "Debug"] for cmd, _ in calls) + + +@pytest.mark.parametrize("built", ["none", "partial", "all"]) +@pytest.mark.parametrize("ctest_rc", [0, 8]) +def test_ctest_readiness_and_failure_propagation( + runner, monkeypatch, tmp_path, capsys, built, ctest_rc +): + (tmp_path / "CTestTestfile.cmake").touch() + executable = tmp_path / "Debug" / "renamed.exe" + executable.parent.mkdir() + if built != "none": + executable.touch() + commands = [[str(executable)], [] if built != "all" else [str(executable)]] + calls = [] + + def run(command, **kwargs): + calls.append(command) + listing = { + "tests": [ + {"name": str(i), "command": cmd} for i, cmd in enumerate(commands) + ] + } + return subprocess.CompletedProcess( + command, + 0 if "--show-only=json-v1" in command else ctest_rc, + stdout=json.dumps(listing), + ) + + monkeypatch.setattr(runner.subprocess, "run", run) + monkeypatch.setattr( + runner.sys, + "argv", + [ + "run_all.py", + "--no-guard", + "--no-gate", + "--no-pytest", + "--build-root", + str(tmp_path), + "--config", + "Debug", + ], + ) + assert runner.main() == (0 if built == "none" else ctest_rc) + executions = [cmd for cmd in calls if cmd[:3] == ["ctest", "-C", "Debug"]] + assert len(executions) == (0 if built == "none" else 1) + if built == "none": + assert "ctest: SKIPPED" in capsys.readouterr().out + + +def test_ctest_discovery_failure_is_an_error(runner, monkeypatch, tmp_path): + (tmp_path / "CTestTestfile.cmake").touch() + + def run(command, **kwargs): + raise subprocess.CalledProcessError(1, command) + + monkeypatch.setattr(runner.subprocess, "run", run) + monkeypatch.setattr( + runner.sys, + "argv", + [ + "run_all.py", + "--no-guard", + "--no-gate", + "--no-pytest", + "--build-root", + str(tmp_path), + ], + ) + assert runner.main() == 1 def test_explicit_fixture_override_is_resolved_before_pytest_changes_directory( From 4b57104778b7b76ee4d8061fa43bfe240c90954e Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:35:42 -0400 Subject: [PATCH 10/32] refactor(rocke): keep bit packing without a duplicate tensor descriptor --- .../rocke/platform/cpp/core/storage.cpp | 60 ------ .../rocke/platform/cpp/helpers/mma_io.cpp | 34 ++- .../rocke/helper_rocke.helpers.mma_io.h | 9 +- .../platform/cpp/include/rocke/storage.h | 28 --- .../architecture/dtype_storage_packing.md | 65 +++--- .../platform/python/rocke/core/storage.py | 61 ------ .../platform/python/rocke/helpers/mma_io.py | 45 ++-- .../rocke/platform/tests/core/test_dtypes.py | 2 - .../rocke/platform/tests/core/test_mma_io.py | 107 +++------- .../platform/tests/core/test_storage.cpp | 193 +++--------------- .../rocke/platform/tests/core/test_storage.py | 31 +-- 11 files changed, 129 insertions(+), 506 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp index 131c00e3989d..c951bf74096f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/storage.cpp @@ -175,66 +175,6 @@ bool rocke_fragment_pack(const rocke_fragment_packing_t* p, return true; } -bool rocke_tensor_storage_init(rocke_tensor_storage_t* out, - const char* dtype, - uint64_t rows, - uint64_t cols, - uint64_t row_stride_bytes, - int slot_bits, - uint64_t base_bit_offset, - uint64_t alignment_bytes) -{ - rocke_tensor_storage_t p = {}; - p.dtype = rocke_dtype_info(dtype); - uint64_t row_bytes, bytes; - if(!out || !p.dtype || !alignment_bytes || (alignment_bytes & (alignment_bytes - 1)) - || !rocke_bit_packing_init(&p.packing, p.dtype->encoded_bits, slot_bits) - || !rocke_bit_packing_bytes(&p.packing, cols, base_bit_offset, &row_bytes)) - return false; - p.rows = rows; - p.cols = cols; - p.row_stride_bytes = row_stride_bytes == UINT64_MAX ? row_bytes : row_stride_bytes; - p.base_bit_offset = base_bit_offset; - p.alignment_bytes = alignment_bytes; - if(!rocke_tensor_storage_bytes(&p, &bytes)) - return false; - *out = p; - return true; -} - -bool rocke_tensor_storage_bytes(const rocke_tensor_storage_t* p, uint64_t* bytes) -{ - uint64_t row_bytes, prefix; - if(!p || !bytes || !p->dtype || !valid(&p->packing) - || rocke_dtype_info(p->dtype->name) != p->dtype - || p->packing.element_bits != p->dtype->encoded_bits || !p->alignment_bytes - || (p->alignment_bytes & (p->alignment_bytes - 1)) - || !rocke_bit_packing_bytes(&p->packing, p->cols, p->base_bit_offset, &row_bytes) - || p->row_stride_bytes < row_bytes) - return false; - if(!p->rows || !p->cols) - { - *bytes = 0; - return true; - } - return mul(p->rows - 1, p->row_stride_bytes, &prefix) && add(prefix, row_bytes, bytes); -} - -bool rocke_tensor_storage_address( - const rocke_tensor_storage_t* p, uint64_t row, uint64_t col, uint64_t* byte, int* shift) -{ - uint64_t bytes; - if(!byte || !shift || !rocke_tensor_storage_bytes(p, &bytes) || row >= p->rows - || col >= p->cols) - return false; - uint64_t bits, prefix; - if(!rocke_bit_packing_offset(&p->packing, col, &bits) || !add(bits, p->base_bit_offset, &bits) - || !mul(row, p->row_stride_bytes, &prefix) || !add(prefix, bits / 8, byte)) - return false; - *shift = bits % 8; - return true; -} - bool rocke_matrix_fragment_layout_init(rocke_matrix_fragment_layout_t* out, const rocke_fragment_packing_t* fragment, int chunk_elements, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp index c4baddbb2557..92a4e6a93396 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp @@ -34,21 +34,18 @@ rocke_value_t* rocke_h_load_matrix_fragment(rocke_ir_builder_t* b, rocke_value_t* row_base, rocke_value_t* lane_group, uint64_t k0, - const rocke_tensor_storage_t* storage, + const char* dtype, const rocke_matrix_fragment_layout_t* layout, - const rocke_type_t* carrier_type) + const rocke_type_t* carrier_type, + int alignment_bytes) { return ckc::guard_builder(b, [&]() -> rocke_value_t* { if(!rocke_i_live(b)) return NULL; - if(!storage || !storage->dtype || !layout || !carrier_type || !ptr || !row_base - || !lane_group) + if(!dtype || !layout || !carrier_type || !ptr || !row_base || !lane_group) ckc::raise_status(ROCKE_ERR_VALUE, "null matrix fragment argument"); - uint64_t storage_bytes; - if(!rocke_tensor_storage_bytes(storage, &storage_bytes)) - ckc::raise_status(ROCKE_ERR_VALUE, "invalid tensor storage descriptor"); - if(storage_bytes == 0) - ckc::raise_status(ROCKE_ERR_VALUE, "matrix fragment requires nonempty tensor storage"); + if(alignment_bytes <= 0 || (alignment_bytes & (alignment_bytes - 1))) + ckc::raise_status(ROCKE_ERR_VALUE, "alignment_bytes must be a positive power of two"); const auto& packing = layout->fragment; rocke_matrix_fragment_layout_t checked; if(!rocke_matrix_fragment_layout_init(&checked, @@ -58,23 +55,17 @@ rocke_value_t* rocke_h_load_matrix_fragment(rocke_ir_builder_t* b, layout->lanes_per_group) || !packing.count || packing.count > INT_MAX || packing.carrier_count > INT_MAX) ckc::raise_status(ROCKE_ERR_VALUE, "invalid matrix fragment chunk layout"); - const rocke_type_t* unit_type = rocke_storage_ir_type(storage->dtype->name); + const rocke_type_t* unit_type = rocke_storage_ir_type(dtype); if(!unit_type || ptr->type->kind != ROCKE_TYPE_PTR || !rocke_type_eq(ptr->type->pointee, unit_type)) ckc::raise_status(ROCKE_ERR_VALUE, "matrix pointer storage type mismatch"); const uint64_t unit_bytes = rocke_dtype_info(unit_type->name)->encoded_bits / 8; - if(storage->packing.element_bits != packing.packing.element_bits - || storage->packing.slot_bits != packing.packing.slot_bits || storage->base_bit_offset) - ckc::raise_status(ROCKE_ERR_VALUE, - "matrix fragment requires matching packing and a zero bit origin"); - const uint64_t span = packing.count * layout->lane_groups; - if(k0 > storage->cols || span > storage->cols - k0) - ckc::raise_status(ROCKE_ERR_VALUE, "matrix fragment exceeds the packed row"); + if(rocke_dtype_info(dtype)->encoded_bits != packing.packing.element_bits) + ckc::raise_status(ROCKE_ERR_VALUE, "matrix dtype and packing width mismatch"); uint64_t origin_bits, chunk_bytes; - if(!rocke_bit_packing_offset(&storage->packing, k0, &origin_bits) + if(!rocke_bit_packing_offset(&packing.packing, k0, &origin_bits) || !rocke_bit_packing_bytes(&packing.packing, layout->chunk_elements, 0, &chunk_bytes) - || origin_bits % (8 * unit_bytes) || chunk_bytes % unit_bytes - || storage->row_stride_bytes % unit_bytes) + || origin_bits % (8 * unit_bytes) || chunk_bytes % unit_bytes) ckc::raise_status(ROCKE_ERR_VALUE, "matrix fragment is not aligned to pointer storage units"); const rocke_dtype_info_t* carrier_info = rocke_dtype_info(carrier_type->name); @@ -98,8 +89,7 @@ rocke_value_t* rocke_h_load_matrix_fragment(rocke_ir_builder_t* b, > (uint64_t(INT_MAX) + 1 - origin_bytes / unit_bytes) / chunk_units / layout->lane_groups) ckc::raise_status(ROCKE_ERR_VALUE, "matrix fragment offset exceeds i32 range"); - int alignment = gcd(gcd(storage->alignment_bytes, storage->row_stride_bytes), - gcd(chunk_bytes, origin_bytes)); + int alignment = gcd(alignment_bytes, gcd(chunk_bytes, origin_bytes)); auto* lane_chunk = rocke_b_mul(b, lane_group, rocke_b_const_i32(b, chunk_units)); auto* step_base = rocke_b_add(b, row_base, rocke_b_const_i32(b, origin_bytes / unit_bytes)); /* Collect loads before concatenation to preserve Python SSA numbering. */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.mma_io.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.mma_io.h index 8f5db63aa960..7dea7828c6f6 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.mma_io.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.mma_io.h @@ -8,6 +8,7 @@ #ifndef ROCKE_HELPER_MMA_IO_H #define ROCKE_HELPER_MMA_IO_H +#include "rocke/dtypes.h" #include "rocke/ir.h" #include "rocke/storage.h" @@ -16,14 +17,18 @@ extern "C" { #endif const rocke_type_t* rocke_storage_ir_type(const char* dtype); +/* row_base is in pointer storage units. alignment_bytes is guaranteed at that + * row address, before k0 and lane/chunk offsets. Caller owns tensor indexing, + * allocation bounds (including empty/partial rows), and lane_group validity. */ rocke_value_t* rocke_h_load_matrix_fragment(rocke_ir_builder_t* b, rocke_value_t* ptr, rocke_value_t* row_base, rocke_value_t* lane_group, uint64_t k0, - const rocke_tensor_storage_t* storage, + const char* dtype, const rocke_matrix_fragment_layout_t* layout, - const rocke_type_t* carrier_type); + const rocke_type_t* carrier_type, + int alignment_bytes); /* Callback returns a canonical unsigned pattern in an integer carrier. * ctx and output pointer array are borrowed for the duration of this call. */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h index 133c1310c9ac..9634dbcc2e1f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h @@ -3,7 +3,6 @@ /* Shared bit storage; mirrors core/storage.py. * BitPacking -> rocke_bit_packing_t * FragmentPacking -> rocke_fragment_packing_t - * TensorStorage -> rocke_tensor_storage_t * All descriptors are caller-owned. Functions return false for invalid inputs, * overflow or insufficient buffers; no exceptions cross this C API. */ @@ -14,8 +13,6 @@ #include #include -#include "rocke/dtypes.h" - #ifdef __cplusplus extern "C" { #endif @@ -50,17 +47,6 @@ bool rocke_matrix_fragment_layout_init(rocke_matrix_fragment_layout_t* out, bool rocke_matrix_fragment_coord( const rocke_matrix_fragment_layout_t* p, int lane, uint64_t slot, uint64_t* row, uint64_t* k); -typedef struct rocke_tensor_storage -{ - const rocke_dtype_info_t* dtype; /* Borrowed static descriptor. */ - uint64_t rows; - uint64_t cols; - uint64_t row_stride_bytes; - uint64_t base_bit_offset; - uint64_t alignment_bytes; - rocke_bit_packing_t packing; -} rocke_tensor_storage_t; - bool rocke_bit_packing_init(rocke_bit_packing_t* out, int element_bits, int slot_bits); bool rocke_bit_packing_group(const rocke_bit_packing_t* p, int carrier_bits, @@ -95,20 +81,6 @@ bool rocke_fragment_pack(const rocke_fragment_packing_t* p, size_t count, uint64_t* carriers, size_t carrier_count); -/* row_stride_bytes=UINT64_MAX selects a minimal packed row stride. */ -bool rocke_tensor_storage_init(rocke_tensor_storage_t* out, - const char* dtype, - uint64_t rows, - uint64_t cols, - uint64_t row_stride_bytes, - int slot_bits, - uint64_t base_bit_offset, - uint64_t alignment_bytes); -/* Size and address queries validate public descriptors; invalid input returns false. */ -bool rocke_tensor_storage_bytes(const rocke_tensor_storage_t* p, uint64_t* bytes); -bool rocke_tensor_storage_address( - const rocke_tensor_storage_t* p, uint64_t row, uint64_t col, uint64_t* byte, int* shift); - #ifdef __cplusplus } #endif diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md index 0d68276de961..aeb0a863af35 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md @@ -3,7 +3,7 @@ Matrix inputs and scales use three separate contracts: 1. A logical dtype identifies the numerical encoding and its bit width. -2. Tensor storage describes how elements occupy an addressable buffer. +2. Bit packing describes how encoded patterns occupy bytes and carrier words. 3. An instruction fragment describes lane mapping, carrier capacity, and padding. The selected atom determines supported operand combinations and instruction @@ -13,7 +13,7 @@ selectors. An integer carrier does not identify the numerical format. - [Logical dtypes](#logical-dtypes) - [Common packing](#common-packing) -- [Tensor storage](#tensor-storage) +- [Fragment loads](#fragment-loads) - [Matrix and scale layouts](#matrix-and-scale-layouts) - [Mirroring and follow-up work](#mirroring-and-follow-up-work) @@ -66,35 +66,36 @@ Host pack/unpack routines process bit patterns only. They do not round, clamp, decode floats, or apply scales. Their tests use literal expected bits and independent integer arithmetic; numerical tests remain separate. -## Tensor storage - -`TensorStorage` / `rocke_tensor_storage_t` initially supports a two-dimensional -logical shape with a contiguous packed inner axis, an explicit byte row stride, -bit offset, and base alignment. Nonnegative sizes and offsets are checked against -the native uint64 range. A/B descriptors are independent. - -For example, dense FP6 uses four elements in three bytes; `slot_bits=8` describes -one six-bit pattern in each byte instead. Both have the same logical dtype. -Address calculation includes the bit offset, and allocation bounds include the -partial tail byte. Empty views require no accesses. -Native size queries, address queries, and fragment loads revalidate the public -storage descriptor, including dtype/packing consistency, stride, alignment, and -allocation bounds, because C callers can construct or mutate these structs. - -[`storage_ir_type`](../../python/rocke/helpers/mma_io.py) chooses an addressable unit. FP4/FP6 use I8 -units while retaining their logical identities in the storage descriptor. -This function is deliberately separate from `dtype_to_ir_type`. -FP8/BF8 retain nominal FP8/BF8 pointer types despite occupying one byte per value. - -The fragment loader accepts complete, byte-aligned chunks. It derives load -alignment from base alignment, row stride, chunk spacing, and origin. A 24-byte -FP6 chunk becomes 16-byte and 8-byte loads without reading past the chunk. -Partial fragments and nonzero bit origins are rejected by this initial loader. -Row strides must also be divisible by the pointer storage unit: an odd-byte -FP16/BF16 stride is valid generic storage but cannot be loaded through a typed -two-byte pointer. `row_base` is an element index in those storage units. -The host bit packer writes a fresh, exclusively owned output buffer; concurrent -packed stores are not provided. +## Fragment loads + +[`storage_ir_type`](../../python/rocke/helpers/mma_io.py) chooses an addressable +unit from the logical dtype. FP4/FP6 use I8; FP8/BF8 retain their nominal pointer +types despite occupying one byte per value. This function is separate from +`dtype_to_ir_type`. + +`load_matrix_fragment` takes a dtype, fragment layout, caller-selected row base, +lane group, and K origin. `row_base` counts pointer storage units. The optional +`alignment_bytes` (default 1) guarantees alignment at that row address, before +the loader adds K and lane/chunk offsets. For a buffer aligned to 16 bytes with +97-byte row spacing, arbitrary rows only guarantee alignment 1; a typed FP16 +buffer with 258-byte row spacing guarantees alignment 2 and uses row offsets +in units of two bytes. + +The loader checks dtype/packing width, pointer identity, whole-unit chunk and +origin alignment, carrier capacity, and static i32 displacement limits. It +reduces the supplied alignment for chunk spacing and K origin. A 24-byte FP6 +chunk becomes 16-byte and 8-byte loads without reading past the chunk. + +Tensor shape, strides, row selection, and allocation bounds belong to the +caller. This helper always loads a complete fragment: the caller must guard +empty or partial rows and provide a valid lane group. It is not a bounds-checked +tensor view. Existing `TensorDescriptor`/`TensorView` integration is deferred; +no separate tensor descriptor is introduced here. + +`BitPacking(6)` describes dense FP6; `BitPacking(6, 8)` describes one six-bit +pattern per byte. Bit offsets and stream sizes are checked against uint64. +Host packers support partial tail bytes and write fresh, exclusively owned +buffers; concurrent packed stores are not provided. ## Matrix and scale layouts @@ -128,7 +129,7 @@ absent from its fixed compatibility prologue. The shared descriptor/helper changes can be consumed by both FP4 and FP6 branches. FP4/FP8 builder migration can use them directly. Future FP6 numerical -integration should consume the six-bit storage and fragment descriptors and +integration should consume the six-bit packing and fragment descriptors and retain separate target and conversion validation. First-class tensor-view/fragment IR nodes, arbitrary packed axes, masked partial diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/storage.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/storage.py index 7c59be34fc0f..c9c1ce3282e9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/storage.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/storage.py @@ -8,8 +8,6 @@ from math import gcd from collections.abc import Sequence -from .dtypes import DTypeInfo, dtype_info - _MAX_BITS = (1 << 64) - 1 @@ -168,62 +166,3 @@ def coord(self, lane: int, slot: int) -> tuple[int, int]: * self.chunk_elements + element, ) - - -@dataclass(frozen=True) -class TensorStorage: - """Two-dimensional storage with a contiguous packed inner axis. - - Shape is logical. Row strides are bytes. Generic axis permutations and - negative strides are intentionally rejected by this initial interface. - """ - - dtype: str - shape: tuple[int, int] - row_stride_bytes: int | None = None - slot_bits: int | None = None - base_bit_offset: int = 0 - alignment_bytes: int = 1 - - def __post_init__(self) -> None: - info = dtype_info(self.dtype) - object.__setattr__(self, "dtype", info.name) - if len(self.shape) != 2: - raise ValueError("storage requires a two-dimensional logical shape") - for dim in self.shape: - _checked(dim) - _checked(self.base_bit_offset) - if self.alignment_bytes <= 0 or self.alignment_bytes & ( - self.alignment_bytes - 1 - ): - raise ValueError("alignment_bytes must be a positive power of two") - row_bytes = self.packing.byte_size(self.shape[1], self.base_bit_offset) - stride = row_bytes if self.row_stride_bytes is None else self.row_stride_bytes - if _checked(stride) < row_bytes: - raise ValueError("row stride is smaller than the packed row") - object.__setattr__(self, "row_stride_bytes", stride) - _checked(self.byte_size) - - @property - def info(self) -> DTypeInfo: - return dtype_info(self.dtype) - - @property - def packing(self) -> BitPacking: - return BitPacking(self.info.encoded_bits, self.slot_bits) - - @property - def byte_size(self) -> int: - rows, cols = self.shape - if rows == 0 or cols == 0: - return 0 - return _checked( - (rows - 1) * self.row_stride_bytes - + self.packing.byte_size(cols, self.base_bit_offset) - ) - - def address(self, row: int, col: int) -> tuple[int, int]: - if not (0 <= row < self.shape[0] and 0 <= col < self.shape[1]): - raise ValueError("storage coordinate out of bounds") - bits = _checked(self.base_bit_offset + self.packing.bit_offset(col)) - return _checked(row * self.row_stride_bytes + bits // 8), bits % 8 diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py index b47f120a0beb..28bfdf60c316 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py @@ -19,7 +19,7 @@ VectorType, dtype_to_ir_type, ) -from ..core.storage import FragmentPacking, MatrixFragmentLayout, TensorStorage +from ..core.storage import FragmentPacking, MatrixFragmentLayout def storage_ir_type(dtype: str) -> Type: @@ -37,38 +37,32 @@ def load_matrix_fragment( lane_group: Value, k0: int, *, - storage: TensorStorage, + dtype: str, layout: MatrixFragmentLayout, carrier_type: Type = I32, + alignment_bytes: int = 1, ) -> Value: - """Load whole-byte chunks directly into the atom's carrier vector. + """Load complete chunks from a caller-selected row into carrier registers. - row_base counts pointer storage units, which must divide the row stride. - lane_group is derived from the atom's - lane mapping by the caller. Partial chunks and nonzero bit origins reject. + row_base counts pointer storage units; alignment_bytes is the guaranteed + alignment at that row address, before k0 and lane/chunk offsets. The caller + owns tensor indexing and allocation bounds, including empty/partial rows. + lane_group comes from the atom's lane mapping. k0 must address a whole + pointer unit. TensorDescriptor/TensorView integration is deferred. """ - if storage.byte_size == 0: - raise ValueError("matrix fragment requires nonempty tensor storage") + if alignment_bytes <= 0 or alignment_bytes & (alignment_bytes - 1): + raise ValueError("alignment_bytes must be a positive power of two") packing = layout.fragment if not (0 < packing.count <= 0x7FFFFFFF and packing.carrier_count <= 0x7FFFFFFF): raise ValueError("invalid matrix fragment chunk layout") - unit_type = storage_ir_type(storage.dtype) + unit_type = storage_ir_type(dtype) unit_bytes = dtype_info(unit_type.name).encoded_bits // 8 if ptr.type.pointee != unit_type: raise ValueError("matrix pointer storage type mismatch") - if storage.packing != packing.packing or storage.base_bit_offset: - raise ValueError( - "matrix fragment requires matching packing and a zero bit origin" - ) - span = layout.chunk_elements * layout.lane_groups * layout.chunks_per_lane - if k0 < 0 or k0 + span > storage.shape[1]: - raise ValueError("matrix fragment exceeds the packed row") - origin_bits = storage.packing.bit_offset(k0) - if ( - origin_bits % (8 * unit_bytes) - or layout.chunk_bytes % unit_bytes - or storage.row_stride_bytes % unit_bytes - ): + if dtype_info(dtype).encoded_bits != packing.packing.element_bits: + raise ValueError("matrix dtype and packing width mismatch") + origin_bits = packing.packing.bit_offset(k0) + if origin_bits % (8 * unit_bytes) or layout.chunk_bytes % unit_bytes: raise ValueError("matrix fragment is not aligned to pointer storage units") if dtype_info(carrier_type.name).encoded_bits != packing.carrier_bits: raise ValueError("matrix carrier type width mismatch") @@ -84,12 +78,7 @@ def load_matrix_fragment( > (0x80000000 - origin_bytes // unit_bytes) // chunk_units // layout.lane_groups ): raise ValueError("matrix fragment offset exceeds i32 range") - alignment = gcd( - storage.alignment_bytes, - storage.row_stride_bytes, - layout.chunk_bytes, - origin_bytes, - ) + alignment = gcd(alignment_bytes, layout.chunk_bytes, origin_bytes) lane_chunk = b.mul(lane_group, b.const_i32(chunk_units)) step_base = b.add(row_base, b.const_i32(origin_bytes // unit_bytes)) chunks = [] diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py index 2222d6ed4727..2fd2eb0ac7e9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py @@ -10,7 +10,6 @@ from rocke.core.dtypes import dtype_info from rocke.core.arch.target import MmaScaleDType from rocke.core.ir import FP8E4M3, I8, IRBuilder, dtype_to_ir_type -from rocke.core.storage import TensorStorage from rocke.helpers.mma_io import storage_ir_type from rocke.core.ir_serialize import parse, serialize from rocke.helpers.quant import ( @@ -102,7 +101,6 @@ def test_e4m3_shares_encoding_type_and_storage(spelling): assert dtype_info(spelling) is dtype_info("fp8e4m3") assert dtype_to_ir_type(spelling) is FP8E4M3 assert storage_ir_type(spelling) is FP8E4M3 - assert TensorStorage(spelling, (2, 128)) == TensorStorage("fp8e4m3", (2, 128)) assert quant_ir_type(spelling) is FP8E4M3 b = IRBuilder("e4m3_conversion") value = b.param("value", dtype_to_ir_type(spelling)) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py index 23515da8a3cd..67caa9541dcf 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py @@ -19,7 +19,6 @@ BitPacking, FragmentPacking, MatrixFragmentLayout, - TensorStorage, ) from rocke.helpers.mma_io import ( load_matrix_fragment, @@ -69,12 +68,6 @@ def build_transport(dtype): for j, value in enumerate(values): b.global_store(o, b.const_i32(j), value, align=4) else: - storage = TensorStorage( - dtype, - (16, 128), - row_stride_bytes=(258 if typed else 97) if padded else None, - alignment_bytes=16, - ) layout = ( MatrixFragmentLayout(FragmentPacking(BitPacking(16), 32, 16, 32), 16, 2, 16) if typed @@ -84,7 +77,15 @@ def build_transport(dtype): lane = b.mod(b.thread_id_x(), b.const_i32(32)) group = b.div(lane, b.const_i32(16)) value = load_matrix_fragment( - b, a, base, group, 0, storage=storage, layout=layout, carrier_type=carrier + b, + a, + base, + group, + 0, + dtype=dtype, + layout=layout, + carrier_type=carrier, + alignment_bytes=(2 if typed else 1) if padded else 16, ) for j in range(value.type.count): index = b.const_i32(j) @@ -124,69 +125,43 @@ def test_native_helper_emits_identical_ir(dtype, route): assert actual.stdout == expected -def test_reject_partial_fragment_and_wrong_pointer(): +def test_reject_bit_origin(): b = IRBuilder("invalid_transport") ptr = b.param("A", PtrType(I8, "global")) zero = b.const_i32(0) - with pytest.raises(ValueError, match="exceeds"): + with pytest.raises(ValueError, match="aligned to pointer storage units"): load_matrix_fragment( b, ptr, zero, zero, 1, - storage=TensorStorage("fp4", (16, 128)), + dtype="fp4", layout=scaled_matrix_layout("fp4", 16), ) - with pytest.raises(ValueError, match="pointer storage type"): - load_matrix_fragment( - b, - ptr, - zero, - zero, - 0, - storage=TensorStorage("fp8", (16, 128)), - layout=scaled_matrix_layout("fp8", 16), - ) -@pytest.mark.parametrize("dtype", ["fp4", "fp6", "fp8", "f16", "bf16"]) -@pytest.mark.parametrize("shape", [(0, 128), (16, 0), (0, 0), (16, 128)]) -def test_fragment_load_requires_nonempty_storage(dtype, shape): - storage = TensorStorage(dtype, shape) - typed = dtype in ("f16", "bf16") - unit = storage_ir_type(dtype) - layout = ( - MatrixFragmentLayout(FragmentPacking(BitPacking(16), 32, 16, 32), 16, 2, 16) - if typed - else scaled_matrix_layout(dtype, 16) - ) - b = IRBuilder("empty_storage") - ptr = b.param("A", PtrType(unit, "global")) +@pytest.mark.parametrize( + "dtype,alignment,error", + [("fp6", 3, "positive power of two"), ("fp4", 1, "packing width mismatch")], +) +def test_fragment_input_contract(dtype, alignment, error): + b = IRBuilder("invalid_fragment") + ptr = b.param("A", PtrType(I8, "global")) zero = b.const_i32(0) before = serialize(b.kernel) - - def load(): - return load_matrix_fragment( + with pytest.raises(ValueError, match=error): + load_matrix_fragment( b, ptr, zero, zero, 0, - storage=storage, - layout=layout, - carrier_type=unit if typed else I32, + dtype=dtype, + layout=scaled_matrix_layout("fp6", 16), + alignment_bytes=alignment, ) - - if 0 in shape: - assert storage.byte_size == 0 - with pytest.raises(ValueError, match="nonempty tensor storage"): - load() - assert serialize(b.kernel) == before - else: - assert storage.byte_size > 0 - assert load() is not None - assert "memref.global_load" in serialize(b.kernel) + assert serialize(b.kernel) == before @pytest.mark.parametrize("dtype", ["e8m0", "fp6", "f16"]) @@ -206,9 +181,6 @@ def test_fragment_offset_bounds(dtype, count, groups, k0, valid): # Padded FP6 and E8M0 use byte units; FP16 uses two-byte pointer units. bits = 16 if dtype == "f16" else 6 if dtype == "fp6" else 8 packing = BitPacking(bits, 16 if dtype == "f16" else 8) - storage = TensorStorage( - dtype, (1, k0 + count * groups), slot_bits=packing.slot_bits - ) layout = MatrixFragmentLayout( FragmentPacking(packing, count, 32, count * packing.slot_bits // 32), 16, @@ -221,12 +193,12 @@ def test_fragment_offset_bounds(dtype, count, groups, k0, valid): before = serialize(b.kernel) if valid: assert ( - load_matrix_fragment(b, ptr, zero, zero, k0, storage=storage, layout=layout) + load_matrix_fragment(b, ptr, zero, zero, k0, dtype=dtype, layout=layout) is not None ) else: with pytest.raises(ValueError, match="offset exceeds i32 range"): - load_matrix_fragment(b, ptr, zero, zero, k0, storage=storage, layout=layout) + load_matrix_fragment(b, ptr, zero, zero, k0, dtype=dtype, layout=layout) assert serialize(b.kernel) == before @@ -240,29 +212,6 @@ def test_hip_declares_only_encountered_missing_vector_widths(): assert "using i8x24 =" not in source -@pytest.mark.parametrize("dtype", ["f16", "bf16"]) -def test_reject_row_stride_not_in_pointer_units(dtype): - b = IRBuilder("odd_stride") - unit = storage_ir_type(dtype) - ptr = b.param("A", PtrType(unit, "global")) - zero = b.const_i32(0) - before = serialize(b.kernel) - with pytest.raises(ValueError, match="aligned to pointer storage units"): - load_matrix_fragment( - b, - ptr, - zero, - zero, - 0, - storage=TensorStorage(dtype, (2, 64), row_stride_bytes=129), - layout=MatrixFragmentLayout( - FragmentPacking(BitPacking(16), 32, 16, 32), 16, 2, 16 - ), - carrier_type=unit, - ) - assert serialize(b.kernel) == before - - @pytest.mark.parametrize("dtype", ["fp8", "bf8"]) def test_byte_width_does_not_erase_pointer_identity(dtype): assert storage_ir_type(dtype) != I8 @@ -276,7 +225,7 @@ def test_byte_width_does_not_erase_pointer_identity(dtype): zero, zero, 0, - storage=TensorStorage(dtype, (16, 128)), + dtype=dtype, layout=scaled_matrix_layout(dtype, 16), ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp index 981597c52560..de507eba1c34 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp @@ -65,9 +65,6 @@ static int emit(const char* dtype, bool hip) } else { - rocke_tensor_storage_t storage; - CHECK(rocke_tensor_storage_init( - &storage, dtype, 16, 128, padded ? (typed ? 258 : 97) : UINT64_MAX, 0, 0, 16)); rocke_matrix_fragment_layout_t layout; if(typed) { @@ -83,8 +80,8 @@ static int emit(const char* dtype, bool hip) auto* thread = rocke_b_thread_id_x(&b); auto* lane = rocke_b_mod(&b, thread, rocke_b_const_i32(&b, 32)); auto* group = rocke_b_div(&b, lane, rocke_b_const_i32(&b, 16)); - auto* value - = rocke_h_load_matrix_fragment(&b, a, base, group, 0, &storage, &layout, carrier); + auto* value = rocke_h_load_matrix_fragment( + &b, a, base, group, 0, dtype, &layout, carrier, padded ? (typed ? 2 : 1) : 16); CHECK(value && rocke_ir_builder_ok(&b)); for(int j = 0; j < value->type->count; ++j) { @@ -115,152 +112,38 @@ static int emit(const char* dtype, bool hip) return 0; } -static int test_storage_validation() +static int test_fragment_inputs() { - rocke_tensor_storage_t storage; - CHECK(rocke_tensor_storage_init(&storage, "fp6", 16, 128, 97, 0, 0, 16)); - const auto layout = rocke_scaled_matrix_layout("fp6", 16); - for(int defect = 0; defect < 10; ++defect) + const struct { - auto invalid = storage; - switch(defect) - { - case 0: - invalid.dtype = NULL; - break; - case 1: - invalid.dtype = rocke_dtype_info("fp8"); - break; - case 2: - invalid.packing.element_bits = 0; - break; - case 3: - invalid.packing.slot_bits = 0; - break; - case 4: - invalid.packing.slot_bits = 4; - break; - case 5: - invalid.row_stride_bytes = 95; - break; - case 6: - invalid.alignment_bytes = 0; - break; - case 7: - invalid.alignment_bytes = 3; - break; - case 8: - invalid.rows = UINT64_MAX; - break; - case 9: - invalid.base_bit_offset = UINT64_MAX; - break; - } - uint64_t bytes; - int shift; - CHECK(!rocke_tensor_storage_bytes(&invalid, &bytes)); - CHECK(!rocke_tensor_storage_address(&invalid, 0, 0, &bytes, &shift)); - rocke_ir_builder_t b; - CHECK(rocke_ir_builder_init(&b, "invalid_storage") == ROCKE_OK); - auto* ptr = rocke_b_param(&b, "A", rocke_ptr_type(&b, rocke_i8(), "global"), NULL); - auto* zero = rocke_b_const_i32(&b, 0); - CHECK( - !rocke_h_load_matrix_fragment(&b, ptr, zero, zero, 0, &invalid, &layout, rocke_i32())); - CHECK(!rocke_ir_builder_ok(&b)); - rocke_ir_builder_free(&b); - if(defect < 8) - { - invalid.rows = 0; - CHECK(!rocke_tensor_storage_bytes(&invalid, &bytes)); - } - } - for(const char* dtype : {"f16", "bf16", "fp8", "e4m3", "bf8"}) + const char* dtype; + const char* layout_dtype; + uint64_t k0; + int alignment; + const char* error; + } cases[] = {{"fp6", "fp6", 0, 3, "positive power of two"}, + {"fp4", "fp6", 0, 1, "packing width mismatch"}, + {"fp4", "fp4", 1, 1, "aligned to pointer storage units"}, + {"fp8", "fp8", 0, 1, "pointer storage type mismatch"}, + {"bf8", "bf8", 0, 1, "pointer storage type mismatch"}}; + for(const auto& c : cases) { - const auto* unit = rocke_storage_ir_type(dtype); - CHECK(unit && !rocke_type_eq(unit, rocke_i8())); - const bool typed = strcmp(dtype, "f16") == 0 || strcmp(dtype, "bf16") == 0; - rocke_tensor_storage_t direct = {rocke_dtype_info(dtype), - 2, - 128, - typed ? 257u : 129u, - 0, - 1, - {typed ? 16 : 8, typed ? 16 : 8}}; - uint64_t bytes; - int shift; - CHECK(rocke_tensor_storage_bytes(&direct, &bytes)); - CHECK(rocke_tensor_storage_address(&direct, 1, 0, &bytes, &shift)); - CHECK(bytes == direct.row_stride_bytes && shift == 0); - rocke_matrix_fragment_layout_t fragment; - if(typed) - { - rocke_fragment_packing_t packing; - CHECK(rocke_fragment_packing_init(&packing, &direct.packing, 32, 16, 32)); - CHECK(rocke_matrix_fragment_layout_init(&fragment, &packing, 16, 2, 16)); - } - else - fragment = rocke_scaled_matrix_layout(dtype, 16); rocke_ir_builder_t b; - CHECK(rocke_ir_builder_init(&b, "invalid_pointer_units") == ROCKE_OK); - auto* ptr - = rocke_b_param(&b, "A", rocke_ptr_type(&b, typed ? unit : rocke_i8(), "global"), NULL); + CHECK(rocke_ir_builder_init(&b, "invalid_fragment") == ROCKE_OK); + auto* ptr = rocke_b_param(&b, "A", rocke_ptr_type(&b, rocke_i8(), "global"), NULL); auto* zero = rocke_b_const_i32(&b, 0); + const auto layout = rocke_scaled_matrix_layout(c.layout_dtype, 16); + const int before = b.kernel->body->num_ops; CHECK(!rocke_h_load_matrix_fragment( - &b, ptr, zero, zero, 0, &direct, &fragment, typed ? unit : rocke_i32())); - CHECK(!rocke_ir_builder_ok(&b)); + &b, ptr, zero, zero, c.k0, c.dtype, &layout, rocke_i32(), c.alignment)); + CHECK(rocke_ir_builder_status(&b) == ROCKE_ERR_VALUE); + CHECK(strstr(rocke_ir_builder_error(&b), c.error)); + CHECK(b.kernel->body->num_ops == before); rocke_ir_builder_free(&b); } return 0; } -static int test_empty_storage_load() -{ - for(const char* dtype : {"fp4", "fp6", "fp8", "f16", "bf16"}) - { - const auto* unit = rocke_storage_ir_type(dtype); - const bool typed = strcmp(dtype, "f16") == 0 || strcmp(dtype, "bf16") == 0; - for(int shape = 0; shape < 4; ++shape) - { - const uint64_t rows = shape == 0 || shape == 2 ? 0 : 16; - const uint64_t cols = shape == 1 || shape == 2 ? 0 : 128; - rocke_tensor_storage_t storage; - CHECK(rocke_tensor_storage_init(&storage, dtype, rows, cols, UINT64_MAX, 0, 0, 1)); - uint64_t bytes; - CHECK(rocke_tensor_storage_bytes(&storage, &bytes)); - CHECK((bytes == 0) == (shape != 3)); - rocke_matrix_fragment_layout_t layout; - if(typed) - { - rocke_fragment_packing_t fragment; - CHECK(rocke_fragment_packing_init(&fragment, &storage.packing, 32, 16, 32)); - CHECK(rocke_matrix_fragment_layout_init(&layout, &fragment, 16, 2, 16)); - } - else - layout = rocke_scaled_matrix_layout(dtype, 16); - rocke_ir_builder_t b; - CHECK(rocke_ir_builder_init(&b, "empty_storage") == ROCKE_OK); - auto* ptr = rocke_b_param(&b, "A", rocke_ptr_type(&b, unit, "global"), NULL); - auto* zero = rocke_b_const_i32(&b, 0); - const int before = b.kernel->body->num_ops; - auto* value = rocke_h_load_matrix_fragment( - &b, ptr, zero, zero, 0, &storage, &layout, typed ? unit : rocke_i32()); - if(bytes == 0) - { - CHECK(!value && rocke_ir_builder_status(&b) == ROCKE_ERR_VALUE); - CHECK(strstr(rocke_ir_builder_error(&b), "nonempty tensor storage")); - CHECK(b.kernel->body->num_ops == before); - } - else - { - CHECK(value && rocke_ir_builder_ok(&b)); - CHECK(b.kernel->body->num_ops > before); - } - rocke_ir_builder_free(&b); - } - } - return 0; -} - static int test_fragment_offset_bounds() { const struct @@ -280,18 +163,12 @@ static int test_fragment_offset_bounds() const int slot_bits = strcmp(dtype, "f16") == 0 ? 16 : 8; for(const auto& c : cases) { - rocke_tensor_storage_t storage; - CHECK(rocke_tensor_storage_init(&storage, - dtype, - 1, - c.k0 + uint64_t(c.count) * c.groups, - UINT64_MAX, - slot_bits, - 0, - 1)); + rocke_bit_packing_t packing; + CHECK(rocke_bit_packing_init( + &packing, strcmp(dtype, "fp6") == 0 ? 6 : slot_bits, slot_bits)); rocke_fragment_packing_t fragment; CHECK(rocke_fragment_packing_init( - &fragment, &storage.packing, c.count, 32, c.count * slot_bits / 32)); + &fragment, &packing, c.count, 32, c.count * slot_bits / 32)); rocke_matrix_fragment_layout_t layout; CHECK(rocke_matrix_fragment_layout_init(&layout, &fragment, 16, c.groups, 1)); rocke_ir_builder_t b; @@ -301,7 +178,7 @@ static int test_fragment_offset_bounds() auto* zero = rocke_b_const_i32(&b, 0); const int before = b.kernel->body->num_ops; auto* value = rocke_h_load_matrix_fragment( - &b, ptr, zero, zero, c.k0, &storage, &layout, rocke_i32()); + &b, ptr, zero, zero, c.k0, dtype, &layout, rocke_i32(), 1); if(c.valid) CHECK(value && rocke_ir_builder_ok(&b)); else @@ -352,8 +229,7 @@ int main(int argc, char** argv) rocke_ir_builder_free(&b); return 0; } - CHECK(test_storage_validation() == 0); - CHECK(test_empty_storage_load() == 0); + CHECK(test_fragment_inputs() == 0); CHECK(test_fragment_offset_bounds() == 0); CHECK(rocke_dtype_info("e4m3") == rocke_dtype_info("fp8e4m3")); CHECK(rocke_dtype_to_ir_type("e4m3") == rocke_fp8e4m3()); @@ -459,17 +335,6 @@ int main(int argc, char** argv) CHECK(words[0] == 0xC0000000 && words[1] == 0xF000000F && words[2] == 3); CHECK(!rocke_bit_packing_bytes(&six, UINT64_MAX, 0, &values)); CHECK(!rocke_fragment_packing_init(&fragment, &six, 16, 32, 2)); - rocke_tensor_storage_t storage; - CHECK(rocke_tensor_storage_init(&storage, "bf6", 3, 5, 8, 0, 3, 1)); - CHECK(rocke_tensor_storage_bytes(&storage, &values) && values == 21); - int shift; - CHECK(rocke_tensor_storage_address(&storage, 2, 4, &values, &shift)); - CHECK(values == 19 && shift == 3); - CHECK(!rocke_tensor_storage_address(&storage, 3, 0, &values, &shift)); - CHECK(!rocke_tensor_storage_init(&storage, "fp6", 3, 5, 3, 0, 0, 1)); - CHECK(!rocke_tensor_storage_init(&storage, "fp8", UINT64_MAX, 3, 3, 0, 0, 1)); - CHECK(rocke_tensor_storage_init(&storage, "fp4", 3, 0, 0, 0, 7, 1)); - CHECK(rocke_tensor_storage_bytes(&storage, &values) && values == 0); for(const char* dtype : {"fp4", "fp6", "bf6", "e8m0", "e4m3", "e5m3"}) { const auto* logical = rocke_dtype_to_ir_type(dtype); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py index 6824621d8f9c..5284914cc389 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py @@ -1,11 +1,11 @@ # Copyright (c) Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT -"""Independent bit-stream fixtures and storage bounds for matrices and scales.""" +"""Independent bit-stream fixtures for matrices and scales.""" import pytest from rocke.core.arch.wmma_scale import ScalePacking, scaled_matrix_layout -from rocke.core.storage import BitPacking, FragmentPacking, TensorStorage +from rocke.core.storage import BitPacking, FragmentPacking @pytest.mark.parametrize("bits", [4, 6, 8, 16, 32, 64]) @@ -23,6 +23,7 @@ def test_dense_patterns_against_integer_oracle(bits, offset): def test_fp6_crosses_words_without_per_word_padding(): packing = BitPacking(6) + assert packing.pack([], bit_offset=7) == b"" assert packing.group(8) == (4, 3) assert packing.group(32) == (16, 3) patterns = [0] * 16 @@ -63,30 +64,6 @@ def test_scales_use_common_bit_packing(count, block_k, word): assert scales.word_bits == count * 8 -def test_storage_rows_offsets_and_empty_views(): - storage = TensorStorage("bf6", (3, 5), row_stride_bytes=8, base_bit_offset=3) - assert storage.dtype == "fp6e3m2" - assert storage.byte_size == 21 - assert storage.address(2, 4) == (19, 3) - assert TensorStorage("fp4", (0, 5)).byte_size == 0 - assert TensorStorage("fp4", (3, 0), base_bit_offset=7).byte_size == 0 - assert BitPacking(6).pack([], bit_offset=7) == b"" - assert TensorStorage("e8m0", (2, 8)).byte_size == 16 - with pytest.raises(ValueError, match="out of bounds"): - storage.address(3, 0) - with pytest.raises(ValueError, match="row stride"): - TensorStorage("fp6", (3, 5), row_stride_bytes=3) - with pytest.raises(ValueError, match="power of two"): - TensorStorage("fp4", (1, 4), alignment_bytes=3) - - -@pytest.mark.parametrize("dtype", ["f16", "bf16"]) -def test_storage_allows_byte_strides(dtype): - storage = TensorStorage(dtype, (2, 64), row_stride_bytes=129) - assert storage.address(1, 0) == (129, 0) - assert storage.byte_size == 257 - - def test_invalid_packing_and_overflow(): for bits, slots in [(0, None), (65, None), (6, 4)]: with pytest.raises(ValueError): @@ -95,7 +72,5 @@ def test_invalid_packing_and_overflow(): FragmentPacking(BitPacking(6), 16, 32, 2) with pytest.raises(ValueError, match="uint64"): BitPacking(6).byte_size(1 << 63) - with pytest.raises(ValueError, match="uint64"): - TensorStorage("fp8", (1 << 63, 3)) with pytest.raises(ValueError, match="fit"): BitPacking(6).pack([64]) From 2bc6e0ff5d2a4cc12331f139ac0366e052def007 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:11:52 -0400 Subject: [PATCH 11/32] fix(rocke): tighten packing scope and prepare complete native tests --- .../rocke/platform/cpp/helpers/mma_io.cpp | 3 ++ .../rocke/helper_rocke.helpers.mma_io.h | 1 + .../cpp/include/rocke/wmma_scale_internal.h | 15 ++------ .../architecture/dtype_storage_packing.md | 9 ++--- .../python/rocke/core/arch/wmma_scale.py | 18 ++-------- .../platform/python/rocke/helpers/mma_io.py | 9 +++-- .../rocke/platform/tests/core/test_mma_io.py | 9 +++++ .../platform/tests/core/test_storage.cpp | 14 ++++++++ .../rocke/platform/tests/core/test_storage.py | 2 +- .../rocke/platform/tests/run_all.py | 4 +-- .../platform/tests/tools/test_run_all.py | 34 +++++++++++++++++-- 11 files changed, 77 insertions(+), 41 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp index 92a4e6a93396..b0a1c5e7ba94 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp @@ -160,6 +160,9 @@ rocke_status_t rocke_h_pack_fragment_bits(rocke_ir_builder_t* b, if(f.carrier_bits != 32 && f.carrier_bits != 64) ckc::raise_status(ROCKE_ERR_VALUE, "IR pattern packing currently requires i32 or i64 carriers"); + if(f.packing.element_bits > 32) + ckc::raise_status(ROCKE_ERR_VALUE, + "IR pattern packing supports encoded fields of at most 32 bits"); const auto* word_type = f.carrier_bits == 64 ? rocke_i64() : rocke_i32(); auto constant = [&](uint64_t value) { return f.carrier_bits == 64 ? rocke_b_const_i64(b, int64_t(value)) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.mma_io.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.mma_io.h index 7dea7828c6f6..7e70c670cf08 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.mma_io.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.mma_io.h @@ -33,6 +33,7 @@ rocke_value_t* rocke_h_load_matrix_fragment(rocke_ir_builder_t* b, /* Callback returns a canonical unsigned pattern in an integer carrier. * ctx and output pointer array are borrowed for the duration of this call. */ typedef rocke_value_t* (*rocke_load_bits_fn)(rocke_ir_builder_t* b, int index, void* ctx); +/* Encoded fields are at most 32 bits; i64 carriers support eight-byte scales. */ rocke_status_t rocke_h_pack_fragment_bits(rocke_ir_builder_t* b, rocke_load_bits_fn load_bits, void* ctx, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h index 08585dd6b734..dcd0e389b07a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h @@ -24,23 +24,12 @@ typedef struct rocke_scale_packing int block_k; } rocke_scale_packing_t; -typedef struct rocke_scale_association -{ - int block_k; -} rocke_scale_association_t; - -static inline rocke_scale_association_t rocke_scale_association(const rocke_scale_packing_t* p) -{ - if(p->block_k <= 0) - ckc::raise_status(ROCKE_ERR_VALUE, "scale block_k must be positive"); - return {p->block_k}; -} - static inline rocke_fragment_packing_t rocke_scale_fragment(const rocke_scale_packing_t* p) { rocke_bit_packing_t bits; rocke_fragment_packing_t result; - rocke_scale_association(p); + if(p->block_k <= 0) + ckc::raise_status(ROCKE_ERR_VALUE, "scale block_k must be positive"); if(p->count != 1 && p->count != 2 && p->count != 4 && p->count != 8) ckc::raise_status(ROCKE_ERR_VALUE, "carrier_bits must be 8, 16, 32, or 64"); if(!rocke_bit_packing_init(&bits, rocke_dtype_info("e8m0")->encoded_bits, 0) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md index aeb0a863af35..0508701c82fe 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md @@ -109,15 +109,16 @@ i32 carriers. FP8 occupies all sixteen words. FP4 occupies eight and pads eight. The FP6 transport descriptor occupies twelve and pads four, without enabling FP6 catalog entries or numerical conversions. -`ScaleAssociation` records `block_k`: how many source K elements share one -scale. This is separate from the bit layout. Existing `ScalePacking(count, -block_k)` remains a compatibility adapter, exposing both the association and a -shared `FragmentPacking`. +`ScalePacking(count, block_k)` records how many source K elements share one +scale in `block_k` and exposes the byte layout through a shared `FragmentPacking`. +Scale association and bit layout remain separate properties of this contract. Current scaled-WMMA scale fragments hold four eight-bit patterns in i32 for K32, or eight in i64 for K16. The first K group occupies the low byte. The A/B scale coordinate maps determine which scales each lane loads. The same `pack_fragment_bits` helper can pack six-bit fields that cross word boundaries. +Its IR contract accepts encoded fields of at most 32 bits in i32 or i64 carriers; +wide encoded fields are outside the current operand requirements. ## Mirroring and follow-up work diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py index 7f0284ed983c..a506ad183e71 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py @@ -11,17 +11,6 @@ from ..storage import BitPacking, FragmentPacking, MatrixFragmentLayout -@dataclass(frozen=True) -class ScaleAssociation: - """Number of source K elements sharing one scale.""" - - block_k: int - - def __post_init__(self) -> None: - if self.block_k <= 0: - raise ValueError("scale block_k must be positive") - - def scaled_matrix_layout(dtype: str, abi_words: int) -> MatrixFragmentLayout: """gfx1250 scaled operand layout, independent of atom availability. @@ -51,13 +40,10 @@ class ScalePacking: block_k: int def __post_init__(self) -> None: - self.association + if self.block_k <= 0: + raise ValueError("scale block_k must be positive") self.fragment - @property - def association(self) -> ScaleAssociation: - return ScaleAssociation(self.block_k) - @property def packing(self) -> BitPacking: return BitPacking(dtype_info("e8m0").encoded_bits) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py index 28bfdf60c316..6972121da6a3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py @@ -124,11 +124,16 @@ def pack_fragment_bits( ) -> list[Value]: """Pack unsigned encoded patterns into integer carriers, including split fields. - The loader must return canonical patterns with zero high bits. Tensor decoding - and numeric quantization are separate operations. + Encoded fields are at most 32 bits; i64 carriers support eight-byte scale + words. The loader must return canonical patterns with zero high bits. Tensor + decoding and numeric quantization are separate operations. """ if fragment.carrier_bits not in (32, 64): raise ValueError("IR pattern packing currently requires i32 or i64 carriers") + if fragment.packing.element_bits > 32: + raise ValueError( + "IR pattern packing supports encoded fields of at most 32 bits" + ) word_type = I64 if fragment.carrier_bits == 64 else I32 constant = b.const_i64 if fragment.carrier_bits == 64 else b.const_i32 words = [constant(0) for _ in range(fragment.carrier_count)] diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py index 67caa9541dcf..c084230d9773 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py @@ -308,3 +308,12 @@ def test_quant_type_diagnostic_matches_native(): text=True, ) assert result.stdout == str(error.value) + + +def test_reject_wide_encoded_fields_before_emission(): + b = IRBuilder("unsupported_packing") + fragment = FragmentPacking(BitPacking(40), 2, 64, 2) + before = len(b.kernel.body.ops) + with pytest.raises(ValueError, match="encoded fields of at most 32 bits"): + pack_fragment_bits(b, lambda j: b.const_i64(1 << 35), fragment) + assert len(b.kernel.body.ops) == before diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp index de507eba1c34..c224e5d9a24a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp @@ -229,6 +229,20 @@ int main(int argc, char** argv) rocke_ir_builder_free(&b); return 0; } + { + rocke_ir_builder_t b; + CHECK(rocke_ir_builder_init(&b, "unsupported_packing") == ROCKE_OK); + rocke_bit_packing_t packing; + rocke_fragment_packing_t fragment; + rocke_value_t* words[2]; + CHECK(rocke_bit_packing_init(&packing, 40, 0)); + CHECK(rocke_fragment_packing_init(&fragment, &packing, 2, 64, 2)); + CHECK(rocke_h_pack_fragment_bits(&b, load_bits, NULL, &fragment, words, 2) + == ROCKE_ERR_VALUE); + CHECK(strstr(rocke_ir_builder_error(&b), "encoded fields of at most 32 bits")); + CHECK(b.kernel->body->num_ops == 0); + rocke_ir_builder_free(&b); + } CHECK(test_fragment_inputs() == 0); CHECK(test_fragment_offset_bounds() == 0); CHECK(rocke_dtype_info("e4m3") == rocke_dtype_info("fp8e4m3")); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py index 5284914cc389..cb572e5af990 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py @@ -58,7 +58,7 @@ def test_matrix_payload_and_padding(dtype, live): ) def test_scales_use_common_bit_packing(count, block_k, word): scales = ScalePacking(count, block_k) - assert scales.association.block_k == block_k + assert scales.block_k == block_k assert scales.fragment.pack(list(range(1, count + 1))) == (word,) assert scales.packing == BitPacking(8) assert scales.word_bits == count * 8 diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py index 7f4b90c6124e..51e12f36ea9a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py @@ -116,7 +116,7 @@ def ctest_ready(build_root: Path, config: str) -> bool: def native_pytest_env(build_root: Path, config: str) -> dict[str, str]: - """Make the native storage comparisons available to both pytest passes.""" + """Build the configured suite and supply native parity to both pytest passes.""" env = dict(os.environ) executable = env.get("ROCKE_STORAGE_TEST") if not executable and (build_root / "CMakeCache.txt").is_file(): @@ -127,8 +127,6 @@ def native_pytest_env(build_root: Path, config: str) -> dict[str, str]: str(build_root), "--config", config, - "--target", - "rocke_storage", ], check=True, ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py index d9dbd4f08bdd..484503960776 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py @@ -6,6 +6,7 @@ import json from pathlib import Path import subprocess +import shutil import pytest @@ -67,8 +68,6 @@ def run(command, **kwargs): str(tmp_path), "--config", "Debug", - "--target", - "rocke_storage", ] assert any(cmd[:3] == ["ctest", "-C", "Debug"] for cmd, _ in calls) @@ -188,3 +187,34 @@ def run(command, **kwargs): ) assert runner.main() == 1 assert all(command[0] != runner.sys.executable for command in calls) + + +def test_fresh_build_prepares_entire_ctest_suite(runner, tmp_path): + if not shutil.which("cmake") or not shutil.which("ctest"): + pytest.skip("CMake and CTest required for fresh-build regression") + source = tmp_path / "source" + source.mkdir() + (source / "main.c").write_text("int main(void) { return 0; }\n") + (source / "CMakeLists.txt").write_text( + "cmake_minimum_required(VERSION 3.20)\n" + "project(runner_fixture C)\n" + "enable_testing()\n" + "add_library(rocke_core STATIC main.c)\n" + "add_executable(rocke_storage main.c)\n" + "add_executable(rocke_dtypes main.c)\n" + "add_test(NAME rocke_storage COMMAND rocke_storage)\n" + "add_test(NAME rocke_dtypes COMMAND rocke_dtypes)\n" + ) + build = tmp_path / "build" + subprocess.run(["cmake", "-S", str(source), "-B", str(build)], check=True) + # Reproduce the byte-identity gate's partial build before pytest setup. + subprocess.run( + ["cmake", "--build", str(build), "--target", "rocke_core"], check=True + ) + env = runner.native_pytest_env(build, "Release") + assert Path(env["ROCKE_STORAGE_TEST"]).is_file() + assert runner.ctest_ready(build, "Release") + subprocess.run( + ["ctest", "--test-dir", str(build), "-C", "Release", "--output-on-failure"], + check=True, + ) From 2732998629265d810793a677f4974733f7c00fea Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:33:39 -0400 Subject: [PATCH 12/32] docs(rocke): clarify complete native suite setup in test runner --- .../hip-kernel-provider/rocke/platform/tests/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md b/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md index 870e008c00f7..82ffdd97c593 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md @@ -109,9 +109,9 @@ divergence - the Python builder correctly rejects wave32 WMMA on gfx942.) ### Native storage parity in the standard runner -`run_all.py --build-root ` builds the `rocke_storage` target in a configured -build and obtains its executable path from CTest. Both pytest passes receive that -path, so storage IR/HIP parity and serialization tests run automatically. +`run_all.py --build-root ` builds all configured targets before pytest, +then obtains the `rocke_storage` executable path from CTest. Both pytest passes +receive that path, so storage IR/HIP parity and serialization tests run automatically. `--config` selects the native test configuration (default `Release`). A build or fixture-discovery failure stops the runner instead of silently skipping coverage. From af73df5ea70b7eaff40602a476ceeec755540ae6 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:43:09 -0400 Subject: [PATCH 13/32] feat(rocke): support contiguous 96-bit global and LDS loads --- .../rocke/platform/cpp/core/ir/ir_mem.cpp | 12 ++-- .../rocke/platform/cpp/core/ir/ir_tile.cpp | 10 +-- .../cpp/core/lower_hip/lower_hip_mem.cpp | 19 +++++- .../platform/cpp/core/lower_llvm/mem.cpp | 3 +- .../architecture/dtype_storage_packing.md | 8 +++ .../rocke/platform/python/rocke/core/ir.py | 41 ++++++------ .../platform/python/rocke/core/lower_hip.py | 14 +++- .../platform/python/rocke/core/lower_llvm.py | 3 +- .../rocke/platform/tests/core/test_mma_io.py | 64 ++++++++++++++++++- .../platform/tests/core/test_storage.cpp | 36 ++++++++++- 10 files changed, 169 insertions(+), 41 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_mem.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_mem.cpp index 2d2f89d72065..b07e81cb6b16 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_mem.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_mem.cpp @@ -227,14 +227,14 @@ rocke_value_t* rocke_b_global_load_vN(rocke_ir_builder_t* b, || rocke_i_type_is(dtype, "i16")) { elem_bytes = 2; - if(n != 2 && n != 4 && n != 8 && n != 16) + if(n != 2 && n != 4 && n != 6 && n != 8 && n != 16) return (rocke_value_t*)rocke_i_set_err( b, ROCKE_ERR_VALUE, "unsupported vector width for global_load_vN: %d", n); } else if(rocke_i_type_is(dtype, "f32") || rocke_i_type_is(dtype, "i32")) { elem_bytes = 4; - if(n != 2 && n != 4 && n != 8) + if(n != 2 && n != 3 && n != 4 && n != 8) return (rocke_value_t*)rocke_i_set_err( b, ROCKE_ERR_VALUE, "unsupported vector width for %s global_load_vN: %d", en, n); } @@ -242,7 +242,7 @@ rocke_value_t* rocke_b_global_load_vN(rocke_ir_builder_t* b, || rocke_i_type_is(dtype, "i8")) { elem_bytes = 1; - if(n != 2 && n != 4 && n != 8 && n != 16) + if(n != 2 && n != 4 && n != 8 && n != 12 && n != 16) return (rocke_value_t*)rocke_i_set_err( b, ROCKE_ERR_VALUE, "unsupported vector width for %s global_load_vN: %d", en, n); } @@ -262,7 +262,11 @@ rocke_value_t* rocke_b_global_load_vN(rocke_ir_builder_t* b, a = rocke_i_attrs(b); rocke_attr_set_str(b, &a, "elem_type", en); rocke_attr_set_int(b, &a, "vec", (int64_t)n); - rocke_attr_set_int(b, &a, "align", (int64_t)(align > 0 ? align : n * elem_bytes)); + rocke_attr_set_int( + b, + &a, + "align", + (int64_t)(align > 0 ? align : (n * elem_bytes == 12 ? elem_bytes : n * elem_bytes))); { char hint[16]; /* result_name_hint = "gv{n}" */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp index 8ebb8bb6191e..0585df2a5aab 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp @@ -349,8 +349,9 @@ rocke_value_t* rocke_b_smem_load_vN(rocke_ir_builder_t* b, const rocke_type_t* vt; rocke_attr_map_t attrs; const char* dn; - static const int allowed_8bit[] = {1, 2, 4, 8, 16}; - static const int allowed_other[] = {1, 2, 4, 8}; + static const int allowed_8bit[] = {1, 2, 4, 8, 12, 16}; + static const int allowed_16bit[] = {1, 2, 4, 6, 8}; + static const int allowed_32bit[] = {1, 2, 3, 4, 8}; char hint[16]; if(!rocke_i_live(b)) { @@ -375,8 +376,9 @@ rocke_value_t* rocke_b_smem_load_vN(rocke_ir_builder_t* b, { bool eight = (strcmp(dn, "fp8e4m3") == 0 || strcmp(dn, "bf8e5m2") == 0 || strcmp(dn, "i8") == 0); - const int* allowed = eight ? allowed_8bit : allowed_other; - int acount = eight ? 5 : 4; + bool half = strcmp(dn, "f16") == 0 || strcmp(dn, "bf16") == 0; + const int* allowed = eight ? allowed_8bit : half ? allowed_16bit : allowed_32bit; + int acount = eight ? 6 : 5; if(!rocke_n_in(n, allowed, acount)) { return (rocke_value_t*)rocke_i_set_err( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mem.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mem.cpp index cdfee832aad9..60129168ffcd 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mem.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mem.cpp @@ -335,6 +335,21 @@ static rocke_status_t _op_tile_smem_load_vN(rocke_h_lowerer_t* lw, const rocke_o } idx_str = mem_idx_join(lw, &op->operands[1], op->num_operands - 1); res = rocke_h_name(lw, op->results[0]); + const int64_t byte_count = n * (rocke_dtype_info(elem_name)->encoded_bits / 8); + if(byte_count & (byte_count - 1)) + { + /* Clang pads vector objects; copy only the actual LDS payload. */ + rocke_h_emitf(lw, + "%s%lld %s; __builtin_memcpy(&%s, &%s[%s], %lld);", + prefix, + (long long)n, + res, + res, + storage, + idx_str, + (long long)byte_count); + return lw->status; + } rocke_h_emitf(lw, "%s%lld %s = *reinterpret_cast(&%s[%s]);", prefix, @@ -536,9 +551,9 @@ static rocke_status_t _op_memref_global_load_vN(rocke_h_lowerer_t* lw, const roc if(align <= 0 || (align & (align - 1))) return rocke_h_fail( lw, ROCKE_ERR_VALUE, "global_load_vN: alignment must be a positive power of two"); - if(align < byte_count) + if(align < byte_count || (byte_count & (byte_count - 1))) { - /* Do not strengthen the IR's alignment through a vector-pointer cast. */ + /* Copy only the payload, not vector padding, with the IR's alignment. */ rocke_h_emitf( lw, "%s%lld %s; __builtin_memcpy(&%s, __builtin_assume_aligned(%s + %s, %lld), %lld);", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mem.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mem.cpp index 1bb057802dab..5cf265459213 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mem.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mem.cpp @@ -570,7 +570,8 @@ static void op_tile_smem_load_vN(rocke_lower_t* L, const rocke_op_t* op) elem_bytes = 8; } } - int64_t align = vec * elem_bytes; + /* New 96-bit widths guarantee only element alignment, including FP8. */ + int64_t align = (vec == 3 || vec == 6 || vec == 12) ? 12 / vec : vec * elem_bytes; /* gfx1250: vec==8 loads are marked volatile to block the WMMA-aware backend * pass from substituting ds_load_tr16_b128 (transposed) for the plain * sequential ds_read_b128. Mirrors Python _op_tile_smem_load_vN lines diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md index 0508701c82fe..5d15503e83d1 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md @@ -86,6 +86,14 @@ origin alignment, carrier capacity, and static i32 displacement limits. It reduces the supplied alignment for chunk spacing and K origin. A 24-byte FP6 chunk becomes 16-byte and 8-byte loads without reading past the chunk. +The existing `global_load_vN` and `smem_load_vN` builders also accept contiguous +96-bit payloads: twelve byte elements, six 16-bit elements, or three 32-bit +elements of their supported types. These loads default to element alignment; +HIP copies exactly twelve bytes, excluding vector-object padding. A single +96-bit machine instruction depends on target and alignment. This does not +change the fragment loader's chunk selection. FP6 transpose loads are deferred +until tensor-descriptor integration defines their lane/layout contract. + Tensor shape, strides, row selection, and allocation bounds belong to the caller. This helper always loads a complete fragment: the caller must guard empty or partial rows and provide a valid lane group. It is not a bounds-checked diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py index 73f01b156cfc..0ebe58891330 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py @@ -1564,34 +1564,30 @@ def global_load_vN( ) -> Value: """Vectorised global load of N consecutive values. - Supports the full element-type catalog the LLVM lowering already - accepts: ``f16`` / ``bf16`` (N in {2, 4, 8}), ``f32`` / ``i32`` - (N in {2, 4, 8}), ``i16`` (N in {2, 4, 8}), ``fp8e4m3`` / - ``bf8e5m2`` / ``i8`` (N in {2, 4, 8, 16}). - - Lowers to a single ``load `` from ``addrspace(1)``; - AMDGPU's backend coalesces these into a single VMEM transaction - (``global_load_dwordxN``) when the address is naturally aligned. - - The per-element size is folded into the default alignment so the - common case (8 fp8 → 8-byte load, 4 f32 → 16-byte load) does - not need an explicit ``align=`` kwarg. + Supports f16/bf16/i16 (N in {2, 4, 6, 8, 16}), f32/i32 + (N in {2, 3, 4, 8}), and fp8e4m3/bf8e5m2/i8 (N in {2, 4, 8, 12, 16}). + Loads exactly N elements. Instruction selection depends on target and + alignment; 96-bit payloads do not require a 96-bit scalar type. + + Default alignment is the payload size for power-of-two loads, and + element alignment for 12-byte loads. An explicit alignment is a caller + guarantee about the address after adding idx. """ if dtype.name in ("f16", "bf16", "i16"): elem_bytes = 2 # n=16 (32-byte `global_load_dwordx8`) is needed for the RDNA WMMA # <16 x half> operand fragment; AMDGPU coalesces it when aligned. - if n not in (2, 4, 8, 16): + if n not in (2, 4, 6, 8, 16): raise ValueError(f"unsupported vector width for global_load_vN: {n}") elif dtype.name in ("f32", "i32"): elem_bytes = 4 - if n not in (2, 4, 8): + if n not in (2, 3, 4, 8): raise ValueError( f"unsupported vector width for {dtype.name} global_load_vN: {n}" ) elif dtype.name in ("fp8e4m3", "bf8e5m2", "i8"): elem_bytes = 1 - if n not in (2, 4, 8, 16): + if n not in (2, 4, 8, 12, 16): raise ValueError( f"unsupported vector width for {dtype.name} global_load_vN: {n}" ) @@ -1607,7 +1603,9 @@ def global_load_vN( attrs={ "elem_type": dtype.name, "vec": n, - "align": int(align or (n * elem_bytes)), + "align": int( + align or (elem_bytes if n * elem_bytes == 12 else n * elem_bytes) + ), }, result_name_hint=f"gv{n}", ).result @@ -1814,10 +1812,9 @@ def smem_load_vN_f16(self, smem: Value, *indices, n: int = 0) -> Value: def smem_load_vN(self, smem: Value, *indices, dtype: Type, n: int = 0) -> Value: """LDS load of ````. Supports 8-bit (fp8e4m3 / bf8e5m2 / i8), 16-bit (f16 / bf16) and 32-bit (f32 / i32) element types; - AMDGPU lowers vector LDS loads to ``ds_read_b{8, 16, 32, 64, 128}`` - based on total payload size. The 8-bit variants must use ``n in {1, - 2, 4, 8, 16}`` so the resulting payload still maps to a single - ``ds_read_b*`` instruction (n=16 → ds_read_b128). + loads exactly N elements. In addition to power-of-two widths, accepts + 96-bit payloads (12 bytes, six halfwords, or three words), using element + alignment. The target and alignment determine instruction selection. """ if dtype.name not in ("f16", "bf16", "f32", "i32", "fp8e4m3", "bf8e5m2", "i8"): raise ValueError( @@ -1825,9 +1822,9 @@ def smem_load_vN(self, smem: Value, *indices, dtype: Type, n: int = 0) -> Value: f"bf8e5m2 / i8, got {dtype.name}" ) allowed_n = ( - (1, 2, 4, 8, 16) + (1, 2, 4, 8, 12, 16) if dtype.name in ("fp8e4m3", "bf8e5m2", "i8") - else (1, 2, 4, 8) + else (1, 2, 4, 6, 8) if dtype.name in ("f16", "bf16") else (1, 2, 3, 4, 8) ) if n not in allowed_n: raise ValueError( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py index 1352f07217ff..18ecd1a9b322 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py @@ -493,9 +493,9 @@ def _op_memref_global_load_vN(self, op: Op) -> None: raise ValueError( "global_load_vN: alignment must be a positive power of two" ) - if align < byte_count: - # A vector-pointer dereference would promise natural alignment. - # Copy the payload using only the alignment guaranteed by the IR. + if align < byte_count or byte_count & (byte_count - 1): + # Non-power-of-two vector objects include padding. Copy only the + # payload, using only the alignment guaranteed by the IR. self._emit( f"{prefix}{vec} {_name(op.result)}; " f"__builtin_memcpy(&{_name(op.result)}, " @@ -1839,6 +1839,14 @@ def _op_tile_smem_load_vN(self, op: Op) -> None: if storage is None: raise RuntimeError("smem load_vN before smem_alloc was lowered") idx_str = "][".join(_name(i) for i in indices) + byte_count = n * (dtype_info(elem_name).encoded_bits // 8) + if byte_count & (byte_count - 1): + # Clang rounds vector object sizes up; LDS payloads have no padding. + self._emit( + f"{prefix}{n} {_name(op.result)}; " + f"__builtin_memcpy(&{_name(op.result)}, &{storage}[{idx_str}], {byte_count});" + ) + return self._emit( f"{prefix}{n} {_name(op.result)} = " f"*reinterpret_cast(&{storage}[{idx_str}]);" diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py index 4743a1335ea8..ebf67fd5af12 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py @@ -3101,7 +3101,8 @@ def _op_tile_smem_load_vN(self, op: Op) -> None: op.result.type.elem.name, 2, # type: ignore[attr-defined] ) - align = vec * elem_bytes + # New 96-bit widths guarantee only element alignment, including FP8. + align = 12 // vec if vec in (3, 6, 12) else vec * elem_bytes # gfx1250: mark 8-wide (128-bit) LDS loads volatile to block the WMMA-aware # pass from substituting ds_load_tr16_b128 (transposed) in place of the plain # sequential ds_read_b128. Only 8-wide loads feed the 16x16x32 WMMA fragment diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py index c084230d9773..8b2897f0edef 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py @@ -11,7 +11,7 @@ import pytest from rocke.core.arch.wmma_scale import scaled_matrix_layout -from rocke.core.ir import FP8E4M3, I8, I32, I64, IRBuilder, PtrType, VectorType +from rocke.core.ir import F16, FP8E4M3, I8, I32, I64, IRBuilder, PtrType, VectorType from rocke.core.ir_serialize import parse, serialize from rocke.core.lower_llvm import lower_kernel_to_llvm from rocke.core.lower_hip import lower_kernel_to_hip @@ -28,6 +28,9 @@ CASES = [ + "load96_i8", + "load96_f16", + "load96_i32", "fp4", "fp6", "fp6_padded", @@ -47,6 +50,28 @@ def build_transport(dtype): b = IRBuilder("transport") + if dtype.startswith("load96_"): + unit, n = { + "load96_i8": (I8, 12), + "load96_f16": (F16, 6), + "load96_i32": (I32, 3), + }[dtype] + a = b.param("A", PtrType(unit, "global")) + o = b.param("O", PtrType(unit, "global")) + one = b.const_i32(1) + value = b.global_load_vN(a, one, unit, n) + smem = b.smem_alloc(unit, (n + 1,), name_hint="payload") + for j in range(n): + b.smem_store_vN(smem, [b.const_i32(j + 1)], b.vec_extract(value, j), 1) + b.s_barrier_bare() + value = b.smem_load_vN(smem, one, dtype=unit, n=n) + aligned = b.global_load_vN(a, b.const_i32(0), unit, n, align=16) + for k, vector in enumerate((value, aligned)): + for j in range(n): + b.global_store( + o, b.const_i32(k * n + j), b.vec_extract(vector, j), align=12 // n + ) + return b.kernel padded = dtype.endswith("_padded") dtype = dtype.removesuffix("_padded") patterns = dtype.startswith("pack_") @@ -317,3 +342,40 @@ def test_reject_wide_encoded_fields_before_emission(): with pytest.raises(ValueError, match="encoded fields of at most 32 bits"): pack_fragment_bits(b, lambda j: b.const_i64(1 << 35), fragment) assert len(b.kernel.body.ops) == before + + +@pytest.mark.parametrize("dtype", ["load96_i8", "load96_f16", "load96_i32"]) +def test_96bit_loads_copy_exact_payload(dtype): + """Offset and over-aligned global loads plus an LDS tail must not read padding.""" + kernel = build_transport(dtype) + llvm = lower_kernel_to_llvm(kernel, arch="gfx1250", llvm_flavor="llvm23") + n, elem, alignment = { + "load96_i8": (12, "i8", 1), + "load96_f16": (6, "half", 2), + "load96_i32": (3, "i32", 4), + }[dtype] + for space in (1, 3): + assert re.search( + rf"load <{n} x {elem}>, ptr addrspace\({space}\).*align {alignment}\b", llvm + ) + hip = lower_kernel_to_hip(kernel, arch="gfx1250") + assert len(re.findall(r"__builtin_memcpy\([^\n]*, 12\);", hip)) == 3 + if not shutil.which("hipcc"): + pytest.skip("hipcc not in PATH") + from rocke.helpers.compile import emit_device_llvm_ir_via_hipcc + + compiled = emit_device_llvm_ir_via_hipcc(kernel, arch="gfx950", extra_flags=["-O0"]) + # Clang's vector object is 16 bytes; the source copy must remain 12 bytes. + compiled = compiled.split("define protected amdgpu_kernel void @transport(", 1)[ + 1 + ].split("\n}", 1)[0] + source_copies = re.findall( + r"@llvm\.memcpy[^\n]*?\([^,]+,\s*ptr[^,]*\balign (\d+) [^,]+,\s*i64 (\d+)", + compiled, + ) + assert len(source_copies) == 3 + for (actual_align, size), max_align in zip( + source_copies, (alignment, alignment, 16) + ): + assert int(size) == 12 + assert int(actual_align) <= max_align diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp index c224e5d9a24a..716855565fb7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp @@ -40,15 +40,45 @@ static int emit(const char* dtype, bool hip) : strcmp(dtype, "bf16_padded") == 0 ? "bf16" : "fp6"; bool patterns = strncmp(dtype, "pack_", 5) == 0; - const auto* unit = patterns ? rocke_i8() : rocke_storage_ir_type(dtype); + const bool load96 = strncmp(dtype, "load96_", 7) == 0; + const auto* unit = load96 ? (strcmp(dtype, "load96_i8") == 0 ? rocke_i8() + : strcmp(dtype, "load96_f16") == 0 ? rocke_f16() + : rocke_i32()) + : patterns ? rocke_i8() + : rocke_storage_ir_type(dtype); CHECK(unit); const bool typed = strcmp(dtype, "f16") == 0 || strcmp(dtype, "bf16") == 0; - const auto* carrier = typed ? unit : rocke_i32(); + const auto* carrier = (typed || load96) ? unit : rocke_i32(); if(strcmp(dtype, "pack_scale_bytes_i64") == 0) carrier = rocke_i64(); auto* a = rocke_b_param(&b, "A", rocke_ptr_type(&b, unit, "global"), NULL); auto* o = rocke_b_param(&b, "O", rocke_ptr_type(&b, carrier, "global"), NULL); - if(patterns) + if(load96) + { + const int n = unit == rocke_i8() ? 12 : unit == rocke_f16() ? 6 : 3; + auto* one = rocke_b_const_i32(&b, 1); + auto* value = rocke_b_global_load_vN(&b, a, one, unit, n, 0); + int shape[] = {n + 1}; + auto* smem = rocke_b_smem_alloc(&b, unit, shape, 1, "payload"); + for(int j = 0; j < n; ++j) + { + rocke_value_t* indices[] = {rocke_b_const_i32(&b, j + 1)}; + auto* element = rocke_b_vec_extract(&b, value, j); + rocke_b_smem_store_vN(&b, smem, indices, 1, element, 1); + } + rocke_b_s_barrier_bare(&b); + value = rocke_b_smem_load_vN(&b, smem, &one, 1, unit, n); + auto* aligned = rocke_b_global_load_vN(&b, a, rocke_b_const_i32(&b, 0), unit, n, 16); + rocke_value_t* vectors[] = {value, aligned}; + for(int k = 0; k < 2; ++k) + for(int j = 0; j < n; ++j) + { + auto* index = rocke_b_const_i32(&b, k * n + j); + auto* element = rocke_b_vec_extract(&b, vectors[k], j); + rocke_b_global_store(&b, o, index, element, 12 / n); + } + } + else if(patterns) { int bits = strcmp(dtype, "pack_fp6_cross_word") == 0 ? 6 : 8; int count = bits == 6 ? 16 : (strcmp(dtype, "pack_scale_bytes_i64") == 0 ? 8 : 4); From 33f4d963c4e2db2bb4268456e400004526d47362 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:25:40 -0400 Subject: [PATCH 14/32] fix(rocke): reject unsupported fragment padding before emission --- .../platform/python/rocke/helpers/mma_io.py | 6 +++--- .../rocke/platform/tests/core/test_mma_io.py | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py index 6972121da6a3..aa0a498b3c38 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py @@ -68,6 +68,9 @@ def load_matrix_fragment( raise ValueError("matrix carrier type width mismatch") if packing.payload_bits % packing.carrier_bits: raise ValueError("matrix fragment payload must occupy whole carriers") + padding = packing.carrier_count - packing.live_carriers + if padding and carrier_type != I32: + raise ValueError("padded matrix fragments currently require i32 carriers") chunk_units = layout.chunk_bytes // unit_bytes origin_bytes = origin_bits // 8 # Bound the last loaded unit across all chunks and lane groups, including k0. @@ -109,10 +112,7 @@ def load_matrix_fragment( payload_type = VectorType(carrier_type, packing.live_carriers) if payload.type != payload_type: payload = b.bitcast(payload, payload_type) - padding = packing.carrier_count - packing.live_carriers if padding: - if carrier_type != I32: - raise ValueError("padded matrix fragments currently require i32 carriers") payload = b.vec_concat(payload, b.vector_splat(b.const_i32(0), padding)) return payload diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py index 8b2897f0edef..b5099a2085a1 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py @@ -168,11 +168,15 @@ def test_reject_bit_origin(): @pytest.mark.parametrize( "dtype,alignment,error", - [("fp6", 3, "positive power of two"), ("fp4", 1, "packing width mismatch")], + [ + ("fp6", 3, "positive power of two"), + ("fp4", 1, "packing width mismatch"), + ("f16", 2, "padded matrix fragments currently require i32 carriers"), + ], ) def test_fragment_input_contract(dtype, alignment, error): b = IRBuilder("invalid_fragment") - ptr = b.param("A", PtrType(I8, "global")) + ptr = b.param("A", PtrType(storage_ir_type(dtype), "global")) zero = b.const_i32(0) before = serialize(b.kernel) with pytest.raises(ValueError, match=error): @@ -183,7 +187,14 @@ def test_fragment_input_contract(dtype, alignment, error): zero, 0, dtype=dtype, - layout=scaled_matrix_layout("fp6", 16), + layout=( + MatrixFragmentLayout( + FragmentPacking(BitPacking(16), 8, 16, 16), 8, 1, 1 + ) + if dtype == "f16" + else scaled_matrix_layout("fp6", 16) + ), + carrier_type=F16 if dtype == "f16" else I32, alignment_bytes=alignment, ) assert serialize(b.kernel) == before From b03f86c95cac8122b875896cbd61dd479a29076c Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Thu, 24 Sep 2026 23:18:18 -0400 Subject: [PATCH 15/32] fix(rocke): preserve vector aliases in bare HIP output --- .../platform/cpp/core/lower_hip/core.cpp | 6 +-- .../platform/python/rocke/core/lower_hip.py | 8 ++-- .../rocke/platform/tests/core/test_mma_io.py | 43 ++++++++++++++++++- .../platform/tests/core/test_storage.cpp | 6 ++- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/core.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/core.cpp index 039b225b394e..de9b2d5cb110 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/core.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/core.cpp @@ -1047,10 +1047,10 @@ rocke_status_t rocke_lower_kernel_to_hip(rocke_ir_builder_t* b, { rocke_strbuf_append(out, ROCKE_HIP_PROLOGUE); rocke_strbuf_append_char(out, '\n'); - for(int j = 0; j < kernel->num_params; ++j) - h_extra_vector_type(&lw, kernel->params[j]->type, out); - h_extra_vector_region(&lw, kernel->body, out); } + for(int j = 0; j < kernel->num_params; ++j) + h_extra_vector_type(&lw, kernel->params[j]->type, out); + h_extra_vector_region(&lw, kernel->body, out); /* head */ rocke_strbuf_appendf(out, "extern \"C\" __global__ __launch_bounds__(%d)\n" diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py index 18ecd1a9b322..ab29bc41586b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py @@ -2513,9 +2513,9 @@ def lower_kernel_to_hip( The output is: 1. The :data:`HIP_PROLOGUE` (typedefs + ```` include + AMDGPU vector typedefs). Disable with - ``include_prologue=False`` when you want only the body text - (e.g. for embedding into a larger TU that already has these - typedefs). + ``include_prologue=False`` for embedding into a larger TU that already + has the shared prologue. Required per-kernel vector typedefs are still + emitted. 2. The kernel's ``__global__`` signature, derived from :attr:`KernelDef.params`. Pointer params get ``__restrict__``; ``__launch_bounds__`` is taken from @@ -2568,7 +2568,7 @@ def lower_kernel_to_hip( parts: List[str] = [] if include_prologue: parts.append(HIP_PROLOGUE) - parts.extend(_extra_vector_declarations(kernel)) + parts.extend(_extra_vector_declarations(kernel)) parts.append(head) if smem_block: parts.append(smem_block) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py index b5099a2085a1..9821489b72c5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py @@ -14,7 +14,7 @@ from rocke.core.ir import F16, FP8E4M3, I8, I32, I64, IRBuilder, PtrType, VectorType from rocke.core.ir_serialize import parse, serialize from rocke.core.lower_llvm import lower_kernel_to_llvm -from rocke.core.lower_hip import lower_kernel_to_hip +from rocke.core.lower_hip import HIP_PROLOGUE, lower_kernel_to_hip from rocke.core.storage import ( BitPacking, FragmentPacking, @@ -248,6 +248,47 @@ def test_hip_declares_only_encountered_missing_vector_widths(): assert "using i8x24 =" not in source +@pytest.mark.parametrize("route", ["python", "native"]) +def test_bare_hip_keeps_required_vector_declarations(route, tmp_path): + source = lower_kernel_to_hip( + build_transport("load96_i8"), arch="gfx1250", include_prologue=False + ) + if route == "native": + executable = os.environ.get("ROCKE_STORAGE_TEST") + if not executable: + pytest.skip("set ROCKE_STORAGE_TEST to the built native storage test") + actual = subprocess.run( + [executable, "--hip-bare", "load96_i8"], + check=True, + capture_output=True, + text=True, + ).stdout + assert actual == source + source = actual + assert "#include" not in source + assert source.count("using i8x12 =") == 1 + if not shutil.which("hipcc"): + pytest.skip("hipcc not in PATH") + path = tmp_path / "composed.hip" + path.write_text( + HIP_PROLOGUE + + source + + source.replace("void transport(", "void transport_second(") + ) + subprocess.run( + [ + "hipcc", + "--offload-arch=gfx950", + "--cuda-device-only", + "-fsyntax-only", + str(path), + ], + check=True, + capture_output=True, + text=True, + ) + + @pytest.mark.parametrize("dtype", ["fp8", "bf8"]) def test_byte_width_does_not_erase_pointer_identity(dtype): assert storage_ir_type(dtype) != I8 diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp index 716855565fb7..53c530399a0d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp @@ -30,7 +30,7 @@ static rocke_value_t* load_bits(rocke_ir_builder_t* b, int j, void* ctx) return rocke_b_global_load(b, (rocke_value_t*)ctx, rocke_b_const_i32(b, j), rocke_i8(), 1); } -static int emit(const char* dtype, bool hip) +static int emit(const char* dtype, bool hip, bool include_prologue = true) { rocke_ir_builder_t b; CHECK(rocke_ir_builder_init(&b, "transport") == ROCKE_OK); @@ -127,6 +127,8 @@ static int emit(const char* dtype, bool hip) CHECK(rocke_strbuf_init(&text, 256) == 0); rocke_lower_hip_opts_t opts = {}; opts.arch = "gfx1250"; + opts.include_prologue = include_prologue; + opts.include_prologue_set = true; CHECK(rocke_lower_kernel_to_hip(&b, b.kernel, &opts, &text) == ROCKE_OK); fputs(rocke_strbuf_cstr(&text), stdout); rocke_strbuf_free(&text); @@ -229,6 +231,8 @@ int main(int argc, char** argv) return emit(argv[2], false); if(argc == 3 && strcmp(argv[1], "--hip") == 0) return emit(argv[2], true); + if(argc == 3 && strcmp(argv[1], "--hip-bare") == 0) + return emit(argv[2], true, false); if(argc == 3 && strcmp(argv[1], "--parse") == 0) { rocke_ir_builder_t b; From e47c76de700871a801e12a04b282832bd9dbd268 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 25 Sep 2026 00:19:47 -0400 Subject: [PATCH 16/32] fix(rocke): load packed matrix fragments into word carriers --- .../rocke/platform/cpp/helpers/mma_io.cpp | 26 +++++++++++++---- .../architecture/dtype_storage_packing.md | 7 +++++ .../platform/python/rocke/helpers/mma_io.py | 28 +++++++++++++++---- .../rocke/platform/tests/core/test_mma_io.py | 23 +++++++++++++-- .../platform/tests/core/test_storage.cpp | 2 ++ 5 files changed, 74 insertions(+), 12 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp index b0a1c5e7ba94..b19383b4668f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp @@ -90,6 +90,11 @@ rocke_value_t* rocke_h_load_matrix_fragment(rocke_ir_builder_t* b, / layout->lane_groups) ckc::raise_status(ROCKE_ERR_VALUE, "matrix fragment offset exceeds i32 range"); int alignment = gcd(alignment_bytes, gcd(chunk_bytes, origin_bytes)); + // Use 2--4-word vectors; retain storage-element loads for a one-word tail. + const bool word_loads = unit_bytes == 1 && rocke_type_eq(carrier_type, rocke_i32()) + && chunk_bytes % 4 == 0 && chunk_bytes % 16 != 4; + const auto* load_type = word_loads ? rocke_i32() : unit_type; + const int load_step = word_loads ? 4 : 1; auto* lane_chunk = rocke_b_mul(b, lane_group, rocke_b_const_i32(b, chunk_units)); auto* step_base = rocke_b_add(b, row_base, rocke_b_const_i32(b, origin_bytes / unit_bytes)); /* Collect loads before concatenation to preserve Python SSA numbering. */ @@ -112,16 +117,27 @@ rocke_value_t* rocke_h_load_matrix_fragment(rocke_ir_builder_t* b, int width = 1; while(width < max_width && uint64_t(width * 2) <= remaining) width *= 2; + if(word_loads) + width = remaining / 4 < 4 ? int(remaining / 4) : 4; auto* at = consumed ? rocke_b_add(b, offset, rocke_b_const_i32(b, consumed)) : offset; int load_align = gcd(alignment, consumed * unit_bytes); + // Form the byte address before loading words; preserve signed i32 offsets. + auto* load_ptr + = word_loads ? rocke_b_global_ptr_add(b, ptr, rocke_b_sext(b, at, rocke_i64())) + : ptr; + auto* load_at = word_loads ? rocke_b_const_i32(b, 0) : at; auto* value - = width == 1 ? rocke_b_vector_splat( - b, rocke_b_global_load(b, ptr, at, unit_type, load_align), 1) - : rocke_b_global_load_vN(b, ptr, at, unit_type, width, load_align); + = width == 1 + ? rocke_b_vector_splat( + b, + rocke_b_global_load(b, load_ptr, load_at, load_type, load_align), + 1) + : rocke_b_global_load_vN( + b, load_ptr, load_at, load_type, width, load_align); chunks[num_chunks++] = value; - consumed += width; - remaining -= width; + consumed += width * load_step; + remaining -= width * load_step; } } auto* payload = chunks[0]; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md index 5d15503e83d1..0180817073e3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md @@ -85,6 +85,13 @@ The loader checks dtype/packing width, pointer identity, whole-unit chunk and origin alignment, carrier capacity, and static i32 displacement limits. It reduces the supplied alignment for chunk spacing and K origin. A 24-byte FP6 chunk becomes 16-byte and 8-byte loads without reading past the chunk. +For byte-stored operands with i32 carriers, the loader selects up to four words +per load. It produces i32 vectors directly when every load has at least two +words. A 12-byte chunk produces three i32 lanes (`i32x3` in HIP), with the +original byte-address alignment. Address calculation remains in bytes, and the +loaded words concatenate into the operand fragment without intermediate byte +vectors. Typed chunks, partial words, and one-word tails retain their storage +element loads and final carrier bitcast when needed. The existing `global_load_vN` and `smem_load_vN` builders also accept contiguous 96-bit payloads: twelve byte elements, six 16-bit elements, or three 32-bit diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py index aa0a498b3c38..ebcfc0e21a42 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/mma_io.py @@ -82,6 +82,15 @@ def load_matrix_fragment( ): raise ValueError("matrix fragment offset exceeds i32 range") alignment = gcd(alignment_bytes, layout.chunk_bytes, origin_bytes) + # Use 2--4-word vectors; retain storage-element loads for a one-word tail. + word_loads = ( + unit_bytes == 1 + and carrier_type == I32 + and layout.chunk_bytes % 4 == 0 + and layout.chunk_bytes % 16 != 4 + ) + load_type = I32 if word_loads else unit_type + load_step = 4 if word_loads else 1 lane_chunk = b.mul(lane_group, b.const_i32(chunk_units)) step_base = b.add(row_base, b.const_i32(origin_bytes // unit_bytes)) chunks = [] @@ -94,18 +103,27 @@ def load_matrix_fragment( consumed = 0 max_width = 8 if unit_bytes == 4 else 16 while remaining: - width = min(max_width, 1 << (remaining.bit_length() - 1)) + width = ( + min(4, remaining // 4) + if word_loads + else min(max_width, 1 << (remaining.bit_length() - 1)) + ) at = b.add(offset, b.const_i32(consumed)) if consumed else offset load_align = gcd(alignment, consumed * unit_bytes) + # Form the byte address before loading words; preserve signed i32 offsets. + load_ptr = b.global_ptr_add(ptr, b.sext(at, I64)) if word_loads else ptr + load_at = b.const_i32(0) if word_loads else at if width == 1: value = b.vector_splat( - b.global_load(ptr, at, unit_type, align=load_align), 1 + b.global_load(load_ptr, load_at, load_type, align=load_align), 1 ) else: - value = b.global_load_vN(ptr, at, unit_type, width, align=load_align) + value = b.global_load_vN( + load_ptr, load_at, load_type, width, align=load_align + ) chunks.append(value) - consumed += width - remaining -= width + consumed += width * load_step + remaining -= width * load_step payload = chunks[0] for chunk in chunks[1:]: payload = b.vec_concat(payload, chunk) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py index 9821489b72c5..59e1bbf1b433 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py @@ -98,6 +98,8 @@ def build_transport(dtype): if typed else scaled_matrix_layout(dtype, 16) ) + if dtype == "fp6" and padded: + layout = MatrixFragmentLayout(layout.fragment, 16, 2, 16) base = b.const_i32((129 if typed else 97) if padded else 0) lane = b.mod(b.thread_id_x(), b.const_i32(32)) group = b.div(lane, b.const_i32(16)) @@ -239,9 +241,24 @@ def test_fragment_offset_bounds(dtype, count, groups, k0, valid): def test_hip_declares_only_encountered_missing_vector_widths(): + for case, widths in (("fp6", [4, 2, 4, 2]), ("fp6_padded", [3, 3, 3, 3])): + kernel = build_transport(case) + loads = [op for op in kernel.body.ops if op.name == "memref.global_load_vN"] + assert [op.result.type for op in loads] == [VectorType(I32, n) for n in widths] + assert not any(op.name == "arith.bitcast" for op in kernel.body.ops) + for count in (4, 20): + b = IRBuilder("scalar_tail") + ptr = b.param("A", PtrType(I8, "global")) + zero = b.const_i32(0) + layout = MatrixFragmentLayout( + FragmentPacking(BitPacking(8), count, 32, count // 4), count, 1, 1 + ) + load_matrix_fragment(b, ptr, zero, zero, 0, dtype="e8m0", layout=layout) + assert not any(op.name == "tile.global_ptr_add" for op in b.kernel.body.ops) source = lower_kernel_to_hip(build_transport("fp6"), arch="gfx1250") - for name in ("i8x24", "i8x40", "i32x12"): + for name in ("i32x6", "i32x10", "i32x12"): assert source.count(f"using {name} =") == 1 + assert "using i8x" not in source assert "using f16x32 =" not in source source = lower_kernel_to_hip(build_transport("f16"), arch="gfx1250") assert source.count("using f16x32 =") == 1 @@ -319,7 +336,7 @@ def test_hip_rejects_unsupported_vector_element(elem_type): @pytest.mark.parametrize( "dtype,copy_bytes,alignment", - [("fp6", 16, 8), ("fp6_padded", 16, 1), ("f16", 32, 16)], + [("fp6", 16, 8), ("fp6_padded", 12, 1), ("f16", 32, 16)], ) def test_hip_compiler_preserves_fragment_load_alignment(dtype, copy_bytes, alignment): """Inspect compiler IR: source parity cannot detect a shared alignment bug.""" @@ -339,6 +356,8 @@ def test_hip_compiler_preserves_fragment_load_alignment(dtype, copy_bytes, align if int(size) == copy_bytes ] assert source_alignments, "expected unaligned-safe fragment payload copies" + if dtype == "fp6_padded": + assert len(source_alignments) == 4 assert all(value <= alignment for value in source_alignments), source_alignments diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp index 53c530399a0d..d170d81d4dd2 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp @@ -106,6 +106,8 @@ static int emit(const char* dtype, bool hip, bool include_prologue = true) } else layout = rocke_scaled_matrix_layout(dtype, 16); + if(strcmp(dtype, "fp6") == 0 && padded) + CHECK(rocke_matrix_fragment_layout_init(&layout, &layout.fragment, 16, 2, 16)); auto* base = rocke_b_const_i32(&b, padded ? (typed ? 129 : 97) : 0); auto* thread = rocke_b_thread_id_x(&b); auto* lane = rocke_b_mod(&b, thread, rocke_b_const_i32(&b, 32)); From 4c7816df47ed5ed6bc60b4e98fc5ae51dc2d27d6 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 25 Sep 2026 02:26:05 -0400 Subject: [PATCH 17/32] fix(rocke): preserve override builds and scalar quant dtype contract --- .../platform/python/rocke/helpers/__init__.py | 2 ++ .../platform/python/rocke/helpers/quant.py | 11 ++++--- .../rocke/platform/tests/README.md | 6 ++-- .../rocke/platform/tests/core/test_dtypes.py | 16 ++++++++++ .../rocke/platform/tests/run_all.py | 21 +++++++----- .../platform/tests/tools/test_run_all.py | 32 ++++++++++++++++--- 6 files changed, 68 insertions(+), 20 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/__init__.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/__init__.py index 0398a767bd09..4e015d6d7411 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/__init__.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/__init__.py @@ -318,6 +318,7 @@ host_preshuffle_layout, ) from .quant import ( + LogicalQDType, QDType, QUANT_MAX_ABS, dequantize_scalar_to_f32, @@ -622,6 +623,7 @@ "mfma_attention_bwd_dq_dk_dv_inner_body", "streamk_num_macro_tiles", # Quantisation (f32 <-> {i8, fp8e4m3, bf8e5m2}) + "LogicalQDType", "QDType", "QUANT_MAX_ABS", "dequantize_scalar_to_f32", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py index eb4604c48809..4b59538287c2 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/quant.py @@ -21,9 +21,10 @@ This module exposes the small set of helpers every quantised op needs: * :class:`QDType` — the output dtype literal alias used in spec dataclasses. +* :class:`LogicalQDType` — recognized types, including FP4/FP6 without scalar conversion. * :data:`QUANT_MAX_ABS` — clamp bound per output type (matches CK Tile's ``ComputeDataType{127.0f}`` / ``448.0f`` / ``57344.0f`` literals). -* :func:`quant_ir_type` — map a ``QDType`` to the IR :class:`Type`. +* :func:`quant_ir_type` — map a recognized dtype string to the IR :class:`Type`. * :func:`quantize_scalar_f32` — the one-element quant fast path (``cvt_f32_to_(clamp(x * inv_scale, -max, max))``). * :func:`dequantize_scalar_to_f32` — the dual; auto-dispatches on the @@ -37,7 +38,7 @@ from __future__ import annotations -from typing import List, Literal +from typing import List, Literal, TypeAlias from ..core.ir import ( BF8E5M2, @@ -54,6 +55,7 @@ __all__ = [ + "LogicalQDType", "QDType", "QUANT_MAX_ABS", "dequantize_scalar_to_f32", @@ -66,7 +68,8 @@ ] -QDType = Literal["i8", "fp8e4m3", "bf8e5m2", "fp4e2m1", "fp6e2m3", "fp6e3m2"] +QDType: TypeAlias = Literal["i8", "fp8e4m3", "bf8e5m2"] +LogicalQDType: TypeAlias = Literal[QDType, "fp4e2m1", "fp6e2m3", "fp6e3m2"] # Per-dtype clamp magnitude (the largest representable absolute value). @@ -154,7 +157,7 @@ def quant_max_abs(qdtype: str) -> float: return QUANT_MAX_ABS[canon] -def ir_to_qdtype(t: Type) -> QDType: +def ir_to_qdtype(t: Type) -> LogicalQDType: """Inverse of :func:`quant_ir_type`. Rejects non-quant types.""" if t.name in _IR_TO_QDTYPE: return _IR_TO_QDTYPE[t.name] # type: ignore[return-value] diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md b/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md index 82ffdd97c593..e4e5da4de8bd 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md @@ -116,6 +116,6 @@ receive that path, so storage IR/HIP parity and serialization tests run automati fixture-discovery failure stops the runner instead of silently skipping coverage. An explicit `ROCKE_STORAGE_TEST` overrides discovery and must name an existing -executable. With no configured build or explicit override, the runner reports -native storage parity as skipped; direct pytest invocations can use the same -override. +executable; it does not skip the configured build. With no configured build or +explicit override, the runner reports native storage parity as skipped; direct +pytest invocations can use the same override. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py index 2fd2eb0ac7e9..f5018fbeef28 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_dtypes.py @@ -3,6 +3,7 @@ """Core dtype names are independent of target support and retain public aliases.""" import ast +from typing import get_args, get_type_hints import pytest @@ -149,3 +150,18 @@ def test_type_alias_does_not_enable_scalar_quantization(dtype): assert quant_ir_type(dtype) is not None with pytest.raises(ValueError, match="unsupported quant dtype"): quant_max_abs(dtype) + + +def test_quant_dtype_annotations_match_scalar_and_logical_contracts(): + from rocke import helpers + from rocke.helpers import quant + + scalar = {"i8", "fp8e4m3", "bf8e5m2"} + assert set(get_args(quant.QDType)) == scalar + assert all(quant_max_abs(dtype) > 0 for dtype in get_args(quant.QDType)) + logical = scalar | {"fp4e2m1", "fp6e2m3", "fp6e3m2"} + assert set(get_args(quant.LogicalQDType)) == logical + assert get_type_hints(ir_to_qdtype)["return"] == quant.LogicalQDType + assert all(ir_to_qdtype(quant_ir_type(dtype)) == dtype for dtype in logical) + assert helpers.QDType is quant.QDType + assert helpers.LogicalQDType is quant.LogicalQDType diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py index 51e12f36ea9a..e39834b58a34 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py @@ -119,7 +119,7 @@ def native_pytest_env(build_root: Path, config: str) -> dict[str, str]: """Build the configured suite and supply native parity to both pytest passes.""" env = dict(os.environ) executable = env.get("ROCKE_STORAGE_TEST") - if not executable and (build_root / "CMakeCache.txt").is_file(): + if (build_root / "CMakeCache.txt").is_file(): subprocess.run( [ "cmake", @@ -130,13 +130,18 @@ def native_pytest_env(build_root: Path, config: str) -> dict[str, str]: ], check=True, ) - tests = ctest_tests(build_root, config, "^rocke_storage$") - commands = [ - test.get("command", []) for test in tests if test["name"] == "rocke_storage" - ] - if len(commands) != 1 or not commands[0]: - raise ValueError("CTest did not resolve the built rocke_storage executable") - executable = commands[0][0] + if not executable: + tests = ctest_tests(build_root, config, "^rocke_storage$") + commands = [ + test.get("command", []) + for test in tests + if test["name"] == "rocke_storage" + ] + if len(commands) != 1 or not commands[0]: + raise ValueError( + "CTest did not resolve the built rocke_storage executable" + ) + executable = commands[0][0] if executable: path = Path(executable).resolve() if not path.is_file(): diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py index 484503960776..eed98eb130ac 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py @@ -25,14 +25,19 @@ def runner(monkeypatch): @pytest.mark.parametrize( "filename", ["rocke_storage", "provider_rocke_storage_test.exe"] ) +@pytest.mark.parametrize("override", [False, True]) def test_runner_passes_registered_fixture_to_both_backends( - runner, monkeypatch, tmp_path, filename + runner, monkeypatch, tmp_path, filename, override ): (tmp_path / "CMakeCache.txt").touch() (tmp_path / "CTestTestfile.cmake").touch() executable = tmp_path / "Debug" / filename executable.parent.mkdir() executable.touch() + selected = tmp_path / "override" if override else executable + if override: + selected.touch() + monkeypatch.setenv("ROCKE_STORAGE_TEST", str(selected)) calls = [] def run(command, **kwargs): @@ -59,7 +64,7 @@ def run(command, **kwargs): (cmd, kw["env"]) for cmd, kw in calls if cmd[0] == runner.sys.executable ] assert len(children) == 3 # default pytest, extension import probe, both pytest - assert all(env["ROCKE_STORAGE_TEST"] == str(executable) for _, env in children) + assert all(env["ROCKE_STORAGE_TEST"] == str(selected) for _, env in children) assert "ROCKE_BACKEND" not in children[0][1] assert children[-1][1]["ROCKE_BACKEND"] == "both" assert calls[0][0] == [ @@ -70,6 +75,7 @@ def run(command, **kwargs): "Debug", ] assert any(cmd[:3] == ["ctest", "-C", "Debug"] for cmd, _ in calls) + assert any("-R" in cmd for cmd, _ in calls) is not override @pytest.mark.parametrize("built", ["none", "partial", "all"]) @@ -164,16 +170,22 @@ def test_invalid_override_is_an_error(runner, monkeypatch, tmp_path): runner.native_pytest_env(tmp_path, "Release") -@pytest.mark.parametrize("failure", ["build", "registration", "executable"]) +@pytest.mark.parametrize( + "failure", ["build", "build_override", "registration", "executable"] +) def test_native_setup_failure_prevents_silently_skipped_pytest( runner, monkeypatch, tmp_path, failure ): (tmp_path / "CMakeCache.txt").touch() + if failure == "build_override": + executable = tmp_path / "override" + executable.touch() + monkeypatch.setenv("ROCKE_STORAGE_TEST", str(executable)) calls = [] def run(command, **kwargs): calls.append(command) - if command[0] == "cmake" and failure == "build": + if command[0] == "cmake" and failure.startswith("build"): raise subprocess.CalledProcessError(1, command) command_path = [str(tmp_path / "missing")] if failure == "executable" else [] listing = {"tests": [{"name": "rocke_storage", "command": command_path}]} @@ -189,7 +201,10 @@ def run(command, **kwargs): assert all(command[0] != runner.sys.executable for command in calls) -def test_fresh_build_prepares_entire_ctest_suite(runner, tmp_path): +@pytest.mark.parametrize("override", [False, True]) +def test_fresh_build_prepares_entire_ctest_suite( + runner, monkeypatch, tmp_path, override +): if not shutil.which("cmake") or not shutil.which("ctest"): pytest.skip("CMake and CTest required for fresh-build regression") source = tmp_path / "source" @@ -211,8 +226,15 @@ def test_fresh_build_prepares_entire_ctest_suite(runner, tmp_path): subprocess.run( ["cmake", "--build", str(build), "--target", "rocke_core"], check=True ) + if override: + # An external fixture must not suppress the configured suite's build. + executable = tmp_path / "override" + executable.touch() + monkeypatch.setenv("ROCKE_STORAGE_TEST", str(executable)) env = runner.native_pytest_env(build, "Release") assert Path(env["ROCKE_STORAGE_TEST"]).is_file() + if override: + assert env["ROCKE_STORAGE_TEST"] == str(executable) assert runner.ctest_ready(build, "Release") subprocess.run( ["ctest", "--test-dir", str(build), "-C", "Release", "--output-on-failure"], From 6e536d15c1a3d27abb10f0313063a41922bc43a9 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 25 Sep 2026 12:35:21 -0400 Subject: [PATCH 18/32] fix(rocke): bound fragment load scratch capacity Size temporary load-pointer storage by addressable units so padded slots cannot overrun the array. Preserve IR order and add mirrored transport coverage for a layout requiring more loads than logical elements. Simplify scale-width queries while retaining the shared packing adapter, and clarify the contiguous-K matrix layout contract. --- .../rocke/platform/cpp/helpers/mma_io.cpp | 5 ++-- .../platform/cpp/include/rocke/storage.h | 2 ++ .../cpp/include/rocke/wmma_scale_internal.h | 21 ++++++------- .../architecture/dtype_storage_packing.md | 4 ++- .../python/rocke/core/arch/wmma_scale.py | 13 ++++---- .../platform/python/rocke/core/storage.py | 2 ++ .../rocke/platform/tests/core/test_mma_io.py | 9 ++++++ .../platform/tests/core/test_storage.cpp | 30 +++++++++++++++++++ .../rocke/platform/tests/core/test_storage.py | 12 +++++++- 9 files changed, 77 insertions(+), 21 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp index b19383b4668f..9d8230329102 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mma_io.cpp @@ -97,9 +97,10 @@ rocke_value_t* rocke_h_load_matrix_fragment(rocke_ir_builder_t* b, const int load_step = word_loads ? 4 : 1; auto* lane_chunk = rocke_b_mul(b, lane_group, rocke_b_const_i32(b, chunk_units)); auto* step_base = rocke_b_add(b, row_base, rocke_b_const_i32(b, origin_bytes / unit_bytes)); - /* Collect loads before concatenation to preserve Python SSA numbering. */ + /* Collect loads before concatenation to preserve Python SSA numbering. + * Each load consumes at least one storage unit, including padded slots. */ rocke_value_t** chunks = (rocke_value_t**)rocke_arena_alloc( - &b->arena, size_t(packing.count) * sizeof(rocke_value_t*)); + &b->arena, size_t(payload_bits / (8 * unit_bytes)) * sizeof(rocke_value_t*)); if(!chunks) ckc::raise_status(ROCKE_ERR_OOM, "matrix fragment allocation failed"); int num_chunks = 0; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h index 9634dbcc2e1f..37a3ee9be0c9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/storage.h @@ -31,6 +31,8 @@ typedef struct rocke_fragment_packing uint64_t carrier_count; } rocke_fragment_packing_t; +/* Interleaved contiguous K chunks, not arbitrary matrix distributions, + * transposed axes, or swizzled tensor addresses. */ typedef struct rocke_matrix_fragment_layout { rocke_fragment_packing_t fragment; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h index dcd0e389b07a..34ae13074d7b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h @@ -24,16 +24,22 @@ typedef struct rocke_scale_packing int block_k; } rocke_scale_packing_t; -static inline rocke_fragment_packing_t rocke_scale_fragment(const rocke_scale_packing_t* p) +static inline int rocke_scale_word_bits(const rocke_scale_packing_t* p) { - rocke_bit_packing_t bits; - rocke_fragment_packing_t result; if(p->block_k <= 0) ckc::raise_status(ROCKE_ERR_VALUE, "scale block_k must be positive"); if(p->count != 1 && p->count != 2 && p->count != 4 && p->count != 8) ckc::raise_status(ROCKE_ERR_VALUE, "carrier_bits must be 8, 16, 32, or 64"); - if(!rocke_bit_packing_init(&bits, rocke_dtype_info("e8m0")->encoded_bits, 0) - || !rocke_fragment_packing_init(&result, &bits, p->count, p->count * 8, 1)) + return p->count * 8; +} + +static inline rocke_fragment_packing_t rocke_scale_fragment(const rocke_scale_packing_t* p) +{ + const int word_bits = rocke_scale_word_bits(p); + rocke_bit_packing_t bits; + rocke_fragment_packing_t result; + if(!rocke_bit_packing_init(&bits, 8, 0) + || !rocke_fragment_packing_init(&result, &bits, p->count, word_bits, 1)) ckc::raise_status(ROCKE_ERR_VALUE, "invalid scale packing"); return result; } @@ -145,9 +151,4 @@ static inline const rocke_mma_op_t* rocke_gfx1250_scaled_wmma_from_op(const rock return rocke_gfx1250_scaled_wmma(op_id ? op_id : op->name); } -static inline int rocke_scale_word_bits(const rocke_scale_packing_t* packing) -{ - return rocke_scale_fragment(packing).carrier_bits; -} - #endif /* ROCKE_WMMA_SCALE_INTERNAL_H */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md index 0180817073e3..ab5a8349a6f6 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/dtype_storage_packing.md @@ -116,7 +116,9 @@ buffers; concurrent packed stores are not provided. `MatrixFragmentLayout` describes contiguous K chunks interleaved between lane groups. Coordinate mapping belongs to the atom-specific layout, not to the -generic bit packer. `ScaledWmmaOp.matrix_layout(operand)` selects independent +generic bit packer. This is a contiguous-K mapping, not a general matrix +distribution or a description of transposed axes and swizzled tensor addresses. +`ScaledWmmaOp.matrix_layout(operand)` selects independent A/B layouts. Existing `a_frag_len`/`b_frag_len` retain their ABI-vector meaning. The current gfx1250 scaled matrix layouts have 64 elements per lane and sixteen diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py index a506ad183e71..3b327b1b4133 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py @@ -42,25 +42,24 @@ class ScalePacking: def __post_init__(self) -> None: if self.block_k <= 0: raise ValueError("scale block_k must be positive") - self.fragment + if not isinstance(self.count, int) or self.count not in (1, 2, 4, 8): + raise ValueError("carrier_bits must be 8, 16, 32, or 64") @property def packing(self) -> BitPacking: - return BitPacking(dtype_info("e8m0").encoded_bits) + return BitPacking(self.element_bits) @property def fragment(self) -> FragmentPacking: - return FragmentPacking( - self.packing, self.count, self.count * self.packing.element_bits, 1 - ) + return FragmentPacking(self.packing, self.count, self.word_bits, 1) @property def element_bits(self) -> int: - return self.packing.element_bits + return 8 @property def word_bits(self) -> int: - return self.fragment.carrier_bits + return self.count * self.element_bits @property def llvm_type(self) -> str: diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/storage.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/storage.py index c9c1ce3282e9..e0770a939c1c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/storage.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/storage.py @@ -127,6 +127,8 @@ class MatrixFragmentLayout: """Contiguous K chunks interleaved between lane groups. The atom selects this mapping. Storage and carrier widths do not determine it. + This describes interleaved contiguous K chunks, not arbitrary matrix + distributions, transposed axes, or swizzled tensor addresses. """ fragment: FragmentPacking diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py index 59e1bbf1b433..96c3e7e930fe 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_io.py @@ -32,6 +32,7 @@ "load96_f16", "load96_i32", "fp4", + "fp4_slot40", "fp6", "fp6_padded", "bf6", @@ -72,6 +73,9 @@ def build_transport(dtype): o, b.const_i32(k * n + j), b.vec_extract(vector, j), align=12 // n ) return b.kernel + padded_slots = dtype == "fp4_slot40" + if padded_slots: + dtype = "fp4" padded = dtype.endswith("_padded") dtype = dtype.removesuffix("_padded") patterns = dtype.startswith("pack_") @@ -100,6 +104,11 @@ def build_transport(dtype): ) if dtype == "fp6" and padded: layout = MatrixFragmentLayout(layout.fragment, 16, 2, 16) + if padded_slots: + # Four logical elements need eight loads (4 + 1 bytes per slot). + layout = MatrixFragmentLayout( + FragmentPacking(BitPacking(4, 40), 4, 32, 5), 1, 2, 16 + ) base = b.const_i32((129 if typed else 97) if padded else 0) lane = b.mod(b.thread_id_x(), b.const_i32(32)) group = b.div(lane, b.const_i32(16)) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp index d170d81d4dd2..185c6f12d3db 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.cpp @@ -34,6 +34,9 @@ static int emit(const char* dtype, bool hip, bool include_prologue = true) { rocke_ir_builder_t b; CHECK(rocke_ir_builder_init(&b, "transport") == ROCKE_OK); + const bool padded_slots = strcmp(dtype, "fp4_slot40") == 0; + if(padded_slots) + dtype = "fp4"; const bool padded = strstr(dtype, "_padded") != NULL; if(padded) dtype = strcmp(dtype, "f16_padded") == 0 ? "f16" @@ -108,6 +111,15 @@ static int emit(const char* dtype, bool hip, bool include_prologue = true) layout = rocke_scaled_matrix_layout(dtype, 16); if(strcmp(dtype, "fp6") == 0 && padded) CHECK(rocke_matrix_fragment_layout_init(&layout, &layout.fragment, 16, 2, 16)); + if(padded_slots) + { + // Four logical elements need eight loads (4 + 1 bytes per slot). + rocke_bit_packing_t packing; + rocke_fragment_packing_t fragment; + CHECK(rocke_bit_packing_init(&packing, 4, 40)); + CHECK(rocke_fragment_packing_init(&fragment, &packing, 4, 32, 5)); + CHECK(rocke_matrix_fragment_layout_init(&layout, &fragment, 1, 2, 16)); + } auto* base = rocke_b_const_i32(&b, padded ? (typed ? 129 : 97) : 0); auto* thread = rocke_b_thread_id_x(&b); auto* lane = rocke_b_mod(&b, thread, rocke_b_const_i32(&b, 32)); @@ -281,6 +293,24 @@ int main(int argc, char** argv) } CHECK(test_fragment_inputs() == 0); CHECK(test_fragment_offset_bounds() == 0); + for(int count : {1, 2, 4, 8}) + { + const rocke_scale_packing_t scales = {count, 32}; + CHECK(rocke_scale_word_bits(&scales) == count * 8); + CHECK(rocke_scale_fragment(&scales).carrier_bits == count * 8); + } + for(const auto scales : {rocke_scale_packing_t{0, 32}, {-1, 32}, {3, 32}, {16, 32}, {4, 0}}) + { + try + { + rocke_scale_word_bits(&scales); + CHECK(false); + } + catch(const ckc::Error& error) + { + CHECK(error.code() == ROCKE_ERR_VALUE); + } + } CHECK(rocke_dtype_info("e4m3") == rocke_dtype_info("fp8e4m3")); CHECK(rocke_dtype_to_ir_type("e4m3") == rocke_fp8e4m3()); CHECK(rocke_storage_ir_type("e4m3") == rocke_fp8e4m3()); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py index cb572e5af990..476276e2fc1e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py @@ -54,7 +54,8 @@ def test_matrix_payload_and_padding(dtype, live): @pytest.mark.parametrize( - "count,block_k,word", [(4, 32, 0x04030201), (8, 16, 0x0807060504030201)] + "count,block_k,word", + [(1, 32, 0x01), (2, 32, 0x0201), (4, 32, 0x04030201), (8, 16, 0x0807060504030201)], ) def test_scales_use_common_bit_packing(count, block_k, word): scales = ScalePacking(count, block_k) @@ -62,6 +63,15 @@ def test_scales_use_common_bit_packing(count, block_k, word): assert scales.fragment.pack(list(range(1, count + 1))) == (word,) assert scales.packing == BitPacking(8) assert scales.word_bits == count * 8 + assert scales.llvm_type == f"i{count * 8}" + + +@pytest.mark.parametrize( + "count,block_k", [(0, 32), (-1, 32), (3, 32), (16, 32), (4.0, 32), (4, 0)] +) +def test_invalid_scale_packing(count, block_k): + with pytest.raises(ValueError): + ScalePacking(count, block_k) def test_invalid_packing_and_overflow(): From 8e865495634e199786661826c5f60d671a5bbe76 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Tue, 29 Sep 2026 20:47:36 -0400 Subject: [PATCH 19/32] test(rocke): validate FP6 GEMM with FP16 output --- .../python/rocke/examples/gfx1250/gemm/FP6.md | 7 +++++ .../gfx1250/gemm/block_scaled_gemm_verify.py | 30 ++++++++++++------- .../test_gfx1250_scaled_wmma_reference.py | 24 +++++++++++++++ .../instances/test_gfx1250_mxfp6_numeric.py | 27 +++++++++++++++++ 4 files changed, 78 insertions(+), 10 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/FP6.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/FP6.md index eb90364dfa9b..853db03a116c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/FP6.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/FP6.md @@ -65,6 +65,13 @@ isolated K groups, K=256, with E8M0 scales. These are bounded correctness fixtures; they do not establish arbitrary-input rounding, E8M0 NaN behavior, or performance. +The numerical suite covers BF16 output throughout and FP16 output for mixed-scale +`M=32, N=48, K=256` cases across both six-bit formats, SCALE/SCALE16, and +COMGR/HIP compilation. The shared `run_cases` verifier selects reference rounding +and output-buffer decoding from `BlockScaledGemmSpec.dtype_c`; BF16 remains the +default. The FP16 cases exercise the existing builder output mode without a +separate launch script. + ```sh ROCKE_LLVM_FLAVOR=llvm23 ROCKE_BACKEND=both ROCKE_CPP_STRICT=1 \ ROCKE_REQUIRE_GFX1250=1 python -m pytest -q -s \ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py index 03fb53e39d3d..78ee429e18a1 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py @@ -5,7 +5,7 @@ The default invocation keeps the K=64 FP8/BF8 WMMA + FP32-scale verifier. Native ``--matrix-path wmma_scale`` / ``wmma_scale16`` use homogeneous FP8 or FP6 and E8M0 scales with K=32 / K=16 groups. Native fixtures cover K=128 or 256 and use -bounded dyadic values, permitting exact comparison after BF16 rounding. +bounded dyadic values, permitting exact comparison after output-type rounding. Run on visible HIP device 0 (must be gfx1250), for example:: @@ -86,6 +86,16 @@ def decode_fp6(packed: np.ndarray, dtype: str = "fp6") -> np.ndarray: return np.copysign(magnitude, np.where(codes & 32, -1.0, 1.0)) +def _output_dtype(dtype: str): + import ml_dtypes + + if dtype == "bf16": + return ml_dtypes.bfloat16 + if dtype in ("fp16", "f16"): + return np.float16 + raise ValueError(f"verifier output must be bf16/fp16 (got {dtype!r})") + + def reference_result( a: np.ndarray, b: np.ndarray, @@ -96,16 +106,17 @@ def reference_result( native: bool, dtype_a: str | None = None, dtype_b: str | None = None, + dtype_c: str = "bf16", ) -> np.ndarray: - """Expand scales onto logical A/B elements, multiply, then round to BF16. + """Expand scales onto logical A/B elements, multiply, then round the output. The native fixtures use quarter-integer inputs of magnitude <=1 and scales 2**[-2,3]. At K<=256, even the sum of absolute products fits in 2**22 units of 2**-8, so every FP32 partial sum is exact. Float64 host - arithmetic and a single BF16 rounding provide an independent oracle. + arithmetic and a single output-type rounding provide an independent oracle. FP6 all-code fixtures isolate one K element, avoiding accumulation error. """ - import ml_dtypes + output_dtype = _output_dtype(dtype_c) sa = decode_e8m0(a_scale) if native else a_scale.astype(np.float64) sb = decode_e8m0(b_scale) if native else b_scale.astype(np.float64) @@ -121,7 +132,7 @@ def matrix_values(data, dtype): scaled_b = b_values * np.repeat(sb.T, block_k, axis=1) ref = scaled_a @ scaled_b.T if native: - ref = ref.astype(ml_dtypes.bfloat16) + ref = ref.astype(output_dtype) return ref.astype(np.float32) @@ -235,10 +246,10 @@ def _u8_buffer(array: np.ndarray): def _launch(rt, fn, spec, inputs): - import ml_dtypes + output_dtype = _output_dtype(spec.dtype_c) # A NaN sentinel makes missing output stores fail, including expected zeros. - out = np.full((spec.M, spec.N), np.nan, dtype=ml_dtypes.bfloat16) + out = np.full((spec.M, spec.N), np.nan, dtype=output_dtype) allocations = [] try: for array in (*inputs, out): @@ -251,7 +262,7 @@ def _launch(rt, fn, spec, inputs): host_out = (ctypes.c_uint8 * int(out.nbytes))() rt.memcpy_d2h(host_out, allocations[-1], out.nbytes) return ( - np.frombuffer(bytes(host_out), dtype=ml_dtypes.bfloat16) + np.frombuffer(bytes(host_out), dtype=output_dtype) .reshape(spec.M, spec.N) .astype(np.float32) ) @@ -271,8 +282,6 @@ def run_cases( ok, reason = is_valid_spec(spec, arch="gfx1250") if not ok: raise ValueError(reason) - if spec.dtype_c != "bf16": - raise ValueError("this verifier requires BF16 output") native = spec.resolved_matrix_path() != "wmma" if native and spec.K not in (128, 256): raise ValueError("exact native fixtures require K=128 or K=256") @@ -305,6 +314,7 @@ def run_cases( native=native, dtype_a=spec.dtype_a, dtype_b=spec.dtype_b, + dtype_c=spec.dtype_c, ) label = ( f"{spec.resolved_matrix_path()}/{spec.dtype_a}/{compile_route}/{case} " diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma_reference.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma_reference.py index e93bd0514f33..ffd87a8944fa 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma_reference.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma_reference.py @@ -40,6 +40,30 @@ def test_reference_matches_hand_computed_group_scales(): check_result(wrong, expected, exact=True) +@pytest.mark.parametrize( + "dtype_c,expected", + [("bf16", 1.0), ("fp16", 1.0 + 2**-10), ("f16", 1.0 + 2**-10)], +) +def test_reference_rounds_to_selected_output_type(dtype_c, expected): + a = np.array([[1.0, 2**-10]], dtype=np.float32) + b = np.ones((1, 2), dtype=np.float32) + scale = np.array([[127]], dtype=np.uint8) + got = reference_result(a, b, scale, scale, 2, native=True, dtype_c=dtype_c) + np.testing.assert_array_equal(got, [[expected]]) + wrong_output_type = np.array( + [[1.0 + 2**-10 if dtype_c == "bf16" else 1.0]], dtype=np.float32 + ) + with pytest.raises(AssertionError, match="bad=1/1"): + check_result(wrong_output_type, got, exact=True) + + +def test_reference_rejects_unsupported_output_type(): + value = np.ones((1, 1), dtype=np.float32) + scale = np.array([[127]], dtype=np.uint8) + with pytest.raises(ValueError, match="output"): + reference_result(value, value, scale, scale, 1, native=True, dtype_c="fp32") + + @pytest.mark.parametrize("path,bk", [("wmma_scale", 32), ("wmma_scale16", 16)]) @pytest.mark.parametrize("k", [128, 256]) def test_fixtures_exercise_each_scale_group_and_high_bytes(path, bk, k): diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp6_numeric.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp6_numeric.py index d1fdc8de05ad..19a47b5f95da 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp6_numeric.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp6_numeric.py @@ -27,6 +27,33 @@ def test_numeric(numeric_case, dtype, matrix_path, route, m, n, k, case, count): numeric_case(dtype, matrix_path, route, m, n, k, case, count) +@pytest.mark.parametrize("dtype", ["fp6", "bf6"]) +@pytest.mark.parametrize("matrix_path", ["wmma_scale", "wmma_scale16"]) +@pytest.mark.parametrize("route", ["comgr", "hip"]) +def test_fp16_output_numeric(gpu_env, dtype, matrix_path, route): + script = """ +from rocke.instances.gfx1250.block_scaled_gemm import BlockScaledGemmSpec +from rocke.examples.gfx1250.gemm.block_scaled_gemm_verify import run_cases +import sys +dtype, path, route = sys.argv[1:] +spec = BlockScaledGemmSpec(name='fp6_fp16_output', M=32, N=48, K=256, + dtype_a=dtype, dtype_b=dtype, dtype_c='fp16', matrix_path=path, + scale_dtype='e8m0', block_k=16 if path == 'wmma_scale16' else 32) +assert run_cases(spec, ('mixed',), compile_route=route) == 1 +""" + result = subprocess.run( + [sys.executable, "-c", script, dtype, matrix_path, route], + env=gpu_env, + capture_output=True, + text=True, + timeout=300, + ) + output = result.stdout + result.stderr + assert result.returncode == 0, output + assert output.count("bad=0") == 1, output + print(output, end="") + + @pytest.mark.parametrize("dtype", ["fp6", "bf6"]) @pytest.mark.parametrize("matrix_path", ["wmma_scale", "wmma_scale16"]) @pytest.mark.parametrize("route", ["comgr", "hip"]) From 227bb9551d7c1dd7f0b11f4bc2f235d66b785be3 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:43:22 -0400 Subject: [PATCH 20/32] test(rocke): check scale fragment carrier width --- .../rocke/platform/tests/core/test_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py index 476276e2fc1e..d97df3bedf81 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_storage.py @@ -63,7 +63,7 @@ def test_scales_use_common_bit_packing(count, block_k, word): assert scales.fragment.pack(list(range(1, count + 1))) == (word,) assert scales.packing == BitPacking(8) assert scales.word_bits == count * 8 - assert scales.llvm_type == f"i{count * 8}" + assert scales.fragment.carrier_bits == count * 8 @pytest.mark.parametrize( From fb3b6b189e67956566303e3891908ace263516af Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Wed, 30 Sep 2026 23:27:31 -0400 Subject: [PATCH 21/32] feat(rocke): enable mixed scaled GEMM on shared operand storage --- .../rocke/platform/cpp/core/arch/data.cpp | 920 ++++++++++++++++++ .../rocke/core/arch/data/arch_specs.json | 44 +- .../platform/python/rocke/core/arch/target.py | 14 + .../rocke/examples/gfx1250/gemm/README.md | 12 + .../gfx1250/gemm/block_scaled_gemm_verify.py | 4 +- .../gfx1250/gemm/mixed_scaled_gemm.py | 50 + .../instances/gfx1250/block_scaled_gemm.py | 2 - .../gfx1250_mixed_scaled_wmma_llvm23.json | 42 + .../platform/tests/core/test_arch_mma_ssot.py | 19 +- .../core/test_gfx1250_fp4_scaled_wmma.py | 2 +- .../core/test_gfx1250_fp6_scaled_wmma.py | 25 - .../tests/core/test_gfx1250_mixed_example.py | 47 + .../core/test_gfx1250_mixed_scaled_wmma.py | 147 +++ .../tests/core/test_mma_catalog_lookup.cpp | 4 +- .../instances/parity/gfx1250_lowering_emit.c | 62 +- .../instances/parity/gfx1250_lowering_emit.py | 9 + .../test_gfx1250_mixed_scaled_gemm_numeric.py | 56 ++ 17 files changed, 1408 insertions(+), 51 deletions(-) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/mixed_scaled_gemm.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_mixed_scaled_wmma_llvm23.json create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_example.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mixed_scaled_gemm_numeric.py diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp index e4f8dabed748..88e13137c1d6 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp @@ -941,6 +941,46 @@ static const rocke_ati_mma_frag_row_t rocke_ati_mma_frag[] = { {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k16", 8}, {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k16", 8}, {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e8m0_k16", 8}, }; int rocke_arch_mma_c_frag_len(const char* op_id) @@ -1942,6 +1982,886 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { 8, &lm_wmma_scale_k16_a, &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, }; /* ========================================================================= diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json index a217985bf85c..53d676fb2b9a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json @@ -142,7 +142,7 @@ ] }, "gfx1250": { - "_comment": "gfx1250-class CDNA multi-chip device on the GFX12 programming model (wave32, WMMA, no MFMA). Primary fp16/bf16 WMMA atom is 16x16x32 (NOT gfx1201's 16x16x16). ABI empirically confirmed on ROCm 7.13 clang 23 (gfx1250 device target): __builtin_amdgcn_wmma_f32_16x16x32_f16 -> llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1,<16 x half>,i1,<16 x half>,i16,<8 x float>,i1,i1); bf16 uses v16bf16 (<16 x bfloat>) directly, not the gfx11 <16 x i16> bitcast. Datalayout/triple byte-identical to gfx950/gfx1201. This catalog ships validated fp16/bf16 and fp8/bf8 WMMA atoms plus homogeneous FP8xFP8 and BF8xBF8 SCALE/SCALE16 with E8M0 scales; FP4/FP6/BF6 block-scaled forms remain intentionally omitted. Lane maps are a HYPOTHESIS verified by examples/gfx1250/wmma_probe.py. MEMORY: gfx1250 HAS async global->LDS DMA (its own GFX12 family: global_load_async_to_lds_b128 + s_wait_asynccnt -- NOT the gfx9 buffer_load_lds/global_load_lds, which do not select), so has_async_global_lds=true (GPU-validated via the DTLA path). It also HAS a hardware transpose-LDS read, ds_load_tr16_b128 (GPU-validated via compute_pv_dstr). The has_async_lds and has_ds_read_tr flags stay false on purpose: in this catalog they denote the gfx9/gfx950 ds_read_*_tr_* / async-LDS *ABI* used to admit the gfx950 MFMA tiled builder (attention_arch.py), which gfx1250 does NOT have -- its async/transpose use distinct GFX12 opcodes gated directly in the ISA backend, not via these MFMA-pipeline bits. has_tdm=false: TDM not verified on this part.", + "_comment": "gfx1250-class CDNA multi-chip device on the GFX12 programming model (wave32, WMMA, no MFMA). Primary fp16/bf16 WMMA atom is 16x16x32 (NOT gfx1201's 16x16x16). ABI empirically confirmed on ROCm 7.13 clang 23 (gfx1250 device target): __builtin_amdgcn_wmma_f32_16x16x32_f16 -> llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1,<16 x half>,i1,<16 x half>,i16,<8 x float>,i1,i1); bf16 uses v16bf16 (<16 x bfloat>) directly, not the gfx11 <16 x i16> bitcast. Datalayout/triple byte-identical to gfx950/gfx1201. This catalog ships validated fp16/bf16 and fp8/bf8 WMMA atoms plus homogeneous and mixed FP8/BF8/FP6/BF6/FP4 SCALE/SCALE16 with E8M0 scales. Lane maps are a HYPOTHESIS verified by examples/gfx1250/wmma_probe.py. MEMORY: gfx1250 HAS async global->LDS DMA (its own GFX12 family: global_load_async_to_lds_b128 + s_wait_asynccnt -- NOT the gfx9 buffer_load_lds/global_load_lds, which do not select), so has_async_global_lds=true (GPU-validated via the DTLA path). It also HAS a hardware transpose-LDS read, ds_load_tr16_b128 (GPU-validated via compute_pv_dstr). The has_async_lds and has_ds_read_tr flags stay false on purpose: in this catalog they denote the gfx9/gfx950 ds_read_*_tr_* / async-LDS *ABI* used to admit the gfx950 MFMA tiled builder (attention_arch.py), which gfx1250 does NOT have -- its async/transpose use distinct GFX12 opcodes gated directly in the ISA backend, not via these MFMA-pipeline bits. has_tdm=false: TDM not verified on this part.", "family": "cdna", "target_family": "gfx12_cdna", "matrix_path": "wmma", @@ -186,7 +186,47 @@ {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k16"}, {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e8m0_k32"}, {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16"} + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e8m0_k16"} ] }, "gfx11-generic": { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py index 00ec50fa5f93..07d9cd34e080 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py @@ -968,6 +968,20 @@ class _FragInfo: } +# Mixed operands keep the same padded ABI and scale-coordinate maps. +_MMA_FRAGMENT_INFO.update( + { + f"wmma_gfx1250_f32_16x16x128_{a}_{b}_scale_e8m0_e8m0_k{block_k}": _MMA_FRAGMENT_INFO[ + f"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k{block_k}" + ] + for block_k in (32, 16) + for a in ("fp8", "bf8", "fp6", "bf6", "fp4") + for b in ("fp8", "bf8", "fp6", "bf6", "fp4") + if a != b + } +) + + def _frag_info(op_id: str) -> _FragInfo: info = _MMA_FRAGMENT_INFO.get(op_id) if info is None: diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md index 3a1fde1a0424..7bd4d13a6b33 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md @@ -14,6 +14,7 @@ LLVM 23 ROCm libraries, NumPy, and `ml_dtypes`. Torch is optional. | `mxfp8_gemm` | FP8 E4M3 (`fp8`) or BF8 E5M2 (`bf8`) on both operands | E8M0 | | `mxfp6_gemm` | Packed FP6 E2M3 or E3M2 on both operands | E8M0 | | `mxfp4_gemm` | Packed FP4 E2M1 on both operands | E8M0 | +| [`mixed_scaled_gemm`](mixed_scaled_gemm.py) | Different A/B formats from FP8, BF8, FP6, BF6, and FP4 | E8M0 | ```sh ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.mxfp8_gemm @@ -48,6 +49,17 @@ or `--dtype both` explicitly. Both operands use the selected format with E8M0 scales. `mxfp6_gemm` similarly defaults to both homogeneous FP6 encodings; use `--dtype fp6` or `--dtype bf6` to select one. Mixed matrix formats are outside these examples. +[`mixed_scaled_gemm`](mixed_scaled_gemm.py) defaults to FP8 E4M3 x FP4 E2M1. +Use `--dtype-a` and `--dtype-b` to select either operand independently; equal +normalized formats are rejected with guidance to the homogeneous examples. +All 20 ordered mixed pairs have SCALE and SCALE16 catalog entries. The builder +uses each operand's atom-selected layout and shared packed-storage helpers. + +```sh +ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.mixed_scaled_gemm --case all +ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.mixed_scaled_gemm --dtype-a fp6 --dtype-b fp4 --matrix-path wmma_scale16 --compile-route hip +``` + The target-independent aliases in `core/dtypes.py` normalize logical format names. Recognition does not imply that a target supports an atom; use its catalog to query supported operand formats and shapes. Each example defines diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py index 6750ce4b7850..e3c00ef32e2b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py @@ -3,7 +3,7 @@ """Launch gfx1250 block-scaled GEMM and compare with an independent reference. The default invocation keeps the K=64 FP8/BF8 WMMA + FP32-scale verifier. -Native ``--matrix-path wmma_scale`` / ``wmma_scale16`` use homogeneous FP8, FP6, or FP4 and +Native ``--matrix-path wmma_scale`` / ``wmma_scale16`` use independent FP8/BF8, FP6/BF6, or FP4 operands and E8M0 scales with K=32 / K=16 groups. Native fixtures cover K=128 or 256 and use bounded dyadic values, permitting exact comparison after output-type rounding. @@ -355,7 +355,7 @@ def run_cases( dtype_c=spec.dtype_c, ) label = ( - f"{spec.resolved_matrix_path()}/{spec.dtype_a}/{compile_route}/{case} " + f"{spec.resolved_matrix_path()}/{spec.dtype_a}x{spec.dtype_b}/{compile_route}/{case} " f"{spec.M}x{spec.N}x{spec.K} bk{spec.block_k}" ) got = _launch(rt, fn, spec, inputs) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/mixed_scaled_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/mixed_scaled_gemm.py new file mode 100644 index 000000000000..87a99df022e5 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/mixed_scaled_gemm.py @@ -0,0 +1,50 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""GEMM with different A/B matrix formats and E8M0 scales on gfx1250.""" + +from __future__ import annotations + +from ....core.dtypes import normalize_dtype +from ....instances.gfx1250.block_scaled_gemm import BlockScaledGemmSpec, _LOWBIT_DTYPES +from ._scaled_gemm_example import argument_parser, verify + +FORMATS = tuple(sorted(_LOWBIT_DTYPES)) + + +def make_spec(args) -> BlockScaledGemmSpec: + if normalize_dtype(args.dtype_a) == normalize_dtype(args.dtype_b): + raise ValueError( + "choose different A/B formats or use a homogeneous mxfp example" + ) + return BlockScaledGemmSpec( + name="mixed_scaled_gemm", + M=args.m, + N=args.n, + K=args.k, + dtype_a=args.dtype_a, + dtype_b=args.dtype_b, + dtype_c="bf16", + scale_dtype="e8m0", + matrix_path=args.matrix_path, + block_k=16 if args.matrix_path == "wmma_scale16" else 32, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argument_parser(__doc__) + parser.add_argument( + "--dtype-a", type=normalize_dtype, choices=FORMATS, default="fp8e4m3" + ) + parser.add_argument( + "--dtype-b", type=normalize_dtype, choices=FORMATS, default="fp4e2m1" + ) + args = parser.parse_args(argv) + try: + spec = make_spec(args) + except ValueError as exc: + parser.error(str(exc)) + return verify(spec, args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py index 53df5096a686..26650aa3a3e0 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py @@ -178,8 +178,6 @@ def is_valid_spec(spec: BlockScaledGemmSpec, arch: str = "gfx1250") -> Tuple[boo ) matrix_path = spec.resolved_matrix_path() native_scale = matrix_path in ("wmma_scale", "wmma_scale16") - if native_scale and _canon_lowbit(spec.dtype_a) != _canon_lowbit(spec.dtype_b): - return False, "native gfx1250 SCALE/SCALE16 requires matching matrix formats" family = matrix_path if native_scale else "wmma" atom_k = _WMMA_SCALE_K if native_scale else _WMMA_K if not native_scale and not target.mma.has_shape( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_mixed_scaled_wmma_llvm23.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_mixed_scaled_wmma_llvm23.json new file mode 100644 index 000000000000..f286ab1fd88b --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_mixed_scaled_wmma_llvm23.json @@ -0,0 +1,42 @@ +{ + "fp8/bf8/32": "2453b0f06d68a61efc927aab8f2fef5d586fe3095d22c99592f9bc9051809dab", + "fp8/fp6/32": "7df27f1e0c4636aba087a57d01774ec76cc112c4fcc5b7c664e4bb5eb33b30a8", + "fp8/bf6/32": "ef3742e918826e75049114255570c6999e9f415de0fec417a6c2b23e1a7114c8", + "fp8/fp4/32": "216ab7a9d9f42c4301bb26d33f053b88ba38769079dbb2e9066e70655db920df", + "bf8/fp8/32": "9e55ea31cefc0f5cc4b259429a67e9d91a0aca8ab86585bed38b5d26c1cb1749", + "bf8/fp6/32": "c6a486709c9c2b37a51cd7d3cbdbf45d4743552b7de6e0e5952c5678eea2ba61", + "bf8/bf6/32": "c8d9892ba05afca6da5ee15f459a9a13469b34788e6597792cb236e500eccae2", + "bf8/fp4/32": "31263ad501e7de16e4ab6ff91875709bcd9bead47cc63f00342857172c07300a", + "fp6/fp8/32": "13bfc1ee83c8b50eedc99ce4f451ac123f3cc50d138b4c13dec406519fe25bca", + "fp6/bf8/32": "1b354e5122fe470f82f41e10446449199bee07cb4d332d2631450f9465aaf88d", + "fp6/bf6/32": "60f8b19b35613be89d62211d0c718b54254bd5ee80d39a1fedcbcd630dd8a8b7", + "fp6/fp4/32": "7edb3961a9ef52933509b647d54396938f3d7c81e15e9aacea1480e4aec34162", + "bf6/fp8/32": "6ee3ad4cef543a4e4a59d2a811323a07e03e6c93e1c7ecd250e766feee1b8c35", + "bf6/bf8/32": "3413cde3195ca3c4d1742fd67f280e0d1cbccd99c3e6fc09ef376f5be60d8228", + "bf6/fp6/32": "4f0a253567b94d6ae1aa5fcba65f2dc2948e5a1092850e1fc6686d588c956d3a", + "bf6/fp4/32": "df8c362fe6a8694675784cad205e880daf09428f873b9901e0ee569332c40115", + "fp4/fp8/32": "4bfd684d62942221c4e0901608b1428fb2b2365e51900d63faf33a94cb557290", + "fp4/bf8/32": "0ab5f1b178913d78ac170f857c51e1b75351dbf5d6e2c169326e34da2f120493", + "fp4/fp6/32": "30161c527e4e7fc6e368610a15bc5c00803da1ca121aa52752e36ebecdb0c629", + "fp4/bf6/32": "86f0e8af8bce094563615446b31b54bab0b7a7fea2d9d9c00030232ea97da81f", + "fp8/bf8/16": "e57dd33198358c16f667867a7027f682fda900833dcb39c91b1195dffab3cd6a", + "fp8/fp6/16": "5b6b3f7b5c65a3684b605d8f04f18efd3e9f7caa23ebea11eca28ae9a4fa3e13", + "fp8/bf6/16": "abdd0da5a131c9c6cf9e60ffe0d8aa316732f578bb49948d53bc14aff9e8ebf5", + "fp8/fp4/16": "d6a13741e80c9b255989f32e00e518d28b361f31d26e1ea57a232bccb7ecccfc", + "bf8/fp8/16": "248e6529a77b5c09fad9057026e09b1de5c8e945fc6bd89b9e757523b69bb158", + "bf8/fp6/16": "ec23237acdb53ddcde070b4a4ad2dada25ee98b14b36e56c340c704b273ae5c1", + "bf8/bf6/16": "272138c6c241cbc3d86f5e8c1e615fba1e5880d59725f0bd8db9c9d16a202373", + "bf8/fp4/16": "68fd5594be2ab90c30ab24a6e0343dc5e801b4039f17ab6115a183d6ef967546", + "fp6/fp8/16": "4b7b5eb7b447b835867d99e61496d75cb65a4dbe6a05b79e922ca940f92b741e", + "fp6/bf8/16": "da38ae67af587cbb4a4e551647fb9f5a5c021dfdb47d3cae15b301fcbb14f7ff", + "fp6/bf6/16": "4d7179871a608bf500bc4dbd3cb8dbf96fbe39f762fbed9789aadc8f77e597ca", + "fp6/fp4/16": "b220bbc950bb387b6fb0fd9030808512acbe673d1105825f0ba602c14b525b68", + "bf6/fp8/16": "0034d7d293a51f0032190e76eaf71093084d4c7485ba1f869e0a62f279e213c7", + "bf6/bf8/16": "40cc852590ea093e3928d467c3fe5987f52f0e2e95faae0e60afdeeec76a1f1b", + "bf6/fp6/16": "2da996a98e87163ed3990e298852269e257386cce751ee21071bcf40e81f73d8", + "bf6/fp4/16": "5f8c2770a3ea5b3a01528b5fa6dc7b53b19d824cad339ef93422c633711ad414", + "fp4/fp8/16": "1742edaf942b5aae7d593a93ad0e8efeb34582117981188759cae46f2f9caae1", + "fp4/bf8/16": "6bfadb88641d4b7f6c4237e3f10b296da710b411d4722ca34829ce8e4d655974", + "fp4/fp6/16": "d59da61d700407e45d262b3b325772fdffd8ad2c0ecb02e5d3c98fde227e2cd1", + "fp4/bf6/16": "a2be51b173003006a9a983ca5c2b3fb64afbfa3d62726f0dc9524d8521544187" +} diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py index 2c2fbdc609ff..eb4412f40757 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py @@ -223,18 +223,18 @@ def test_e5m2_is_not_a_scale_dtype_alias(dtype, field): def test_scaled_catalog_identity_and_backend_contract(): catalog = ArchTarget.from_gfx("gfx1250").mma rows = [row for row in catalog.ops if row.family == "wmma_scaled"] - assert len(rows) == 10 - assert len({row.op_id for row in rows}) == 10 + assert len(rows) == 50 + assert len({row.op_id for row in rows}) == 50 for row in rows: - dtype = { + tokens = { "fp8e4m3": "fp8", "bf8e5m2": "bf8", "fp6e2m3": "fp6", "fp6e3m2": "bf6", "fp4e2m1": "fp4", - }[row.a_dtype] + } assert row.op_id == ( - f"wmma_gfx1250_f32_16x16x128_{dtype}_{dtype}" + f"wmma_gfx1250_f32_16x16x128_{tokens[row.a_dtype]}_{tokens[row.b_dtype]}" f"_scale_e8m0_e8m0_k{row.scale_block_k}" ) assert row.a_scale_dtype == row.b_scale_dtype == "e8m0" @@ -242,14 +242,17 @@ def test_scaled_catalog_identity_and_backend_contract(): assert isinstance(row.scale_block_k, MmaScaleBlockK) packing = gfx1250_scaled_wmma(row.op_id) assert packing.atom is row - selector = { + selectors = { "fp8e4m3": 0, "bf8e5m2": 1, "fp6e2m3": 2, "fp6e3m2": 3, "fp4e2m1": 4, - }[row.a_dtype] - assert packing.matrix_formats == (selector, selector) + } + assert packing.matrix_formats == ( + selectors[row.a_dtype], + selectors[row.b_dtype], + ) assert packing.scales.count * packing.scales.block_k == row.k assert (row.a_frag_len, row.b_frag_len) == (16, 16) for family in ("wmma_scale", "wmma_scale16"): diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_fp4_scaled_wmma.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_fp4_scaled_wmma.py index 4483328b50cc..6b99fc4c1632 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_fp4_scaled_wmma.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_fp4_scaled_wmma.py @@ -193,7 +193,7 @@ def test_fp4_catalog_signature_and_lowering(path, scale_type): @pytest.mark.parametrize( "changes", [ - {"dtype_b": "fp8"}, + {"dtype_b": "i8"}, {"matrix_path": "wmma"}, {"K": 192}, {"scale_dtype": "fp32"}, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_fp6_scaled_wmma.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_fp6_scaled_wmma.py index 72464ac7008e..8dd48f66f398 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_fp6_scaled_wmma.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_fp6_scaled_wmma.py @@ -161,31 +161,6 @@ def test_fp6_aliases(): assert is_valid_spec(alias)[0] -@pytest.mark.parametrize("a,b", [("fp6", "bf6"), ("fp8", "fp4"), ("fp6", "fp4")]) -def test_other_matrix_contracts_are_not_admitted(a, b): - assert not is_valid_spec(spec_for(a, b))[0] - - -@pytest.mark.parametrize("mode", ["wmma_scale", "wmma_scale16"]) -@pytest.mark.parametrize("a", FORMATS) -@pytest.mark.parametrize("b", FORMATS) -def test_homogeneous_catalog_boundary(a, b, mode): - spec = spec_for(a, b, mode) - accepted = a == b and a in ("fp8", "bf8", "fp6", "bf6", "fp4") - assert is_valid_spec(spec)[0] == accepted - atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( - family="wmma_scaled", - a_dtype=a, - b_dtype=b, - c_dtype="fp32", - m=16, - n=16, - k=128, - scales=("e8m0", "e8m0", spec.block_k), - ) - assert (atom is not None) == accepted - - @pytest.mark.parametrize("dtype", ["fp6", "bf6"]) @pytest.mark.parametrize("block_k", [16, 32]) def test_fp6_atom_storage_contract(dtype, block_k): diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_example.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_example.py new file mode 100644 index 000000000000..b2559284153f --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_example.py @@ -0,0 +1,47 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Mixed example selects independent normalized operands and the shared verifier.""" + +import pytest + +from rocke.examples.gfx1250.gemm import mixed_scaled_gemm + + +def test_default_pair_and_verification_options(monkeypatch): + seen = [] + monkeypatch.setattr( + mixed_scaled_gemm, "verify", lambda spec, args: seen.append((spec, args)) or 0 + ) + assert ( + mixed_scaled_gemm.main( + ["--matrix-path", "wmma_scale16", "--compile-route", "hip", "--case", "all"] + ) + == 0 + ) + spec, args = seen[0] + assert (spec.dtype_a, spec.dtype_b) == ("fp8e4m3", "fp4e2m1") + assert spec.block_k == 16 and spec.scale_dtype == "e8m0" + assert args.compile_route == "hip" and args.case == "all" + + +@pytest.mark.parametrize( + "a,b,expected", + [("fp6", "bf6", ("fp6e2m3", "fp6e3m2")), ("fp4", "bf8", ("fp4e2m1", "bf8e5m2"))], +) +def test_aliases_preserve_operand_order(monkeypatch, a, b, expected): + seen = [] + monkeypatch.setattr( + mixed_scaled_gemm, "verify", lambda spec, args: seen.append(spec) or 0 + ) + assert mixed_scaled_gemm.main(["--dtype-a", a, "--dtype-b", b]) == 0 + assert (seen[0].dtype_a, seen[0].dtype_b) == expected + + +@pytest.mark.parametrize( + "a,b", [("fp8", "fp8e4m3"), ("bf6", "fp6e3m2"), ("fp4", "fp4e2m1")] +) +def test_equal_normalized_formats_rejected(a, b, capsys): + with pytest.raises(SystemExit) as exc: + mixed_scaled_gemm.main(["--dtype-a", a, "--dtype-b", b]) + assert exc.value.code == 2 + assert "choose different A/B formats" in capsys.readouterr().err diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py new file mode 100644 index 000000000000..4c750ca6b1e2 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py @@ -0,0 +1,147 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Ordered mixed-format atoms retain independent operand storage contracts.""" + +import hashlib +from itertools import product +import json +from pathlib import Path +import re + +import pytest + +from rocke.core.arch import ArchTarget +from rocke.core.arch.wmma_scale import gfx1250_scaled_wmma +from rocke.core.backend import resolve_backend +from rocke.core.lower_hip import lower_kernel_to_hip +from rocke.core.lower_llvm import lower_kernel_to_llvm +from rocke.instances.gfx1250.block_scaled_gemm import ( + BlockScaledGemmSpec, + build_block_scaled_gemm, + is_valid_spec, +) + +FORMATS = ("fp8", "bf8", "fp6", "bf6", "fp4") +MIXED_PAIRS = [(a, b) for a, b in product(FORMATS, repeat=2) if a != b] + + +@pytest.mark.parametrize("a,b", MIXED_PAIRS) +@pytest.mark.parametrize("block_k", [32, 16]) +def test_mixed_catalog_storage_and_lowering(a, b, block_k): + mode = "wmma_scale16" if block_k == 16 else "wmma_scale" + spec = BlockScaledGemmSpec( + name="mixed_test", + M=32, + N=48, + K=256, + dtype_a=a, + dtype_b=b, + matrix_path=mode, + scale_dtype="e8m0", + block_k=block_k, + ) + assert is_valid_spec(spec) == (True, f"ok: gfx1250 K=128 native {mode} {a} GEMM") + atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( + family="wmma_scaled", + a_dtype=a, + b_dtype=b, + c_dtype="fp32", + m=16, + n=16, + k=128, + scales=("e8m0", "e8m0", block_k), + ) + assert ( + atom.op_id == f"wmma_gfx1250_f32_16x16x128_{a}_{b}_scale_e8m0_e8m0_k{block_k}" + ) + contract = gfx1250_scaled_wmma(atom.op_id) + widths = {"fp8": 8, "bf8": 8, "fp6": 6, "bf6": 6, "fp4": 4} + for operand, dtype in (("a", a), ("b", b)): + layout = contract.matrix_layout(operand) + bits = widths[dtype] + assert layout.fragment.packing.element_bits == bits + assert layout.fragment.live_carriers == bits * 2 + assert layout.fragment.padding_bits == (8 - bits) * 64 + assert layout.chunk_bytes == (16 if bits in (4, 8) else 24) + assert contract.scale_packing(operand).count == 128 // block_k + kernel = build_block_scaled_gemm(spec) + llvm = lower_kernel_to_llvm(kernel, arch="gfx1250", llvm_flavor="llvm23") + calls = [ + line + for line in llvm.splitlines() + if "call <8 x float> @llvm.amdgcn.wmma.scale" in line + ] + assert len(calls) == 2 + for call in calls: + # Match selector positions in order; a set-membership check misses swaps. + assert re.findall(r"i32 (\d+), <16 x i32>", call) == [ + str(FORMATS.index(a)), + str(FORMATS.index(b)), + ] + hip = lower_kernel_to_hip(kernel, arch="gfx1250") + builtin = f"__builtin_amdgcn_{mode}_f32_16x16x128_f8f6f4" + assert f"{builtin}({FORMATS.index(a)}," in hip + for flavor in ("llvm20", "llvm22"): + error = RuntimeError if resolve_backend() == "cpp" else NotImplementedError + with pytest.raises(error, match="requires llvm23"): + lower_kernel_to_llvm(kernel, arch="gfx1250", llvm_flavor=flavor) + + +@pytest.mark.parametrize("a,b", MIXED_PAIRS) +def test_mixed_atoms_do_not_leak_to_other_targets(a, b): + for arch in ("gfx942", "gfx950", "gfx1201"): + assert ( + ArchTarget.from_gfx(arch).mma.op_for_shape( + family="wmma_scaled", + a_dtype=a, + b_dtype=b, + c_dtype="fp32", + m=16, + n=16, + k=128, + scales=("e8m0", "e8m0", 32), + ) + is None + ) + + +@pytest.mark.parametrize("scales", [("e4m3", "e8m0", 32), ("e8m0", "e5m3", 16)]) +def test_mixed_atoms_do_not_admit_other_scale_contracts(scales): + assert ( + ArchTarget.from_gfx("gfx1250").mma.op_for_shape( + family="wmma_scaled", + a_dtype="fp8", + b_dtype="fp4", + c_dtype="fp32", + m=16, + n=16, + k=128, + scales=scales, + ) + is None + ) + + +@pytest.mark.parametrize( + "case,expected_sha", + json.loads( + Path(__file__).with_name("gfx1250_mixed_scaled_wmma_llvm23.json").read_text() + ).items(), +) +def test_mixed_llvm23_golden(case, expected_sha): + a, b, block_k = case.split("/") + spec = BlockScaledGemmSpec( + name="mixed_golden", + M=32, + N=48, + K=256, + dtype_a=a, + dtype_b=b, + matrix_path="wmma_scale16" if block_k == "16" else "wmma_scale", + scale_dtype="e8m0", + block_k=int(block_k), + ) + llvm = lower_kernel_to_llvm( + build_block_scaled_gemm(spec), arch="gfx1250", llvm_flavor="llvm23" + ) + assert hashlib.sha256(llvm.encode()).hexdigest() == expected_sha diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp index bd945b8a88ab..6804d32756bb 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp @@ -307,7 +307,7 @@ static int test_scale_contracts() ++scaled_rows; } } - CHECK(packed_rows == 4 && scaled_rows == 10); + CHECK(packed_rows == 4 && scaled_rows == 50); return 0; } @@ -376,7 +376,7 @@ static int test_scale_layouts_and_families() } } } - CHECK(scaled == 10); + CHECK(scaled == 50); return 0; } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c index 40a8ef65f611..c4f0c90efda8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c @@ -120,13 +120,13 @@ static void build_wmma_k64_bf8_bf8(rocke_ir_builder_t* b) /* K=128 FP8 SCALE/SCALE16 WMMA. Matrix fragments are <16 x i32>; packed * E8M0 scale operands are i32 for SCALE and i64 for SCALE16. */ -static void wmma_scaled(rocke_ir_builder_t* b, bool scale16, const char* dtype) +static void wmma_scaled(rocke_ir_builder_t* b, bool scale16, const char* dtype, const char* dtype_b) { const rocke_mma_scale_block_k_t block = scale16 ? ROCKE_MMA_SCALE_K16 : ROCKE_MMA_SCALE_K32; const rocke_mma_scale_filter_t query = {"e8m0", "e8m0", block}; const rocke_arch_target_t* target = rocke_arch_target_from_gfx("gfx1250"); const rocke_mma_op_t* atom = rocke_mma_catalog_op_for_shape( - &target->mma, "wmma_scaled", dtype, dtype, "fp32", 16, 16, 128, &query); + &target->mma, "wmma_scaled", dtype, dtype_b, "fp32", 16, 16, 128, &query); const char* op_id = atom->op_id; const rocke_type_t* scale_ty = scale16 ? rocke_i64() : rocke_i32(); rocke_value_t* a_ptr = frag_param(b, "A", rocke_i32(), true); @@ -152,32 +152,32 @@ static void wmma_scaled(rocke_ir_builder_t* b, bool scale16, const char* dtype) static void build_wmma_scale(rocke_ir_builder_t* b) { - wmma_scaled(b, false, "fp8e4m3"); + wmma_scaled(b, false, "fp8e4m3", "fp8e4m3"); } static void build_wmma_scale_bf8(rocke_ir_builder_t* b) { - wmma_scaled(b, false, "bf8e5m2"); + wmma_scaled(b, false, "bf8e5m2", "bf8e5m2"); } static void build_wmma_scale16(rocke_ir_builder_t* b) { - wmma_scaled(b, true, "fp8e4m3"); + wmma_scaled(b, true, "fp8e4m3", "fp8e4m3"); } static void build_wmma_scale16_bf8(rocke_ir_builder_t* b) { - wmma_scaled(b, true, "bf8e5m2"); + wmma_scaled(b, true, "bf8e5m2", "bf8e5m2"); } static void build_wmma_scale_fp4(rocke_ir_builder_t* b) { - wmma_scaled(b, false, "fp4e2m1"); + wmma_scaled(b, false, "fp4e2m1", "fp4e2m1"); } static void build_wmma_scale16_fp4(rocke_ir_builder_t* b) { - wmma_scaled(b, true, "fp4e2m1"); + wmma_scaled(b, true, "fp4e2m1", "fp4e2m1"); } /* ds_read_b128_tr_b16. gfx950 has one type-agnostic opcode returning @@ -395,6 +395,7 @@ typedef struct config const char* arch; const char* dtype; bool scale16; + const char* dtype_b; } config_t; /* Each gfx1250 config that tests a *choice* of encoding is followed by its @@ -433,6 +434,46 @@ static const config_t CONFIGS[] = { {build_wmma_scale16_bf8, "gfx1250"}, {build_scale_coordinates_k32, "gfx1250"}, {build_scale_coordinates_k16, "gfx1250"}, + {NULL, "gfx1250", "fp8", false, "bf8"}, + {NULL, "gfx1250", "fp8", false, "fp6"}, + {NULL, "gfx1250", "fp8", false, "bf6"}, + {NULL, "gfx1250", "fp8", false, "fp4"}, + {NULL, "gfx1250", "bf8", false, "fp8"}, + {NULL, "gfx1250", "bf8", false, "fp6"}, + {NULL, "gfx1250", "bf8", false, "bf6"}, + {NULL, "gfx1250", "bf8", false, "fp4"}, + {NULL, "gfx1250", "fp6", false, "fp8"}, + {NULL, "gfx1250", "fp6", false, "bf8"}, + {NULL, "gfx1250", "fp6", false, "bf6"}, + {NULL, "gfx1250", "fp6", false, "fp4"}, + {NULL, "gfx1250", "bf6", false, "fp8"}, + {NULL, "gfx1250", "bf6", false, "bf8"}, + {NULL, "gfx1250", "bf6", false, "fp6"}, + {NULL, "gfx1250", "bf6", false, "fp4"}, + {NULL, "gfx1250", "fp4", false, "fp8"}, + {NULL, "gfx1250", "fp4", false, "bf8"}, + {NULL, "gfx1250", "fp4", false, "fp6"}, + {NULL, "gfx1250", "fp4", false, "bf6"}, + {NULL, "gfx1250", "fp8", true, "bf8"}, + {NULL, "gfx1250", "fp8", true, "fp6"}, + {NULL, "gfx1250", "fp8", true, "bf6"}, + {NULL, "gfx1250", "fp8", true, "fp4"}, + {NULL, "gfx1250", "bf8", true, "fp8"}, + {NULL, "gfx1250", "bf8", true, "fp6"}, + {NULL, "gfx1250", "bf8", true, "bf6"}, + {NULL, "gfx1250", "bf8", true, "fp4"}, + {NULL, "gfx1250", "fp6", true, "fp8"}, + {NULL, "gfx1250", "fp6", true, "bf8"}, + {NULL, "gfx1250", "fp6", true, "bf6"}, + {NULL, "gfx1250", "fp6", true, "fp4"}, + {NULL, "gfx1250", "bf6", true, "fp8"}, + {NULL, "gfx1250", "bf6", true, "bf8"}, + {NULL, "gfx1250", "bf6", true, "fp6"}, + {NULL, "gfx1250", "bf6", true, "fp4"}, + {NULL, "gfx1250", "fp4", true, "fp8"}, + {NULL, "gfx1250", "fp4", true, "bf8"}, + {NULL, "gfx1250", "fp4", true, "fp6"}, + {NULL, "gfx1250", "fp4", true, "bf6"}, }; static const int NUM_CONFIGS = (int)(sizeof(CONFIGS) / sizeof(CONFIGS[0])); @@ -473,7 +514,10 @@ int main(int argc, char** argv) } else { - wmma_scaled(&b, CONFIGS[idx].scale16, CONFIGS[idx].dtype); + wmma_scaled(&b, + CONFIGS[idx].scale16, + CONFIGS[idx].dtype, + CONFIGS[idx].dtype_b ? CONFIGS[idx].dtype_b : CONFIGS[idx].dtype); } if(!rocke_ir_builder_ok(&b)) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py index 4a157eb735f9..b74097b930bc 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py @@ -312,6 +312,15 @@ def build(b): CONFIGS.extend((_scale_coordinates(block), "gfx1250") for block in (32, 16)) +CONFIGS.extend( + (_wmma_scaled(a, b, mode), "gfx1250") + for mode in ("scale", "scale16") + for a in ("fp8", "bf8", "fp6", "bf6", "fp4") + for b in ("fp8", "bf8", "fp6", "bf6", "fp4") + if a != b +) + + def _spec(idx: int): """Config selector: the (builder, arch) pair the shared driver expects.""" if not 0 <= idx < len(CONFIGS): diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mixed_scaled_gemm_numeric.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mixed_scaled_gemm_numeric.py new file mode 100644 index 000000000000..51edacfea1d5 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mixed_scaled_gemm_numeric.py @@ -0,0 +1,56 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Mixed A/B matrix-format numerical cases with E8M0 scales.""" + +import subprocess +import sys +from itertools import product + +import pytest + +_MODULE = "rocke.examples.gfx1250.gemm.mixed_scaled_gemm" + +# Every ordered mixed pair with E8M0 through both compiler routes. +_FORMATS = ("fp8e4m3", "bf8e5m2", "fp6", "bf6", "fp4") +_MIXED = [(a, b) for a, b in product(_FORMATS, repeat=2) if a != b] + + +@pytest.mark.parametrize("matrix_path", ["wmma_scale", "wmma_scale16"]) +@pytest.mark.parametrize( + "a,b,route", + [(*case, route) for case in _MIXED for route in ("comgr", "hip")], +) +def test_mixed_scaled_wmma_numeric(gpu_env, matrix_path, a, b, route): + result = subprocess.run( + [ + sys.executable, + "-m", + _MODULE, + "--dtype-a", + a, + "--dtype-b", + b, + "--matrix-path", + matrix_path, + "--compile-route", + route, + "--m", + "32", + "--n", + "48", + "--k", + "256", + "--case", + "all", + ], + env=gpu_env, + capture_output=True, + text=True, + timeout=300, + ) + output = result.stdout + result.stderr + assert result.returncode == 0, output + count = 20 if matrix_path == "wmma_scale16" else 12 + assert f"PASS: verified {count} cases" in output, output + assert output.count("bad=0") == count, output + print(output, end="") From e82b05bfe692b5c7f52123d5ba5a0debf50f20aa Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:33:42 -0400 Subject: [PATCH 22/32] feat(rocke): select independent scales through canonical MMA contracts --- .../rocke/platform/cpp/core/arch/data.cpp | 828 ++++++++++++++++++ .../cpp/include/rocke/wmma_scale_internal.h | 14 +- .../rocke/core/arch/data/arch_specs.json | 38 +- .../platform/python/rocke/core/arch/target.py | 24 + .../python/rocke/core/arch/wmma_scale.py | 22 +- .../rocke/examples/gfx1250/gemm/FP4_SCALE.md | 4 + .../rocke/examples/gfx1250/gemm/README.md | 11 + .../examples/gfx1250/gemm/SCALE_FORMATS.md | 79 ++ .../gfx1250/gemm/SCALE_FORMAT_EXPERIMENT.md | 109 +++ .../gfx1250/gemm/block_scaled_gemm_verify.py | 67 +- .../gfx1250/gemm/scale_formats_gemm.py | 54 ++ .../python/rocke/instances/SUPPORT_MATRIX.md | 9 +- .../instances/gfx1250/block_scaled_gemm.py | 31 +- .../core/gfx1250_scale_formats_llvm23.json | 38 + .../platform/tests/core/test_arch_mma_ssot.py | 14 +- .../core/test_gfx1250_mixed_scaled_wmma.py | 2 +- .../tests/core/test_gfx1250_scale_formats.py | 239 +++++ .../tests/core/test_mma_catalog_lookup.cpp | 16 +- .../instances/parity/gfx1250_lowering_emit.c | 63 +- .../instances/parity/gfx1250_lowering_emit.py | 30 +- .../test_gfx1250_scale_formats_example.py | 82 ++ .../test_gfx1250_scale_formats_numeric.py | 63 ++ 22 files changed, 1792 insertions(+), 45 deletions(-) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMATS.md create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMAT_EXPERIMENT.md create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/scale_formats_gemm.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scale_formats_llvm23.json create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp index 88e13137c1d6..748021cb8147 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp @@ -941,6 +941,42 @@ static const rocke_ati_mma_frag_row_t rocke_ati_mma_frag[] = { {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k16", 8}, {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k16", 8}, {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e4m3_k16", 8}, {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e8m0_k32", 8}, {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e8m0_k32", 8}, {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e8m0_k32", 8}, @@ -2862,6 +2898,798 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { 8, &lm_wmma_scale_k16_a, &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, }; /* ========================================================================= diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h index c67649bb8aef..b53babe1ab19 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h @@ -115,11 +115,21 @@ static inline rocke_scaled_wmma_op_t rocke_scaled_wmma_contract(const rocke_mma_ spec.matrix_formats[i] = 4; else ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA matrix format"); - if(!scale_dtypes[i] || strcmp(scale_dtypes[i], "e8m0") != 0) + if(scale_dtypes[i] && strcmp(scale_dtypes[i], "e8m0") == 0) + spec.scale_formats[i] = 0; + else if(scale_dtypes[i] && strcmp(scale_dtypes[i], "e5m3") == 0) + spec.scale_formats[i] = 1; + else if(scale_dtypes[i] && strcmp(scale_dtypes[i], "e4m3") == 0) + spec.scale_formats[i] = 2; + else ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA scale format"); - spec.scale_formats[i] = 0; // E8M0. + if(spec.matrix_formats[i] != 4 && spec.scale_formats[i] != 0) + ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA backend contract"); spec.matrix_words[i] = words[i]; } + if(spec.matrix_formats[0] == 4 && spec.matrix_formats[1] == 4 + && spec.scale_formats[0] != spec.scale_formats[1]) + ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA backend contract"); if((atom->scale_block_k != 16 && atom->scale_block_k != 32) || strcmp(atom->c_dtype, "fp32") != 0 || atom->m != 16 || atom->n != 16 || atom->k != 128) ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA backend contract"); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json index 53d676fb2b9a..545bf65882a1 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json @@ -226,7 +226,43 @@ {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e8m0_k16"}, {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e8m0_k16"}, {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e8m0_k16"} + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e4m3_k16"} ] }, "gfx11-generic": { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py index 07d9cd34e080..79a9ba0d706e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py @@ -982,6 +982,30 @@ class _FragInfo: ) +# Alternative scale encodings retain the same byte carriers and lane maps. +_MMA_FRAGMENT_INFO.update( + { + f"wmma_gfx1250_f32_16x16x128_{a}_{b}_scale_{sa}_{sb}_k{block_k}": _MMA_FRAGMENT_INFO[ + f"wmma_gfx1250_f32_16x16x128_{a}_{b}_scale_e8m0_e8m0_k{block_k}" + ] + for block_k in (32, 16) + for a, b, sa, sb in ( + [ + (d, "fp4", "e8m0", scale) + for d in ("fp8", "bf8", "fp6", "bf6") + for scale in ("e5m3", "e4m3") + ] + + [ + ("fp4", d, scale, "e8m0") + for d in ("fp8", "bf8", "fp6", "bf6") + for scale in ("e5m3", "e4m3") + ] + + [("fp4", "fp4", scale, scale) for scale in ("e5m3", "e4m3")] + ) + } +) + + def _frag_info(op_id: str) -> _FragInfo: info = _MMA_FRAGMENT_INFO.get(op_id) if info is None: diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py index 271a3661a15e..ecc95b1b66c9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py @@ -102,13 +102,24 @@ def gfx1250_scaled_wmma(op_id: str) -> ScaledWmmaOp | None: if atom is None or atom.family != "wmma_scaled": return None formats = {"fp8e4m3": 0, "bf8e5m2": 1, "fp6e2m3": 2, "fp6e3m2": 3, "fp4e2m1": 4} - # The current backend supports E8M0 for both inputs and a shared K-group size. + scale_selectors = { + MmaScaleDType.E8M0: 0, + MmaScaleDType.E5M3: 1, + MmaScaleDType.E4M3: 2, + } + # Alternative scales require FP4; FP4 x FP4 requires matching formats. # Keep these restrictions here, independently of the catalog query model. if ( atom.a_dtype not in formats or atom.b_dtype not in formats - or atom.a_scale_dtype != MmaScaleDType.E8M0 - or atom.b_scale_dtype != MmaScaleDType.E8M0 + or atom.a_scale_dtype not in scale_selectors + or atom.b_scale_dtype not in scale_selectors + or (atom.a_dtype != "fp4e2m1" and atom.a_scale_dtype != MmaScaleDType.E8M0) + or (atom.b_dtype != "fp4e2m1" and atom.b_scale_dtype != MmaScaleDType.E8M0) + or ( + atom.a_dtype == atom.b_dtype == "fp4e2m1" + and atom.a_scale_dtype != atom.b_scale_dtype + ) or atom.scale_block_k not in (16, 32) or atom.c_dtype != "fp32" or atom.shape != (16, 16, 128) @@ -120,6 +131,9 @@ def gfx1250_scaled_wmma(op_id: str) -> ScaledWmmaOp | None: return ScaledWmmaOp( atom=atom, matrix_formats=(formats[atom.a_dtype], formats[atom.b_dtype]), - scale_formats=(0, 0), # E8M0 for each source. + scale_formats=( + scale_selectors[atom.a_scale_dtype], + scale_selectors[atom.b_scale_dtype], + ), scales=ScalePacking(count=atom.a_scale_frag_len, block_k=atom.scale_block_k), ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/FP4_SCALE.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/FP4_SCALE.md index 63337de453a3..0e1ce297604e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/FP4_SCALE.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/FP4_SCALE.md @@ -66,3 +66,7 @@ COMGR requires a matching LLVM 23 toolchain. With the C++ extension installed, set `ROCKE_BACKEND=both` to compare Python/C++ LLVM before COMGR compilation. The HIP verifier invokes the Python HIP lowerer directly. Numerical validation requires a matching gfx1250 device, NumPy, and ml_dtypes; it does not need torch. + +Additional E4M3/E5M3 scale encodings use the same packed matrix storage. +See [independent scale formats](SCALE_FORMATS.md) for the accepted combinations +and the scale-format example. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md index 7bd4d13a6b33..49557268034b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md @@ -15,6 +15,7 @@ LLVM 23 ROCm libraries, NumPy, and `ml_dtypes`. Torch is optional. | `mxfp6_gemm` | Packed FP6 E2M3 or E3M2 on both operands | E8M0 | | `mxfp4_gemm` | Packed FP4 E2M1 on both operands | E8M0 | | [`mixed_scaled_gemm`](mixed_scaled_gemm.py) | Different A/B formats from FP8, BF8, FP6, BF6, and FP4 | E8M0 | +| [`scale_formats_gemm`](scale_formats_gemm.py) | FP4 alone or mixed with FP8/BF8/FP6/BF6 | E8M0, E4M3, or E5M3, subject to operand legality | ```sh ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.mxfp8_gemm @@ -82,3 +83,13 @@ packed byte order. The opt-in numerical suite exercises both encodings and scale group sizes, one-sided/group-isolated inputs, multiple tiles and K steps, and HIP/COMGR compilation. These cases bound the validation; dtype recognition alone does not establish support for other operand contracts. + +The [scale-format example](SCALE_FORMATS.md) defaults to FP4 x FP4 with E4M3 +scales. `--scale-dtype-a` and `--scale-dtype-b` select the input encodings; +non-E8M0 scales require an FP4 operand, and FP4 x FP4 requires matching scale +formats. The shared verifier decodes each scale buffer independently. + +```sh +ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.scale_formats_gemm --case all +ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.scale_formats_gemm --dtype-a fp6 --dtype-b fp4 --scale-dtype-a e8m0 --scale-dtype-b e5m3 --matrix-path wmma_scale16 +``` diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMATS.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMATS.md new file mode 100644 index 000000000000..c61892c2bc2e --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMATS.md @@ -0,0 +1,79 @@ +# Independent scale formats on gfx1250 + +The [block-scaled GEMM builder](../../../instances/gfx1250/block_scaled_gemm.py) +accepts independent A/B scale-format selections on its native SCALE/SCALE16 +paths. See [packed FP6](FP6.md) and [packed FP4](FP4_SCALE.md) for matrix storage. + +## Contents + +- [Scale formats](#scale-formats) +- [Evidence and planned experiment](#evidence-and-planned-experiment) + +## Scale formats + +`scale_dtype` remains the shared default. Optional `scale_dtype_a` and +`scale_dtype_b` override it independently. All native scale buffers contain +encoded bytes; choosing a format does not quantize or convert the buffer. + +| A data | A scale | B data | B scale | +| --- | --- | --- | --- | +| FP8 / FP6 / FP4 | E8M0 | FP8 / FP6 / FP4 | E8M0 | +| FP8 / FP6 | E8M0 | FP4 | E5M3 or E4M3 | +| FP4 | E5M3 or E4M3 | FP8 / FP6 | E8M0 | +| FP4 | E5M3 | FP4 | E5M3 | +| FP4 | E4M3 | FP4 | E4M3 | + +Here FP8 and FP6 each include both exponent/mantissa formats. E8M0 has selector +0, E5M3 selector 1, and E4M3 selector 2. The reference decoder interprets E5M3 +as an unsigned eight-bit scale format with bias 15, distinct from E5M2. +The implementation requires matching scale formats for FP4 x FP4. +The `i8` scale spelling remains an alias for E8M0 for existing callers. + +```python +from rocke.instances.gfx1250.block_scaled_gemm import ( + BlockScaledGemmSpec, build_block_scaled_gemm, +) + +spec = BlockScaledGemmSpec( + name="mixed_fp6_fp4", M=32, N=48, K=256, + dtype_a="fp6e2m3", dtype_b="fp4", + matrix_path="wmma_scale16", block_k=16, + scale_dtype="e8m0", scale_dtype_b="e4m3", +) +kernel = build_block_scaled_gemm(spec) +``` + +The atom catalog owns the scale formats and K-group size. Query +`target.mma.op_for_shape(family="wmma_scaled", ..., scales=("e8m0", "e4m3", 16))` +and pass the returned `op_id` to the ordinary `IRBuilder.mma` / `rocke_b_mma` +APIs. Python and native lowering derive both hardware selectors from that +contract. Existing E8M0 calls keep their operation IDs and emitted code. +Nondefault scale types are included in generated kernel names. The builder +accepts the existing `fp8e4m3` scale alias for E4M3; E5M2 matrix spellings are +not aliases for the unsigned E5M3 scale format. + +## Evidence and planned experiment + +The public [machine-readable ISA](https://gpuopen.com/download/machine-readable-isa/latest/) +2026-08-06 snapshot's `amdgpu_isa_cdna5.xml` defines SCALE/SCALE16 operand widths +and block sizes, but does not enumerate E5M3 scale decoding or the complete +scale legality table. LLVM's `WMMA::MatrixScaleFmt` defines the selector names +and values used here. The table above describes this implementation's accepted +combinations; compiler acceptance alone does not establish numerical behavior. + +Existing mixed-format fixtures cover bounded finite values and compare against +an independent host calculation. They do not establish arbitrary-input rounding, +scale NaN behavior, or performance. The [scale-format discrimination experiment](SCALE_FORMAT_EXPERIMENT.md) +is staged separately and has not been run. It compares identical raw scale bytes +against literal predictions and verifies the emitted instruction fields. + +```sh +ROCKE_LLVM_FLAVOR=llvm23 ROCKE_BACKEND=both ROCKE_CPP_STRICT=1 \ +ROCKE_REQUIRE_GFX1250=1 python -m pytest -q -s \ + tests/instances/test_gfx1250_scale_formats_numeric.py +``` + +Use a matching gfx1250 device, LLVM 23 COMGR/HIP, NumPy/ml_dtypes, and a freshly +built C++ extension. COMGR checks Python/C++ LLVM identity; HIP fixtures compile +Python-generated HIP. Choosing a scale format does not quantize or convert input +buffers. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMAT_EXPERIMENT.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMAT_EXPERIMENT.md new file mode 100644 index 000000000000..7c911873e516 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMAT_EXPERIMENT.md @@ -0,0 +1,109 @@ +# gfx1250 scale-format discrimination experiment + +Status: planned; this experiment has not been implemented or run. Complete the +FP6 / scale-format PR split before executing it. Existing mixed-format fixtures +are separate evidence and do not count as execution of this plan. + +## Contents + +- [Question and evidence boundary](#question-and-evidence-boundary) +- [Minimal experiment](#minimal-experiment) +- [Literal predictions](#literal-predictions) +- [Controls and follow-up](#controls-and-follow-up) +- [Evidence and interpretation](#evidence-and-interpretation) + +## Question and evidence boundary + +Does scale selector 1 numerically decode unsigned E5M3 on gfx1250, distinctly +from selector 2 (E4M3), selector 0 (E8M0), or ignored scales? + +The public [machine-readable ISA archive](https://gpuopen.com/download/machine-readable-isa/latest/) +snapshot `AMD_GPU_MR_ISA_XML_2026_08_06.zip`, specifically +`amdgpu_isa_cdna5.xml`, describes SCALE and SCALE16, their block sizes of 32 and +16, and B32/B64 scale operands. It does not define E5M3 scale decoding or the +complete scale-format legality table. LLVM's `WMMA::MatrixScaleFmt` names E8, +E5M3, and E4M3 selectors 0, 1, and 2. These are distinct evidence sources: +compiler acceptance alone does not establish hardware numerical behavior. + +The experiment tests the encoding hypotheses below. It does not establish +undocumented combinations as an architectural guarantee. + +## Minimal experiment + +1. Use one native 16x16x128 atom with FP4 A, FP8 E4M3 B, and a zero accumulator. + Store BF16 output using the existing builder. Every predicted finite value + below is exactly representable in BF16. +2. Set `A[m, k0] = 1` and `B[k0, n] = 1` for every output row/column. Set all + other matrix entries to zero. For the builder's RCR storage, B is stored as + `[n, k0]`. Use raw FP4 code `0x2` and FP8 E4M3 code `0x38` for +1. +3. Set B scale selector to 0 and every B scale byte to E8M0 unity, `0x7f`. +4. Fill A's scale bytes with one raw code from the table. Compile variants with + A selectors 0, 1, and 2, changing only this selector. Keep the raw input bytes, + dimensions, and B selector identical across these variants. +5. Run first with `k0 = 0` and SCALE. Each of the 256 output elements should + equal the decoded A scale: there is exactly one nonzero product. +6. Compare with literal expected values, without calling the production or + verifier scale decoder. Record all outputs, including unexpected values. + +Initialize output memory to a NaN sentinel and fail missing stores. Distinguish +compilation, module load, launch, and numeric failures. Use a subprocess timeout +for each probe. A failure in the mixed-format case alone cannot distinguish +scale-format support from matrix-pair restrictions; use the FP4 control below. + +## Literal predictions + +| Raw A scale | Unsigned E5M3 (bias 15) | E4M3 (bias 7) | E8M0 | Scale ignored | +| --- | ---: | ---: | ---: | ---: | +| `0x3c` | `3/512` | `1.5` | `2**-67` | `1` | +| `0x7c` | `1.5` | `384` | `0.125` | `1` | +| `0x84` | `3` | `-1/128` | `32` | `1` | + +The first two rows distinguish the principal hypotheses using positive normal +values. The last row probes use of bit 7 as an exponent bit under E5M3; its +E4M3 value assumes ordinary signed decoding. Negative-scale handling may impose +additional restrictions, so an E4M3 mismatch on that row is not by itself a +failure of positive E4M3 support. Record any clamping or other behavior. + +For an E5M2 confusion control, ordinary E5M2 decodes `0x3c` as 1 and `0x7c` as +positive infinity, also distinct from the E5M3 hypothesis. Do not use infinity +as an expected pass value in the main finite-value test. + +## Controls and follow-up + +- Verify unity separately for selectors 0/1/2 with raw bytes `0x7f`/`0x78`/ + `0x38`. These controls do not replace the fixed-byte discrimination cases. +- Inspect the emitted LLVM/HIP arguments and decoded instruction fields/raw + instruction bytes. Confirm selector changes survive compilation to SCALE or + SCALE16. A compiler remapping must not be attributed to hardware semantics. +- Swap A and B: put FP4 and the tested scale on B, leaving FP8 A with E8M0 unity. +- Repeat for SCALE16. Move `k0` through 0, 15, 16, 31, 32, 63, 64, 95, 96, and + 127 to cover scale-group and lane-half boundaries. Uniform scales isolate + decoding; then vary only the active group's scale to check routing. +- Run FP4 x FP4 with matching selectors on A and B. Keep the untested operand's + scale at that selector's unity encoding. Since unity bytes differ between + selectors, report this as a separate control from the fixed-buffer experiment. +- Validate with COMGR and Python-generated HIP, using the same literal inputs. + Require Python/C++ LLVM identity where the C++ backend is available. Do not + infer C++ HIP compilation from Python HIP success. +- After these discriminating cases pass, sweep candidate finite E5M3 codes + 0..254 with an independent host reference, including zero, subnormals, and + high-exponent values. Treat code 255 and other special-value semantics as a + separate experiment. Retain raw output bits for any deviation. + +## Evidence and interpretation + +Record source SHA and patch hash, resolved device target, compiler/COMGR/runtime +versions, compilation route, generated source/IR, HSACO hash, disassembly, raw +input bytes, selectors, active K position, raw output bits, and expected values. +Use a machine-readable row for each combination and separate stage statuses. + +If selector 1 returns `3/512` and `1.5`, while selector 2 returns `1.5` and `384` +for the same raw bytes, the tested routes implement distinct numerical decoding. +Selector 1 returning 3 for `0x84` additionally supports unsigned E5M3 decoding. +Identical output vectors across selectors require checking instruction encoding +before concluding that the hardware ignores or aliases a selector. + +Passing establishes numerical behavior only for the tested target, instruction, +matrix pairs, selectors, and code points. It does not prove arbitrary-input +rounding, all format combinations, NaN behavior, or performance. Keep these +results separate from the FP6 packing validation and architectural documentation. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py index e3c00ef32e2b..c9fc4501ced3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py @@ -4,7 +4,7 @@ The default invocation keeps the K=64 FP8/BF8 WMMA + FP32-scale verifier. Native ``--matrix-path wmma_scale`` / ``wmma_scale16`` use independent FP8/BF8, FP6/BF6, or FP4 operands and -E8M0 scales with K=32 / K=16 groups. Native fixtures cover K=128 or 256 and use +encoded scales with K=32 / K=16 groups. Native fixtures cover K=128 or 256 and use bounded dyadic values, permitting exact comparison after output-type rounding. Run on visible HIP device 0 (must be gfx1250), for example:: @@ -123,6 +123,24 @@ def pack_fp4_codes(codes: np.ndarray) -> np.ndarray: return codes[:, 0::2] | (codes[:, 1::2] << 4) +def decode_scale(encoded: np.ndarray, dtype: str) -> np.ndarray: + """Decode finite nonnegative scale fixtures, independently of lowering.""" + if dtype in ("e8m0", "i8"): + return decode_e8m0(encoded) + if dtype not in ("e4m3", "e5m3"): + raise ValueError("scale dtype must be e8m0, e4m3, or e5m3") + maximum, bias = (126, 7) if dtype == "e4m3" else (254, 15) + if encoded.dtype != np.uint8 or np.any(encoded > maximum): + raise ValueError("expected finite nonnegative encoded scale bytes") + exponent = (encoded >> 3).astype(np.int32) + fraction = (encoded & 7).astype(np.float64) / 8 + return np.where( + exponent == 0, + np.ldexp(fraction, 1 - bias), + np.ldexp(1 + fraction, exponent - bias), + ) + + def reference_result( a: np.ndarray, b: np.ndarray, @@ -134,6 +152,8 @@ def reference_result( dtype_a: str | None = None, dtype_b: str | None = None, dtype_c: str = "bf16", + scale_dtype_a: str = "e8m0", + scale_dtype_b: str = "e8m0", ) -> np.ndarray: """Expand scales onto logical A/B elements, multiply, then round the output. @@ -141,6 +161,8 @@ def reference_result( scales 2**[-2,3]. At K<=256, even the sum of absolute products fits in 2**22 units of 2**-8, so every FP32 partial sum is exact. Float64 host arithmetic and a single output-type rounding provide an independent oracle. + Alternative scale fixtures use the bounded dyadic values 0.5, 0.75, 1, + 1.25, 1.5, and 1.75 with the same bounded matrix inputs. FP6 all-code fixtures isolate one K element, avoiding accumulation error. FP4 fixtures cover all E2M1 values (magnitude <=6), scales 2**[-2,1], and K<=256: absolute partial sums are below 2**22 units of 2**-6, @@ -148,8 +170,8 @@ def reference_result( """ output_dtype = _output_dtype(dtype_c) - sa = decode_e8m0(a_scale) if native else a_scale.astype(np.float64) - sb = decode_e8m0(b_scale) if native else b_scale.astype(np.float64) + sa = decode_scale(a_scale, scale_dtype_a) if native else a_scale.astype(np.float64) + sb = decode_scale(b_scale, scale_dtype_b) if native else b_scale.astype(np.float64) def matrix_values(data, dtype): if dtype in ("fp6", "fp6e2m3", "bf6", "fp6e3m2"): @@ -209,13 +231,33 @@ def operand(dtype, rows): for d in (spec.dtype_a, spec.dtype_b) ) - sa = rng.integers( - 125, 129 if small else 131, size=(spec.M, groups), dtype=np.uint8 - ) - sb = rng.integers( - 125, 129 if small else 131, size=(groups, spec.N), dtype=np.uint8 - ) - neutral_a = neutral_b = 127 + da, db = spec.resolved_scale_dtypes() + + def scales(dtype, shape): + if dtype in ("e8m0", "i8"): + return ( + rng.integers( + 125, 129 if small else 131, size=shape, dtype=np.uint8 + ), + 127, + ) + bias = 7 if dtype == "e4m3" else 15 + # Include non-power-of-two scales to distinguish these from E8M0. + codes = np.array( + [ + (bias - 1) * 8, + (bias - 1) * 8 + 4, + bias * 8, + bias * 8 + 2, + bias * 8 + 4, + bias * 8 + 6, + ], + dtype=np.uint8, + ) + return rng.choice(codes, size=shape), bias * 8 + + sa, neutral_a = scales(da, (spec.M, groups)) + sb, neutral_b = scales(db, (groups, spec.N)) else: sa = rng.uniform(0.5, 1.5, size=(spec.M, groups)).astype(np.float32) sb = rng.uniform(0.5, 1.5, size=(groups, spec.N)).astype(np.float32) @@ -346,6 +388,7 @@ def run_cases( fn = module.get_function(art.kernel_name) for case in cases: inputs = make_case_inputs(spec, case) + da, db = spec.resolved_scale_dtypes() expected = reference_result( *inputs, spec.block_k, @@ -353,10 +396,12 @@ def run_cases( dtype_a=spec.dtype_a, dtype_b=spec.dtype_b, dtype_c=spec.dtype_c, + scale_dtype_a=da, + scale_dtype_b=db, ) label = ( f"{spec.resolved_matrix_path()}/{spec.dtype_a}x{spec.dtype_b}/{compile_route}/{case} " - f"{spec.M}x{spec.N}x{spec.K} bk{spec.block_k}" + f"{spec.M}x{spec.N}x{spec.K} bk{spec.block_k} scales={da}/{db}" ) got = _launch(rt, fn, spec, inputs) try: diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/scale_formats_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/scale_formats_gemm.py new file mode 100644 index 000000000000..711078927fc8 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/scale_formats_gemm.py @@ -0,0 +1,54 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""GEMM with explicit A/B block-scale formats on gfx1250.""" + +from __future__ import annotations + +from ....core.dtypes import normalize_dtype +from ....instances.gfx1250.block_scaled_gemm import BlockScaledGemmSpec, _LOWBIT_DTYPES +from ._scaled_gemm_example import argument_parser, verify + +FORMATS = tuple(sorted(_LOWBIT_DTYPES)) + + +def make_spec(args) -> BlockScaledGemmSpec: + return BlockScaledGemmSpec( + name="scale_formats_gemm", + M=args.m, + N=args.n, + K=args.k, + dtype_a=args.dtype_a, + dtype_b=args.dtype_b, + dtype_c="bf16", + scale_dtype="e8m0", + scale_dtype_a=args.scale_dtype_a, + scale_dtype_b=args.scale_dtype_b, + matrix_path=args.matrix_path, + block_k=16 if args.matrix_path == "wmma_scale16" else 32, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argument_parser(__doc__) + parser.add_argument( + "--dtype-a", type=normalize_dtype, choices=FORMATS, default="fp4e2m1" + ) + parser.add_argument( + "--dtype-b", type=normalize_dtype, choices=FORMATS, default="fp4e2m1" + ) + parser.add_argument( + "--scale-dtype-a", choices=("e8m0", "e4m3", "e5m3"), default="e4m3" + ) + parser.add_argument( + "--scale-dtype-b", choices=("e8m0", "e4m3", "e5m3"), default="e4m3" + ) + args = parser.parse_args(argv) + try: + spec = make_spec(args) + except ValueError as exc: + parser.error(str(exc)) + return verify(spec, args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md index 69996035af60..c53b671ff30c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md @@ -44,9 +44,12 @@ These emit generic AMDGPU IR; arch only sets the comgr target triple. ## gfx1250 native scaled GEMM -`block_scaled_gemm` supports homogeneous FP8 E4M3, BF8 E5M2, FP6 E2M3, and FP6 E3M2 -through SCALE and SCALE16 with LLVM 23 and E8M0 scales. -See the [packed FP6 input contract](../examples/gfx1250/gemm/FP6.md). +`block_scaled_gemm` supports homogeneous and mixed FP8 E4M3, BF8 E5M2, +FP6 E2M3/E3M2, and FP4 E2M1 through SCALE and SCALE16 with LLVM 23. +E8M0 scales are accepted for every matrix pair. E4M3/E5M3 scales require the +corresponding operand to be FP4; FP4 x FP4 requires matching scale formats. +See the [scale-format contract](../examples/gfx1250/gemm/SCALE_FORMATS.md) +and [packed FP6 input contract](../examples/gfx1250/gemm/FP6.md). ## GEMM family diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py index 26650aa3a3e0..c98ac0f8d7f8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py @@ -4,7 +4,7 @@ The default path preserves the existing K=64 FP8/BF8 WMMA plus software post-scaling contract. ``matrix_path="wmma_scale"`` and ``"wmma_scale16"`` -select the native gfx1250 K=128 instructions and consume packed E8M0 scale +select the native gfx1250 K=128 instructions and consume packed encoded scale bytes directly. """ @@ -99,6 +99,18 @@ class BlockScaledGemmSpec: tile_m: int = 16 tile_n: int = 16 tile_k: int = 128 + scale_dtype_a: str | None = None + scale_dtype_b: str | None = None + + def resolved_scale_dtypes(self) -> tuple[str, str]: + aliases = {"i8": "e8m0", "fp8e4m3": "e4m3"} + return tuple( + aliases.get(dtype, dtype) + for dtype in ( + self.scale_dtype if self.scale_dtype_a is None else self.scale_dtype_a, + self.scale_dtype if self.scale_dtype_b is None else self.scale_dtype_b, + ) + ) def __post_init__(self) -> None: object.__setattr__(self, "dtype_a", normalize_dtype(self.dtype_a)) @@ -116,7 +128,15 @@ def kernel_name(self) -> str: f"M{self.M}N{self.N}K{self.K}", f"bk{self.block_k}", f"t{self.tile_m}x{self.tile_n}x{self.tile_k}", - flags={self.resolved_matrix_path(): True}, + flags={ + self.resolved_matrix_path(): True, + **{ + f"s{operand}_{dtype}": True + for operand, dtype in zip("ab", self.resolved_scale_dtypes()) + if self.resolved_matrix_path() in ("wmma_scale", "wmma_scale16") + and dtype != "e8m0" + }, + }, ) def resolved_matrix_path(self) -> str: @@ -128,13 +148,12 @@ def resolved_matrix_path(self) -> str: def _native_scaled_atom(spec: BlockScaledGemmSpec, target: ArchTarget): - scale_dtype = "e8m0" if spec.scale_dtype == "i8" else spec.scale_dtype return target.mma.op_for_shape( family="wmma_scaled", a_dtype=spec.dtype_a, b_dtype=spec.dtype_b, c_dtype=spec.dtype_acc, - scales=(scale_dtype, scale_dtype, spec.block_k), + scales=(*spec.resolved_scale_dtypes(), spec.block_k), m=_BLOCK_M, n=_BLOCK_N, k=_WMMA_SCALE_K, @@ -217,6 +236,8 @@ def is_valid_spec(spec: BlockScaledGemmSpec, arch: str = "gfx1250") -> Tuple[boo ) else: + if spec.scale_dtype_a is not None or spec.scale_dtype_b is not None: + return False, "per-operand scale types require native scaled WMMA" try: _wire_scale_dtype(spec.scale_dtype) except ValueError as e: @@ -295,7 +316,7 @@ def build_block_scaled_gemm( One wave (32 lanes) computes one 16x16 output tile without LDS. The legacy ``wmma`` path uses K=64 FP8/BF8 atoms, accumulates each ``block_k`` group, and applies FP16/FP32 A/B scales in software. The native ``wmma_scale`` and - ``wmma_scale16`` paths use K=128 scaled WMMA atoms and pass packed E8M0 scale bytes + ``wmma_scale16`` paths use K=128 scaled WMMA atoms and pass packed encoded scale bytes directly to the instruction, with K=32 and K=16 scale groups respectively. Lane ``l`` owns output column ``l % 16`` and rows diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scale_formats_llvm23.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scale_formats_llvm23.json new file mode 100644 index 000000000000..5586419c9775 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scale_formats_llvm23.json @@ -0,0 +1,38 @@ +{ + "fp8/fp4/e8m0/e5m3/32": "30ccc3e6c174fb04ee0eea2a53785b94abea4718ad12e05ed096b2d61304e33a", + "fp8/fp4/e8m0/e4m3/32": "e128ba512121528e58d278de8c915141fe132cbdcbb9e7c83b7977d2f4816a8a", + "bf8/fp4/e8m0/e5m3/32": "42c2b4e2aaa30d213877b5f382f6c497b1501734b78f89833bf9e9b9f0ef5b27", + "bf8/fp4/e8m0/e4m3/32": "fb879f5bf3638dababa5ec0566f9363576197f764e941449ab5c3466765f8824", + "fp6/fp4/e8m0/e5m3/32": "10a54a109d1bd5fa194987d58af80c4da5bec73f122a2beec0ea13e26e3cec07", + "fp6/fp4/e8m0/e4m3/32": "45caa681fb90e97b91cc019ff9f463deb3dbc9e6f39dbfd4f57d6595ed3b999f", + "bf6/fp4/e8m0/e5m3/32": "64678e92c553bde8f84e5408bfaf8edf8a7598f8611dc31e87a78d8a4eb6fd96", + "bf6/fp4/e8m0/e4m3/32": "2b3baae1500037984118e285daf36f9bc0bd750e8605b9582575502442e87156", + "fp4/fp8/e5m3/e8m0/32": "90d73b9042991c85c8c2754017bc8851cac85c35e4042040860e7dfa36e06f92", + "fp4/fp8/e4m3/e8m0/32": "82c5a3b98b01ec3d800ad037a7d7b36038c3d2c4529f68394818ed7b6fdf49b9", + "fp4/bf8/e5m3/e8m0/32": "7982a79f47498266261f06b0327d818d02f04c84028becbcde435f79f5524c0d", + "fp4/bf8/e4m3/e8m0/32": "f4289f3bf2c6aa8ca77e20271a40beff2c57d25a1b2e76310e0b632abe09a3aa", + "fp4/fp6/e5m3/e8m0/32": "fbf66e7b980a8ccff36f5aac10d27bfab392603f31c4c0fc0f638a7e5434c296", + "fp4/fp6/e4m3/e8m0/32": "78c944f67b5cf567bd52e1de05b0a772ac3f5ee3ad80a23e9308ba219515a952", + "fp4/bf6/e5m3/e8m0/32": "403bfd647ec51292028c68de470eb953d17b721b99554c2029c978ca4b96d275", + "fp4/bf6/e4m3/e8m0/32": "635fd80af649d1371230ffb4944246ad1003526e193aa646a08196a120d62de7", + "fp4/fp4/e5m3/e5m3/32": "f2e235e6f5a5ec9507d62c0861c26b55528e700c9a0820bbf9a6e17a0c9bc073", + "fp4/fp4/e4m3/e4m3/32": "b1915e0f0fa86dc3e264ac9b209e3206e19c40533dd7636ad4e249f23ac9d3cd", + "fp8/fp4/e8m0/e5m3/16": "c7aa1f952ae7d0edb5e793bd85aead1699cce86490e667ea18c7c3f39da34677", + "fp8/fp4/e8m0/e4m3/16": "2b4ff129e0cdb1aef990a687159920ba87433f800b246660aa05d4f9efb3fdc5", + "bf8/fp4/e8m0/e5m3/16": "853a99283a3a48fbb0ae4ddded015b933b86e6645681dd224aa68e0d5a861100", + "bf8/fp4/e8m0/e4m3/16": "89c881d05ec14564f167754bb431573b896653116d228ea3d568abefc566ef2d", + "fp6/fp4/e8m0/e5m3/16": "3aa354187620ed4dec9ec1261812ad3442f8447d6ea0de8ede4f7fdf0375c6af", + "fp6/fp4/e8m0/e4m3/16": "fa9ae50bb05a53d7e72aa6f9f7d2c40548f6cdd8a48274ffb61eff1ab53ccf7d", + "bf6/fp4/e8m0/e5m3/16": "696f88b8f459df4c890c187c8d101c448cd90bcb5b00dec81efb622a82f8e86f", + "bf6/fp4/e8m0/e4m3/16": "2ddc5ce2de8d62d3fc1bf7c4e5625ef340b9e3528468d1bd65ad9d8eb845b4c1", + "fp4/fp8/e5m3/e8m0/16": "760cca5c37584b92929e489341606154e4e64f2bb5d4767e7f76d6d6b02c8339", + "fp4/fp8/e4m3/e8m0/16": "1a1d3b1db8bfe2be84d86d4b2f07296ae27c72fddd2c51a6da4c60bd2cbfac82", + "fp4/bf8/e5m3/e8m0/16": "55f458946efa8f78ccda0655644ca9a0fb9511e7995cd1c3e396b522b217c0ea", + "fp4/bf8/e4m3/e8m0/16": "6f34408015fddf8b0c5226c16cd4ed5ef22383185a4e4f62020e9cbd2cffa003", + "fp4/fp6/e5m3/e8m0/16": "d6ca8e28c6713a1e0c4774327d605c3e2ad04d532899b6558f87be3d7b0b15de", + "fp4/fp6/e4m3/e8m0/16": "ede9e8e558679ee8a219dad177519d57082100fe9b4fe1bba855c1ca3ee7dd4d", + "fp4/bf6/e5m3/e8m0/16": "019966f88128db2888868c5568e5e67c6f6253d0bfc9ba146df3c994b511ebd7", + "fp4/bf6/e4m3/e8m0/16": "63d78afafe18897be39f198b35e45c8bdc784da473740fcfa62ccfe2e1228d98", + "fp4/fp4/e5m3/e5m3/16": "26357c251abb22209343989411c24fdfad92966edf1f0def917b5f200f95785c", + "fp4/fp4/e4m3/e4m3/16": "550edeea0920380451e2cbc22d4893c68c464c4634fdc85aa2f49d8ce35c0094" +} diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py index eb4412f40757..3df2d4600ef5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py @@ -223,8 +223,8 @@ def test_e5m2_is_not_a_scale_dtype_alias(dtype, field): def test_scaled_catalog_identity_and_backend_contract(): catalog = ArchTarget.from_gfx("gfx1250").mma rows = [row for row in catalog.ops if row.family == "wmma_scaled"] - assert len(rows) == 50 - assert len({row.op_id for row in rows}) == 50 + assert len(rows) == 86 + assert len({row.op_id for row in rows}) == 86 for row in rows: tokens = { "fp8e4m3": "fp8", @@ -235,10 +235,10 @@ def test_scaled_catalog_identity_and_backend_contract(): } assert row.op_id == ( f"wmma_gfx1250_f32_16x16x128_{tokens[row.a_dtype]}_{tokens[row.b_dtype]}" - f"_scale_e8m0_e8m0_k{row.scale_block_k}" + f"_scale_{row.a_scale_dtype}_{row.b_scale_dtype}_k{row.scale_block_k}" ) - assert row.a_scale_dtype == row.b_scale_dtype == "e8m0" - assert row.a_scale_dtype is row.b_scale_dtype is MmaScaleDType.E8M0 + assert isinstance(row.a_scale_dtype, MmaScaleDType) + assert isinstance(row.b_scale_dtype, MmaScaleDType) assert isinstance(row.scale_block_k, MmaScaleBlockK) packing = gfx1250_scaled_wmma(row.op_id) assert packing.atom is row @@ -253,6 +253,10 @@ def test_scaled_catalog_identity_and_backend_contract(): selectors[row.a_dtype], selectors[row.b_dtype], ) + assert packing.scale_formats == tuple( + {"e8m0": 0, "e5m3": 1, "e4m3": 2}[dtype] + for dtype in (row.a_scale_dtype, row.b_scale_dtype) + ) assert packing.scales.count * packing.scales.block_k == row.k assert (row.a_frag_len, row.b_frag_len) == (16, 16) for family in ("wmma_scale", "wmma_scale16"): diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py index 4c750ca6b1e2..7e4431ab3868 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py @@ -105,7 +105,7 @@ def test_mixed_atoms_do_not_leak_to_other_targets(a, b): ) -@pytest.mark.parametrize("scales", [("e4m3", "e8m0", 32), ("e8m0", "e5m3", 16)]) +@pytest.mark.parametrize("scales", [("e4m3", "e8m0", 32), ("e5m3", "e8m0", 16)]) def test_mixed_atoms_do_not_admit_other_scale_contracts(scales): assert ( ArchTarget.from_gfx("gfx1250").mma.op_for_shape( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py new file mode 100644 index 000000000000..b006dcda7dfc --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py @@ -0,0 +1,239 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Scale legality, independent selectors, and encoded-byte reference checks.""" + +import hashlib +from itertools import product +import json +from pathlib import Path +from dataclasses import replace +import re + +import numpy as np +import pytest + +from rocke.core.arch import ArchTarget +from rocke.core.arch.wmma_scale import gfx1250_scaled_wmma +from rocke.core.backend import resolve_backend +from rocke.core.lower_hip import lower_kernel_to_hip +from rocke.core.lower_llvm import lower_kernel_to_llvm +from rocke.examples.gfx1250.gemm.block_scaled_gemm_verify import ( + decode_scale, + make_case_inputs, + reference_result, +) +from rocke.instances.gfx1250.block_scaled_gemm import ( + BlockScaledGemmSpec, + build_block_scaled_gemm, + is_valid_spec, +) + +FORMATS = ("fp8", "bf8", "fp6", "bf6", "fp4") +SCALES = ("e8m0", "e5m3", "e4m3") +PAIRS = ( + [(d, "fp4", "e8m0", s) for d in FORMATS[:-1] for s in SCALES[1:]] + + [("fp4", d, s, "e8m0") for d in FORMATS[:-1] for s in SCALES[1:]] + + [("fp4", "fp4", s, s) for s in SCALES[1:]] +) + + +def spec_for(a="fp4", b="fp4", sa="e4m3", sb="e4m3", block_k=32, **kwargs): + return BlockScaledGemmSpec( + name="scale_formats_test", + M=32, + N=48, + K=256, + dtype_a=a, + dtype_b=b, + scale_dtype="e8m0", + scale_dtype_a=sa, + scale_dtype_b=sb, + matrix_path="wmma_scale16" if block_k == 16 else "wmma_scale", + block_k=block_k, + **kwargs, + ) + + +@pytest.mark.parametrize("a,b,sa,sb", list(product(FORMATS, FORMATS, SCALES, SCALES))) +@pytest.mark.parametrize("block_k", [32, 16]) +def test_catalog_and_builder_agree_on_legal_scale_combinations(a, b, sa, sb, block_k): + expected = ( + (sa == "e8m0" or a == "fp4") + and (sb == "e8m0" or b == "fp4") + and (a != "fp4" or b != "fp4" or sa == sb) + ) + atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( + family="wmma_scaled", + a_dtype=a, + b_dtype=b, + c_dtype="fp32", + m=16, + n=16, + k=128, + scales=(sa, sb, block_k), + ) + assert (atom is not None) == expected + assert is_valid_spec(spec_for(a, b, sa, sb, block_k))[0] == expected + + +@pytest.mark.parametrize("a,b,sa,sb", PAIRS) +@pytest.mark.parametrize("block_k", [32, 16]) +@pytest.mark.parametrize("dtype_c", ["bf16", "fp16"]) +def test_scale_formats_lowering_and_reference(a, b, sa, sb, block_k, dtype_c): + spec = spec_for(a, b, sa, sb, block_k, dtype_c=dtype_c) + atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( + family="wmma_scaled", + a_dtype=a, + b_dtype=b, + c_dtype="fp32", + m=16, + n=16, + k=128, + scales=(sa, sb, block_k), + ) + assert ( + atom.op_id == f"wmma_gfx1250_f32_16x16x128_{a}_{b}_scale_{sa}_{sb}_k{block_k}" + ) + contract = gfx1250_scaled_wmma(atom.op_id) + assert contract.scale_formats == (SCALES.index(sa), SCALES.index(sb)) + kernel = build_block_scaled_gemm(spec) + llvm = lower_kernel_to_llvm(kernel, arch="gfx1250", llvm_flavor="llvm23") + calls = [ + line + for line in llvm.splitlines() + if "call <8 x float> @llvm.amdgcn.wmma.scale" in line + ] + assert len(calls) == 2 + for call in calls: + assert re.findall(r"i32 0, i32 (\d+), i(?:32|64) %", call) == [ + str(SCALES.index(sa)), + str(SCALES.index(sb)), + ] + hip = lower_kernel_to_hip(kernel, arch="gfx1250") + hip_calls = [ + line for line in hip.splitlines() if "__builtin_amdgcn_wmma_scale" in line + ] + assert len(hip_calls) == 2 + for call in hip_calls: + assert re.findall(r", 0, (\d+), ", call) == [ + str(SCALES.index(sa)), + str(SCALES.index(sb)), + ] + for flavor in ("llvm20", "llvm22"): + error = RuntimeError if resolve_backend() == "cpp" else NotImplementedError + with pytest.raises(error, match="requires llvm23"): + lower_kernel_to_llvm(kernel, arch="gfx1250", llvm_flavor=flavor) + inputs = make_case_inputs(spec, "mixed") + result = reference_result( + *inputs, + block_k, + native=True, + dtype_a=a, + dtype_b=b, + dtype_c=dtype_c, + scale_dtype_a=sa, + scale_dtype_b=sb, + ) + assert np.isfinite(result).all() and np.any(result) + + +@pytest.mark.parametrize( + "dtype,bias,max_code,max_value", [("e4m3", 7, 126, 448), ("e5m3", 15, 254, 114688)] +) +def test_scale_decoding(dtype, bias, max_code, max_value): + codes = np.array([0, 1, 8, bias * 8, bias * 8 + 4, max_code], dtype=np.uint8) + np.testing.assert_array_equal( + decode_scale(codes, dtype), + [0, 2.0 ** (-bias - 2), 2.0 ** (1 - bias), 1, 1.5, max_value], + ) + with pytest.raises(ValueError): + decode_scale(np.array([max_code + 1], dtype=np.uint8), dtype) + + +def test_e4m3_decoder_against_independent_dtype(): + ml = pytest.importorskip("ml_dtypes") + codes = np.arange(127, dtype=np.uint8) + np.testing.assert_array_equal( + decode_scale(codes, "e4m3"), codes.view(ml.float8_e4m3fn).astype(np.float64) + ) + + +@pytest.mark.parametrize("dtype", ["e5m2", "bf8e5m2", "fp8e5m2", "bf8", "", "i32"]) +@pytest.mark.parametrize("field", ["scale_dtype_a", "scale_dtype_b"]) +def test_invalid_scale_spellings(dtype, field): + spec = replace(spec_for(), **{field: dtype}) + assert not is_valid_spec(spec)[0] + with pytest.raises(ValueError): + build_block_scaled_gemm(spec) + + +def test_per_operand_overrides_preserve_defaults_and_aliases(): + implicit = spec_for(sa=None, sb=None) + explicit = replace(implicit, scale_dtype_a="e8m0", scale_dtype_b="i8") + assert implicit.kernel_name() == explicit.kernel_name() + lower = lambda spec: lower_kernel_to_llvm( + build_block_scaled_gemm(spec), arch="gfx1250", llvm_flavor="llvm23" + ) + assert lower(implicit) == lower(explicit) + alias = replace( + spec_for(), scale_dtype_a="fp8e4m3", scale_dtype_b=None, scale_dtype="e4m3" + ) + assert alias.resolved_scale_dtypes() == ("e4m3", "e4m3") + assert alias.kernel_name() == spec_for().kernel_name() + assert lower(alias) == lower(spec_for()) + assert lower(implicit) != lower(spec_for()) + + +def test_legacy_wmma_rejects_per_operand_scale_overrides(): + spec = replace( + spec_for(), + dtype_a="fp8", + dtype_b="fp8", + matrix_path="wmma", + scale_dtype="fp32", + block_k=128, + ) + assert is_valid_spec(spec) == ( + False, + "per-operand scale types require native scaled WMMA", + ) + + +@pytest.mark.parametrize( + "case,expected_sha", + json.loads( + Path(__file__).with_name("gfx1250_scale_formats_llvm23.json").read_text() + ).items(), +) +def test_scale_formats_golden(case, expected_sha): + a, b, sa, sb, block_k = case.split("/") + llvm = lower_kernel_to_llvm( + build_block_scaled_gemm(spec_for(a, b, sa, sb, int(block_k))), + arch="gfx1250", + llvm_flavor="llvm23", + ) + assert hashlib.sha256(llvm.encode()).hexdigest() == expected_sha + + +@pytest.mark.parametrize( + "a,b,sa,sb", [("fp8", "fp4", "e4m3", "e8m0"), ("fp4", "fp4", "e4m3", "e5m3")] +) +def test_backend_rejects_invalid_contract_even_if_catalog_injected( + monkeypatch, a, b, sa, sb +): + from rocke.core.arch.target import MmaCatalog + + atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( + family="wmma_scaled", + a_dtype=a, + b_dtype=b, + c_dtype="fp32", + m=16, + n=16, + k=128, + scales=("e8m0", "e8m0", 32), + ) + invalid = replace(atom, a_scale_dtype=sa, b_scale_dtype=sb) + monkeypatch.setattr(MmaCatalog, "by_op_id", lambda self, op_id: invalid) + with pytest.raises(ValueError, match="unsupported scaled WMMA backend contract"): + gfx1250_scaled_wmma(atom.op_id) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp index 6804d32756bb..e898d5041246 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp @@ -304,10 +304,22 @@ static int test_scale_contracts() op.scale_block_k); CHECK(strcmp(id, op.op_id) == 0); CHECK(rocke_mma_catalog_by_op_id(&arch->mma, id) == &op); + if(strcmp(op.a_dtype, "fp4e2m1") != 0) + { + auto invalid = op; + invalid.a_scale_dtype = "e4m3"; + CHECK(rejects_query([&] { rocke_scaled_wmma_contract(&invalid); })); + } + if(strcmp(op.a_dtype, "fp4e2m1") == 0 && strcmp(op.b_dtype, "fp4e2m1") == 0) + { + auto invalid = op; + invalid.b_scale_dtype = strcmp(op.a_scale_dtype, "e4m3") == 0 ? "e5m3" : "e4m3"; + CHECK(rejects_query([&] { rocke_scaled_wmma_contract(&invalid); })); + } ++scaled_rows; } } - CHECK(packed_rows == 4 && scaled_rows == 50); + CHECK(packed_rows == 4 && scaled_rows == 86); return 0; } @@ -376,7 +388,7 @@ static int test_scale_layouts_and_families() } } } - CHECK(scaled == 50); + CHECK(scaled == 86); return 0; } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c index c4f0c90efda8..477657a47305 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c @@ -120,10 +120,15 @@ static void build_wmma_k64_bf8_bf8(rocke_ir_builder_t* b) /* K=128 FP8 SCALE/SCALE16 WMMA. Matrix fragments are <16 x i32>; packed * E8M0 scale operands are i32 for SCALE and i64 for SCALE16. */ -static void wmma_scaled(rocke_ir_builder_t* b, bool scale16, const char* dtype, const char* dtype_b) +static void wmma_scaled(rocke_ir_builder_t* b, + bool scale16, + const char* dtype, + const char* dtype_b, + const char* scale_a, + const char* scale_b) { const rocke_mma_scale_block_k_t block = scale16 ? ROCKE_MMA_SCALE_K16 : ROCKE_MMA_SCALE_K32; - const rocke_mma_scale_filter_t query = {"e8m0", "e8m0", block}; + const rocke_mma_scale_filter_t query = {scale_a, scale_b, block}; const rocke_arch_target_t* target = rocke_arch_target_from_gfx("gfx1250"); const rocke_mma_op_t* atom = rocke_mma_catalog_op_for_shape( &target->mma, "wmma_scaled", dtype, dtype_b, "fp32", 16, 16, 128, &query); @@ -152,32 +157,32 @@ static void wmma_scaled(rocke_ir_builder_t* b, bool scale16, const char* dtype, static void build_wmma_scale(rocke_ir_builder_t* b) { - wmma_scaled(b, false, "fp8e4m3", "fp8e4m3"); + wmma_scaled(b, false, "fp8e4m3", "fp8e4m3", "e8m0", "e8m0"); } static void build_wmma_scale_bf8(rocke_ir_builder_t* b) { - wmma_scaled(b, false, "bf8e5m2", "bf8e5m2"); + wmma_scaled(b, false, "bf8e5m2", "bf8e5m2", "e8m0", "e8m0"); } static void build_wmma_scale16(rocke_ir_builder_t* b) { - wmma_scaled(b, true, "fp8e4m3", "fp8e4m3"); + wmma_scaled(b, true, "fp8e4m3", "fp8e4m3", "e8m0", "e8m0"); } static void build_wmma_scale16_bf8(rocke_ir_builder_t* b) { - wmma_scaled(b, true, "bf8e5m2", "bf8e5m2"); + wmma_scaled(b, true, "bf8e5m2", "bf8e5m2", "e8m0", "e8m0"); } static void build_wmma_scale_fp4(rocke_ir_builder_t* b) { - wmma_scaled(b, false, "fp4e2m1", "fp4e2m1"); + wmma_scaled(b, false, "fp4e2m1", "fp4e2m1", "e8m0", "e8m0"); } static void build_wmma_scale16_fp4(rocke_ir_builder_t* b) { - wmma_scaled(b, true, "fp4e2m1", "fp4e2m1"); + wmma_scaled(b, true, "fp4e2m1", "fp4e2m1", "e8m0", "e8m0"); } /* ds_read_b128_tr_b16. gfx950 has one type-agnostic opcode returning @@ -396,6 +401,8 @@ typedef struct config const char* dtype; bool scale16; const char* dtype_b; + const char* scale_a; + const char* scale_b; } config_t; /* Each gfx1250 config that tests a *choice* of encoding is followed by its @@ -474,6 +481,42 @@ static const config_t CONFIGS[] = { {NULL, "gfx1250", "fp4", true, "bf8"}, {NULL, "gfx1250", "fp4", true, "fp6"}, {NULL, "gfx1250", "fp4", true, "bf6"}, + {NULL, "gfx1250", "fp8", false, "fp4", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp8", false, "fp4", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf8", false, "fp4", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf8", false, "fp4", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp6", false, "fp4", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp6", false, "fp4", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf6", false, "fp4", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf6", false, "fp4", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "fp8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp4", false, "fp8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp4", false, "bf8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp4", false, "bf8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp4", false, "fp6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp4", false, "fp6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp4", false, "bf6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp4", false, "bf6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp4", false, "fp4", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp4", false, "fp4", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp8", true, "fp4", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "fp4", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf8", true, "fp4", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf8", true, "fp4", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp6", true, "fp4", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp6", true, "fp4", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf6", true, "fp4", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf6", true, "fp4", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "fp8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp4", true, "fp8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp4", true, "bf8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp4", true, "bf8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp4", true, "fp6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp4", true, "fp6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp4", true, "bf6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp4", true, "bf6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp4", true, "fp4", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp4", true, "fp4", "e4m3", "e4m3"}, }; static const int NUM_CONFIGS = (int)(sizeof(CONFIGS) / sizeof(CONFIGS[0])); @@ -517,7 +560,9 @@ int main(int argc, char** argv) wmma_scaled(&b, CONFIGS[idx].scale16, CONFIGS[idx].dtype, - CONFIGS[idx].dtype_b ? CONFIGS[idx].dtype_b : CONFIGS[idx].dtype); + CONFIGS[idx].dtype_b ? CONFIGS[idx].dtype_b : CONFIGS[idx].dtype, + CONFIGS[idx].scale_a ? CONFIGS[idx].scale_a : "e8m0", + CONFIGS[idx].scale_b ? CONFIGS[idx].scale_b : "e8m0"); } if(!rocke_ir_builder_ok(&b)) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py index b74097b930bc..e2b4ddd16522 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py @@ -75,7 +75,7 @@ def build(b: IRBuilder) -> None: return build -def _wmma_scaled(a_kind, b_kind, scale_mode): +def _wmma_scaled(a_kind, b_kind, scale_mode, scale_a="e8m0", scale_b="e8m0"): """K=128 scaled WMMA, parameterized by operand dtypes and scale mode.""" scale_ty = {"scale": I32, "scale16": I64}[scale_mode] block_k = 16 if scale_mode == "scale16" else 32 @@ -84,7 +84,7 @@ def _wmma_scaled(a_kind, b_kind, scale_mode): a_dtype=a_kind, b_dtype=b_kind, c_dtype="fp32", - scales=("e8m0", "e8m0", block_k), + scales=(scale_a, scale_b, block_k), m=16, n=16, k=128, @@ -321,6 +321,32 @@ def build(b): ) +CONFIGS.extend( + (_wmma_scaled(a, b, mode, sa, sb), "gfx1250") + for mode in ("scale", "scale16") + for a, b, sa, sb in [ + ("fp8", "fp4", "e8m0", "e5m3"), + ("fp8", "fp4", "e8m0", "e4m3"), + ("bf8", "fp4", "e8m0", "e5m3"), + ("bf8", "fp4", "e8m0", "e4m3"), + ("fp6", "fp4", "e8m0", "e5m3"), + ("fp6", "fp4", "e8m0", "e4m3"), + ("bf6", "fp4", "e8m0", "e5m3"), + ("bf6", "fp4", "e8m0", "e4m3"), + ("fp4", "fp8", "e5m3", "e8m0"), + ("fp4", "fp8", "e4m3", "e8m0"), + ("fp4", "bf8", "e5m3", "e8m0"), + ("fp4", "bf8", "e4m3", "e8m0"), + ("fp4", "fp6", "e5m3", "e8m0"), + ("fp4", "fp6", "e4m3", "e8m0"), + ("fp4", "bf6", "e5m3", "e8m0"), + ("fp4", "bf6", "e4m3", "e8m0"), + ("fp4", "fp4", "e5m3", "e5m3"), + ("fp4", "fp4", "e4m3", "e4m3"), + ] +) + + def _spec(idx: int): """Config selector: the (builder, arch) pair the shared driver expects.""" if not 0 <= idx < len(CONFIGS): diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py new file mode 100644 index 000000000000..bd62c9433384 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py @@ -0,0 +1,82 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Scale example preserves independent selectors through real lowering.""" + +import argparse +import pytest +from rocke.examples.gfx1250.gemm.scale_formats_gemm import make_spec +from rocke.instances.gfx1250.block_scaled_gemm import ( + build_block_scaled_gemm, + is_valid_spec, +) +from rocke.core.arch.target import normalize_dtype +from rocke.core.lower_llvm import lower_kernel_to_llvm + + +@pytest.mark.parametrize( + "a,b,sa,sb", [("fp4", "fp4", "e4m3", "e4m3"), ("fp6", "fp4", "e8m0", "e5m3")] +) +def test_scale_example_contract(a, b, sa, sb): + spec = make_spec( + argparse.Namespace( + m=32, + n=48, + k=256, + matrix_path="wmma_scale16", + dtype_a=a, + dtype_b=b, + scale_dtype_a=sa, + scale_dtype_b=sb, + ) + ) + assert spec.resolved_scale_dtypes() == (sa, sb) + assert is_valid_spec(spec)[0] + assert "@llvm.amdgcn.wmma.scale16" in lower_kernel_to_llvm( + build_block_scaled_gemm(spec), arch="gfx1250", llvm_flavor="llvm23" + ) + + +@pytest.mark.parametrize( + "dtype", + [ + "fp8", + "bf8", + "fp8e4m3", + "bf8e5m2", + "fp6", + "bf6", + "fp6e2m3", + "fp6e3m2", + "fp4", + "fp4e2m1", + ], +) +@pytest.mark.parametrize("operand", ["a", "b"]) +def test_scale_formats_gemm_cli_normalizes_matrix_names(monkeypatch, dtype, operand): + from rocke.examples.gfx1250.gemm import scale_formats_gemm as example + + calls = [] + + def verify(spec, args): + calls.append(spec) + llvm = lower_kernel_to_llvm( + build_block_scaled_gemm(spec), arch="gfx1250", llvm_flavor="llvm23" + ) + assert "@llvm.amdgcn.wmma.scale" in llvm + return 0 + + monkeypatch.setattr(example, "verify", verify) + argv = [ + "--dtype-a", + "fp4", + "--dtype-b", + "fp4", + "--scale-dtype-a", + "e8m0", + "--scale-dtype-b", + "e8m0", + ] + argv += [f"--dtype-{operand}", dtype] + assert example.main(argv) == 0 + assert len(calls) == 1 + assert getattr(calls[0], f"dtype_{operand}") == normalize_dtype(dtype) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py new file mode 100644 index 000000000000..0c066343328c --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py @@ -0,0 +1,63 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""gfx1250 independent-scale numeric fixtures; distinct from the planned probe.""" + +import subprocess +import sys + +import pytest + +_MODULE = "rocke.examples.gfx1250.gemm.scale_formats_gemm" + +# Additional scale encodings through COMGR, with representative HIP cases. +_FORMATS = ("fp8e4m3", "bf8e5m2", "fp6", "bf6", "fp4") +_MIXED = ( + [(a, "fp4", "e8m0", scale) for a in _FORMATS[:-1] for scale in ("e5m3", "e4m3")] + + [("fp4", b, scale, "e8m0") for b in _FORMATS[:-1] for scale in ("e5m3", "e4m3")] + + [("fp4", "fp4", scale, scale) for scale in ("e5m3", "e4m3")] +) + + +@pytest.mark.parametrize("matrix_path", ["wmma_scale", "wmma_scale16"]) +@pytest.mark.parametrize( + "a,b,sa,sb,route", + [(*case, route) for case in _MIXED for route in ("comgr", "hip")], +) +def test_scale_format_wmma_numeric(gpu_env, matrix_path, a, b, sa, sb, route): + result = subprocess.run( + [ + sys.executable, + "-m", + _MODULE, + "--dtype-a", + a, + "--dtype-b", + b, + "--scale-dtype-a", + sa, + "--scale-dtype-b", + sb, + "--matrix-path", + matrix_path, + "--compile-route", + route, + "--m", + "32", + "--n", + "48", + "--k", + "256", + "--case", + "all", + ], + env=gpu_env, + capture_output=True, + text=True, + timeout=300, + ) + output = result.stdout + result.stderr + assert result.returncode == 0, output + count = 20 if matrix_path == "wmma_scale16" else 12 + assert f"PASS: verified {count} cases" in output, output + assert output.count("bad=0") == count, output + print(output, end="") From f11189fa5ac04d2e39f1b79baf253ff3a96e4ef2 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:23:07 -0400 Subject: [PATCH 23/32] fix(rocke): honor scale example output dtype Forward the shared CLI output choice into the scale-format spec. Cover BF16 and FP16 through SCALE/SCALE16 lowering, CLI defaults and rejection, and both numerical compiler routes. --- .../gfx1250/gemm/scale_formats_gemm.py | 2 +- .../test_gfx1250_scale_formats_example.py | 44 +++++++++++++++++-- .../test_gfx1250_scale_formats_numeric.py | 10 ++++- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/scale_formats_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/scale_formats_gemm.py index 711078927fc8..29eb2bfd3cd0 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/scale_formats_gemm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/scale_formats_gemm.py @@ -19,7 +19,7 @@ def make_spec(args) -> BlockScaledGemmSpec: K=args.k, dtype_a=args.dtype_a, dtype_b=args.dtype_b, - dtype_c="bf16", + dtype_c=args.output_dtype, scale_dtype="e8m0", scale_dtype_a=args.scale_dtype_a, scale_dtype_b=args.scale_dtype_b, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py index bd62c9433384..34875c5df316 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py @@ -16,13 +16,16 @@ @pytest.mark.parametrize( "a,b,sa,sb", [("fp4", "fp4", "e4m3", "e4m3"), ("fp6", "fp4", "e8m0", "e5m3")] ) -def test_scale_example_contract(a, b, sa, sb): +@pytest.mark.parametrize("matrix_path", ["wmma_scale", "wmma_scale16"]) +@pytest.mark.parametrize("output_dtype", ["bf16", "fp16"]) +def test_scale_example_contract(a, b, sa, sb, matrix_path, output_dtype): spec = make_spec( argparse.Namespace( m=32, n=48, k=256, - matrix_path="wmma_scale16", + matrix_path=matrix_path, + output_dtype=output_dtype, dtype_a=a, dtype_b=b, scale_dtype_a=sa, @@ -30,10 +33,45 @@ def test_scale_example_contract(a, b, sa, sb): ) ) assert spec.resolved_scale_dtypes() == (sa, sb) + assert spec.dtype_c == output_dtype assert is_valid_spec(spec)[0] - assert "@llvm.amdgcn.wmma.scale16" in lower_kernel_to_llvm( + llvm = lower_kernel_to_llvm( build_block_scaled_gemm(spec), arch="gfx1250", llvm_flavor="llvm23" ) + intrinsic = "scale16" if matrix_path == "wmma_scale16" else "scale" + assert f"@llvm.amdgcn.wmma.{intrinsic}." in llvm + storage_type = "half" if output_dtype == "fp16" else "bfloat" + assert f"store {storage_type} " in llvm + + +@pytest.mark.parametrize("matrix_path", ["wmma_scale", "wmma_scale16"]) +@pytest.mark.parametrize("output_dtype", [None, "bf16", "fp16"]) +def test_scale_formats_gemm_cli_selects_output(monkeypatch, matrix_path, output_dtype): + from rocke.examples.gfx1250.gemm import _scaled_gemm_example + from rocke.examples.gfx1250.gemm import scale_formats_gemm as example + + seen = [] + + def run(spec, cases, **kwargs): + seen.append(spec.dtype_c) + return len(cases) + + monkeypatch.setattr(_scaled_gemm_example, "run_cases", run) + argv = ["--matrix-path", matrix_path] + if output_dtype is not None: + argv += ["--output-dtype", output_dtype] + assert example.main(argv) == 0 + assert seen == [output_dtype or "bf16"] + + +def test_scale_formats_gemm_cli_rejects_unsupported_output(monkeypatch, capsys): + from rocke.examples.gfx1250.gemm import scale_formats_gemm as example + + monkeypatch.setattr(example, "verify", lambda *args: pytest.fail("must not launch")) + with pytest.raises(SystemExit) as exc: + example.main(["--output-dtype", "fp32"]) + assert exc.value.code == 2 + assert "invalid choice" in capsys.readouterr().err @pytest.mark.parametrize( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py index 0c066343328c..268a150e7e2c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py @@ -19,16 +19,21 @@ @pytest.mark.parametrize("matrix_path", ["wmma_scale", "wmma_scale16"]) +@pytest.mark.parametrize("output_dtype", ["bf16", "fp16"]) @pytest.mark.parametrize( "a,b,sa,sb,route", [(*case, route) for case in _MIXED for route in ("comgr", "hip")], ) -def test_scale_format_wmma_numeric(gpu_env, matrix_path, a, b, sa, sb, route): +def test_scale_format_wmma_numeric( + gpu_env, matrix_path, output_dtype, a, b, sa, sb, route +): result = subprocess.run( [ sys.executable, "-m", _MODULE, + "--output-dtype", + output_dtype, "--dtype-a", a, "--dtype-b", @@ -57,6 +62,9 @@ def test_scale_format_wmma_numeric(gpu_env, matrix_path, a, b, sa, sb, route): ) output = result.stdout + result.stderr assert result.returncode == 0, output + compiled = [line for line in output.splitlines() if "] compiled " in line] + assert len(compiled) == 1, output + assert f"_out_{output_dtype}_" in compiled[0], output count = 20 if matrix_path == "wmma_scale16" else 12 assert f"PASS: verified {count} cases" in output, output assert output.count("bad=0") == count, output From a1495640a260d0be80e4423d484e401b6dfa212a Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:03:25 -0400 Subject: [PATCH 24/32] fix(rocke): validate scale example arguments before verification --- .../examples/gfx1250/gemm/scale_formats_gemm.py | 9 ++++++++- .../test_gfx1250_scale_formats_example.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/scale_formats_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/scale_formats_gemm.py index 29eb2bfd3cd0..4b43bdbf02fb 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/scale_formats_gemm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/scale_formats_gemm.py @@ -5,7 +5,11 @@ from __future__ import annotations from ....core.dtypes import normalize_dtype -from ....instances.gfx1250.block_scaled_gemm import BlockScaledGemmSpec, _LOWBIT_DTYPES +from ....instances.gfx1250.block_scaled_gemm import ( + BlockScaledGemmSpec, + _LOWBIT_DTYPES, + is_valid_spec, +) from ._scaled_gemm_example import argument_parser, verify FORMATS = tuple(sorted(_LOWBIT_DTYPES)) @@ -47,6 +51,9 @@ def main(argv: list[str] | None = None) -> int: spec = make_spec(args) except ValueError as exc: parser.error(str(exc)) + ok, reason = is_valid_spec(spec, arch="gfx1250") + if not ok: + parser.error(reason) return verify(spec, args) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py index 34875c5df316..a3501500790c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py @@ -118,3 +118,19 @@ def verify(spec, args): assert example.main(argv) == 0 assert len(calls) == 1 assert getattr(calls[0], f"dtype_{operand}") == normalize_dtype(dtype) + + +@pytest.mark.parametrize("axis", ["m", "n"]) +@pytest.mark.parametrize("value", ["0", "-1"]) +def test_scale_formats_gemm_cli_rejects_invalid_spec_before_verify( + monkeypatch, capsys, axis, value +): + from rocke.examples.gfx1250.gemm import scale_formats_gemm as example + + monkeypatch.setattr(example, "verify", lambda *args: pytest.fail("must not verify")) + with pytest.raises(SystemExit) as exc: + example.main([f"--{axis}", value]) + assert exc.value.code == 2 + err = capsys.readouterr().err + assert "M/N/K must be positive" in err + assert "Traceback" not in err From 57edad4391a92a72ec819cc0aff6dfdf2a46e8cb Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:23:29 -0400 Subject: [PATCH 25/32] feat(rocke): decouple matrix and block-scale formats --- .../rocke/platform/cpp/core/arch/data.cpp | 8376 ++++++++++++++++- .../cpp/include/rocke/wmma_scale_internal.h | 5 - .../rocke/core/arch/data/arch_specs.json | 366 +- .../platform/python/rocke/core/arch/target.py | 18 +- .../python/rocke/core/arch/wmma_scale.py | 10 +- .../rocke/examples/gfx1250/gemm/README.md | 4 +- .../examples/gfx1250/gemm/SCALE_FORMATS.md | 18 +- .../gfx1250/gemm/SCALE_FORMAT_EXPERIMENT.md | 9 +- .../python/rocke/instances/SUPPORT_MATRIX.md | 5 +- .../core/gfx1250_scale_formats_llvm23.json | 366 +- .../platform/tests/core/test_arch_mma_ssot.py | 4 +- .../core/test_gfx1250_mixed_scaled_wmma.py | 4 +- .../tests/core/test_gfx1250_scale_formats.py | 33 +- .../tests/core/test_gfx1250_scaled_wmma.py | 21 +- .../tests/core/test_mma_catalog_lookup.cpp | 37 +- .../tests/instances/differential/run_diff.py | 5 +- .../differential/test_gate_verdict.py | 12 + .../instances/parity/gfx1250_lowering_emit.c | 372 +- .../instances/parity/gfx1250_lowering_emit.py | 190 + .../test_gfx1250_scale_formats_example.py | 20 +- .../test_gfx1250_scale_formats_numeric.py | 95 +- 21 files changed, 9858 insertions(+), 112 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp index 77a13cd64dbb..d7afd829fcaf 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp @@ -22,12 +22,12 @@ * borrows, mirroring the Python @lru_cache singletons); the only arena/IR use is * inside the layout-map emitters, which create Values in the supplied builder. */ -#include "rocke/arch_target_internal.h" - #include #include #include +#include "rocke/arch_target_internal.h" + /* ========================================================================= * lane-coord emitters (static rocke_lane_coord_fn) * ========================================================================= @@ -1080,6 +1080,370 @@ static const rocke_ati_mma_frag_row_t rocke_ati_mma_frag[] = { {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e8m0_k16", 8}, {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e8m0_k16", 8}, {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e5m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e4m3_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e4m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e5m3_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e4m3_k16", 8}, }; int rocke_arch_mma_c_frag_len(const char* op_id) @@ -3784,6 +4148,8014 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { 8, &lm_wmma_scale_k16_a, &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e8m0_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e5m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e4m3_k32", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp8e4m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "bf8e5m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e2m3", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e8m0_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e8m0", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp6e3m2", + "fp4e2m1", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp8e4m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e2m3", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e8m0", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e5m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e5m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e5m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, + {"wmma_scaled", + "fp4e2m1", + "fp6e3m2", + "fp32", + 16, + 16, + 128, + "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e4m3_k16", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + "e4m3", + "e4m3", + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, }; /* ========================================================================= diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h index b53babe1ab19..469df96e75b8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h @@ -123,13 +123,8 @@ static inline rocke_scaled_wmma_op_t rocke_scaled_wmma_contract(const rocke_mma_ spec.scale_formats[i] = 2; else ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA scale format"); - if(spec.matrix_formats[i] != 4 && spec.scale_formats[i] != 0) - ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA backend contract"); spec.matrix_words[i] = words[i]; } - if(spec.matrix_formats[0] == 4 && spec.matrix_formats[1] == 4 - && spec.scale_formats[0] != spec.scale_formats[1]) - ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA backend contract"); if((atom->scale_block_k != 16 && atom->scale_block_k != 32) || strcmp(atom->c_dtype, "fp32") != 0 || atom->m != 16 || atom->n != 16 || atom->k != 128) ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA backend contract"); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json index 4e9083a19ac0..b9b6836e110a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json @@ -271,7 +271,371 @@ {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e8m0_k16"}, {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e8m0_k16"}, {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e4m3_k16"} + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e5m3_k32"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e4m3_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e4m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e5m3_k16"}, + {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e4m3_k16"} ] }, "gfx11-generic": { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py index c5aea6fc3849..4eb248048167 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py @@ -1027,19 +1027,11 @@ class _FragInfo: f"wmma_gfx1250_f32_16x16x128_{a}_{b}_scale_e8m0_e8m0_k{block_k}" ] for block_k in (32, 16) - for a, b, sa, sb in ( - [ - (d, "fp4", "e8m0", scale) - for d in ("fp8", "bf8", "fp6", "bf6") - for scale in ("e5m3", "e4m3") - ] - + [ - ("fp4", d, scale, "e8m0") - for d in ("fp8", "bf8", "fp6", "bf6") - for scale in ("e5m3", "e4m3") - ] - + [("fp4", "fp4", scale, scale) for scale in ("e5m3", "e4m3")] - ) + for a in ("fp8", "bf8", "fp6", "bf6", "fp4") + for b in ("fp8", "bf8", "fp6", "bf6", "fp4") + for sa in ("e8m0", "e5m3", "e4m3") + for sb in ("e8m0", "e5m3", "e4m3") + if (sa, sb) != ("e8m0", "e8m0") } ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py index ecc95b1b66c9..2a51db1cb363 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py @@ -107,19 +107,13 @@ def gfx1250_scaled_wmma(op_id: str) -> ScaledWmmaOp | None: MmaScaleDType.E5M3: 1, MmaScaleDType.E4M3: 2, } - # Alternative scales require FP4; FP4 x FP4 requires matching formats. - # Keep these restrictions here, independently of the catalog query model. + # Matrix and scale formats are independent selectors for each operand. + # Validate the backend contract independently of the catalog query model. if ( atom.a_dtype not in formats or atom.b_dtype not in formats or atom.a_scale_dtype not in scale_selectors or atom.b_scale_dtype not in scale_selectors - or (atom.a_dtype != "fp4e2m1" and atom.a_scale_dtype != MmaScaleDType.E8M0) - or (atom.b_dtype != "fp4e2m1" and atom.b_scale_dtype != MmaScaleDType.E8M0) - or ( - atom.a_dtype == atom.b_dtype == "fp4e2m1" - and atom.a_scale_dtype != atom.b_scale_dtype - ) or atom.scale_block_k not in (16, 32) or atom.c_dtype != "fp32" or atom.shape != (16, 16, 128) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md index 9446a79e8c78..402dca910c00 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md @@ -92,8 +92,8 @@ alone does not establish support for other operand contracts. The [scale-format example](SCALE_FORMATS.md) defaults to FP4 x FP4 with E4M3 scales. `--scale-dtype-a` and `--scale-dtype-b` select the input encodings; -non-E8M0 scales require an FP4 operand, and FP4 x FP4 requires matching scale -formats. The shared verifier decodes each scale buffer independently. +each operand independently accepts E8M0, E4M3, or E5M3 scales across +FP8/BF8, FP6/BF6, and FP4 matrix formats. The shared verifier decodes each scale buffer independently. ```sh ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.scale_formats_gemm --case all diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMATS.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMATS.md index c61892c2bc2e..7aa6fa345a03 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMATS.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMATS.md @@ -15,18 +15,16 @@ paths. See [packed FP6](FP6.md) and [packed FP4](FP4_SCALE.md) for matrix storag `scale_dtype_b` override it independently. All native scale buffers contain encoded bytes; choosing a format does not quantize or convert the buffer. -| A data | A scale | B data | B scale | -| --- | --- | --- | --- | -| FP8 / FP6 / FP4 | E8M0 | FP8 / FP6 / FP4 | E8M0 | -| FP8 / FP6 | E8M0 | FP4 | E5M3 or E4M3 | -| FP4 | E5M3 or E4M3 | FP8 / FP6 | E8M0 | -| FP4 | E5M3 | FP4 | E5M3 | -| FP4 | E4M3 | FP4 | E4M3 | +| Matrix operand (A or B) | Scale dtype (independent for A and B) | +| --- | --- | +| FP8 / BF8 / FP6 / BF6 / FP4 | E8M0, E5M3, or E4M3 | Here FP8 and FP6 each include both exponent/mantissa formats. E8M0 has selector 0, E5M3 selector 1, and E4M3 selector 2. The reference decoder interprets E5M3 as an unsigned eight-bit scale format with bias 15, distinct from E5M2. -The implementation requires matching scale formats for FP4 x FP4. +A and B may use different scale formats, including FP4 x FP4. +Standard MXFP formats use E8M0 scales over K32 blocks; E4M3/E5M3 +and K16 variants are broader block-scaled contracts. The `i8` scale spelling remains an alias for E8M0 for existing callers. ```python @@ -38,7 +36,7 @@ spec = BlockScaledGemmSpec( name="mixed_fp6_fp4", M=32, N=48, K=256, dtype_a="fp6e2m3", dtype_b="fp4", matrix_path="wmma_scale16", block_k=16, - scale_dtype="e8m0", scale_dtype_b="e4m3", + scale_dtype="e8m0", scale_dtype_a="e5m3", scale_dtype_b="e4m3", ) kernel = build_block_scaled_gemm(spec) ``` @@ -60,6 +58,8 @@ and block sizes, but does not enumerate E5M3 scale decoding or the complete scale legality table. LLVM's `WMMA::MatrixScaleFmt` defines the selector names and values used here. The table above describes this implementation's accepted combinations; compiler acceptance alone does not establish numerical behavior. +The expanded FP6/FP8 alternate-scale and mixed-scale combinations have host +admission/lowering coverage; their new on-device numerical campaign is pending. Existing mixed-format fixtures cover bounded finite values and compare against an independent host calculation. They do not establish arbitrary-input rounding, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMAT_EXPERIMENT.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMAT_EXPERIMENT.md index 7c911873e516..75a0021f8d90 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMAT_EXPERIMENT.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMAT_EXPERIMENT.md @@ -1,8 +1,7 @@ # gfx1250 scale-format discrimination experiment -Status: planned; this experiment has not been implemented or run. Complete the -FP6 / scale-format PR split before executing it. Existing mixed-format fixtures -are separate evidence and do not count as execution of this plan. +Status: the expanded matrix/scale campaign is planned and has not been run +on the current source. Existing mixed-format fixtures are separate evidence. ## Contents @@ -82,6 +81,10 @@ as an expected pass value in the main finite-value test. - Run FP4 x FP4 with matching selectors on A and B. Keep the untested operand's scale at that selector's unity encoding. Since unity bytes differ between selectors, report this as a separate control from the fixed-buffer experiment. +- Repeat the selector sweep with FP8/BF8, FP6/BF6, and FP4 as the tested + matrix operand. Cover both A/B positions, homogeneous and mixed matrix pairs, + and independent scale selections, including FP4 x FP4 with differing scales. + Keep the raw matrices and tested scale bytes fixed within each selector triplet. - Validate with COMGR and Python-generated HIP, using the same literal inputs. Require Python/C++ LLVM identity where the C++ backend is available. Do not infer C++ HIP compilation from Python HIP success. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md index 603741f1c817..4e0302e95323 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md @@ -46,8 +46,9 @@ These emit generic AMDGPU IR; arch only sets the comgr target triple. `block_scaled_gemm` supports homogeneous and mixed FP8 E4M3, BF8 E5M2, FP6 E2M3/E3M2, and FP4 E2M1 through SCALE and SCALE16 with LLVM 23. -E8M0 scales are accepted for every matrix pair. E4M3/E5M3 scales require the -corresponding operand to be FP4; FP4 x FP4 requires matching scale formats. +E8M0, E4M3, and E5M3 scales are accepted independently for each operand +across all matrix pairs. The expanded alternate-scale combinations have host +admission/lowering coverage; fresh gfx1250 numerical validation is pending. See the [scale-format contract](../examples/gfx1250/gemm/SCALE_FORMATS.md) and [packed FP6 input contract](../examples/gfx1250/gemm/FP6.md). diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scale_formats_llvm23.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scale_formats_llvm23.json index c3c784f04926..6f9799667ff0 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scale_formats_llvm23.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scale_formats_llvm23.json @@ -34,5 +34,369 @@ "fp4/bf6/e5m3/e8m0/16": "03e7e256e3b5e60f13283dfd594442e586615272ad9ceba478dacfb6e4de4cc9", "fp4/bf6/e4m3/e8m0/16": "5964c480fdba4385fd92c18d15b9e2a85e0dfdf9f2f63c2f4758bef9ca3a32e3", "fp4/fp4/e5m3/e5m3/16": "01036c856966384b28eb2bd63673fac0b64fb06deb46aa3ea33973b1933da937", - "fp4/fp4/e4m3/e4m3/16": "e68ccf62c4dbccbe3a20d32399aa3d54e48e40ccd5e72d4fa52a54aa888999f7" + "fp4/fp4/e4m3/e4m3/16": "e68ccf62c4dbccbe3a20d32399aa3d54e48e40ccd5e72d4fa52a54aa888999f7", + "fp8/fp8/e8m0/e5m3/32": "5ddcd9cdc0d562f34a0c23bdd03e9910590e2521ef94d4e139d3b24fffad3f2d", + "fp8/fp8/e8m0/e5m3/16": "729bbaa07e6c122f5ef8bfda8c699bf76a49d90e7dc08714d2a4ec73892772e5", + "fp8/fp8/e8m0/e4m3/32": "93ec0e6a7d763bf45e80f5c2f1069e5250750c5acfe1160eb76c0f6b7968c108", + "fp8/fp8/e8m0/e4m3/16": "27349a23bae88d7b57e160a587bbf7ff08d78447dc8f0d8830acda4f411b9e0d", + "fp8/fp8/e5m3/e8m0/32": "a3f3a6a947030ce7634553ee3a670270c846c80ed12fc46ce496c9c184200d83", + "fp8/fp8/e5m3/e8m0/16": "2e9248cc4cc55e0a5ca681f87526aa4852b975af93b455405a20c028b984067f", + "fp8/fp8/e5m3/e5m3/32": "c5fa0d358031ab853df0717a75ad210fccb31a0f60a5bd1e392ef02853df3ef0", + "fp8/fp8/e5m3/e5m3/16": "fa43cb8dc0ff683b01c85ea5e0578f64a12e0fe912e10befe8931ea2833a424f", + "fp8/fp8/e5m3/e4m3/32": "5fe9925c36b1aa53f9b345e36d140161b7af4e2686d33b2ded6ff06146e270db", + "fp8/fp8/e5m3/e4m3/16": "dc21d3b9a559df664b5a22b6df8c114854c1e7d745c07ddb2eda77bc6eebd5fe", + "fp8/fp8/e4m3/e8m0/32": "0fdaf2ee8e5d1f8df0b549d526ab5c906e4612f52a22dae3a637eceab3388e48", + "fp8/fp8/e4m3/e8m0/16": "e738cfb7fbddd60be96e2c5191df3e5c8699bfdfd25728cb93f1aade964fa8d7", + "fp8/fp8/e4m3/e5m3/32": "60a668bcc02a3507e8d8cd71206fa4b51a637b14f2169ac78ebe141db1aba768", + "fp8/fp8/e4m3/e5m3/16": "92d6fabbefabf7b0c9c3e1df84b527a93630b73b17b214db2f316b895e817b5a", + "fp8/fp8/e4m3/e4m3/32": "92222e6c3d0a38def2058f81661d588f36cb1b4218052c99898097ad3bb9092f", + "fp8/fp8/e4m3/e4m3/16": "66550fd3d90acf9221fc3fea5adf3206fee1f01830a43c58f15dfdb40b53448e", + "fp8/bf8/e8m0/e5m3/32": "baecc775fa51273e6e3993f31f8dd1a9e873f2cba26c588cc891bf2752f398f1", + "fp8/bf8/e8m0/e5m3/16": "537a720869a0afce1e56a22a5bdaa2150508587d74a378711eac456a7bfbb5ac", + "fp8/bf8/e8m0/e4m3/32": "ef9a3f839f9144f59b9d887a44aa3128995d7d53c86833c26e4017cdc39ff1dd", + "fp8/bf8/e8m0/e4m3/16": "efb0c6f3ac1a966a82b8abd3b3965da9bf284f8382458a52815320a53bd73641", + "fp8/bf8/e5m3/e8m0/32": "a7ab5c9220709abfb3b93218e114f8ac467ab8d3f256ebd0a89d40bf5618eb46", + "fp8/bf8/e5m3/e8m0/16": "cc833efde8cfcc429aa623ab7b3a57712da1c56b078fc27552a2a9fee51f7a76", + "fp8/bf8/e5m3/e5m3/32": "e20f6763bf2756ca0905cec15e2ef71e6aedc95c85eb4a0e9a944f1094dc08a6", + "fp8/bf8/e5m3/e5m3/16": "5d383f520f735aebc98cbd358ebbad92070b6af62d3fd97eb7ce52ce85a88086", + "fp8/bf8/e5m3/e4m3/32": "aa1ab94069e76b7b426c82f9eed7ba8a015ca38d85f8c66aee665bcffb960364", + "fp8/bf8/e5m3/e4m3/16": "8313c5c34de8c4ca292e39130a7158b2ccf79ec4df74c91d2f1fdda2b775bf0d", + "fp8/bf8/e4m3/e8m0/32": "93b2d7c755d307c85912604b8935d3242d5fb41f2a846668bc4985c6b1e66d4a", + "fp8/bf8/e4m3/e8m0/16": "52cb586d2f50b522a48fd138ff72eecfac004ebcf2958e5e17d286e039a722d6", + "fp8/bf8/e4m3/e5m3/32": "dd0e4ee57541076c13f09668fbc9a17d66324351c9e364a64b2c14b1378e2a70", + "fp8/bf8/e4m3/e5m3/16": "7c1525e4c8f6448f9430f913625c2a711e1f9436089738653653f304152bea60", + "fp8/bf8/e4m3/e4m3/32": "6cea2a69a2f3b250024ecdf9d19868fc8a7a3cb15ebb2f678ec69085462b8b9b", + "fp8/bf8/e4m3/e4m3/16": "8687c3de8d5e8a64254a7e9787ac8bcdfbfb2705c26661a0607295c7317bccfb", + "fp8/fp6/e8m0/e5m3/32": "0a6244431d32e516373d8e6568065cedbf43519b04949375403f552ae10dd7af", + "fp8/fp6/e8m0/e5m3/16": "ab5e5613e9a70f7988bdae63fb71dd0daca1b57b2cca4bfe56a1808df5b7af34", + "fp8/fp6/e8m0/e4m3/32": "a44cc7caea604794bacc6e514e82a333da07c1bcdc9508c01998879dab6c5480", + "fp8/fp6/e8m0/e4m3/16": "c6516f22bee494cdb4282aa1df22b9c027fea32744f81d9830ec51cfc52a30a8", + "fp8/fp6/e5m3/e8m0/32": "f7c4594e03eece7299b9ae42e37a3e75b5ad8ff8a6366748eab32f0226eca273", + "fp8/fp6/e5m3/e8m0/16": "0ed60656771791810c66a2788e280c5cb6fa133cd2f53d789ece087270151aca", + "fp8/fp6/e5m3/e5m3/32": "eb29a85078c3402fdd38452b96a4f3d6d71714d12569bd488cf07a369002752b", + "fp8/fp6/e5m3/e5m3/16": "ba6de49079741ca210396b1b8273ef40ee7449d3446c5bdd275df421b8951198", + "fp8/fp6/e5m3/e4m3/32": "e0b19bea3a5c80d7c18c0c36aab334671c89d9d30ada0878cfbeb3a3efdefcec", + "fp8/fp6/e5m3/e4m3/16": "f781c95a9087f8f2ecadc4ce3a43ac07add4047e7a4506e8443e5faff0809b92", + "fp8/fp6/e4m3/e8m0/32": "997a4e38b3ce3c9a39c60ec8f347ae5935256b05957ceee9c904c2333af7d6a9", + "fp8/fp6/e4m3/e8m0/16": "af6d4bd5f08cc97a58fe2c89c7899f6520f3227f49f6c86864f8d64c9370eced", + "fp8/fp6/e4m3/e5m3/32": "204e973eb30d6949003aa4f10e0c4f5889a4b55ec7b98a3efeb427c12223b87e", + "fp8/fp6/e4m3/e5m3/16": "de265db176ed45a95a0e4ea17731f3334072947c06c29f5dbe12c84d1acd3a43", + "fp8/fp6/e4m3/e4m3/32": "12ad1db41fdf4674386ca2a3d989d5d187ed41d87f95474c68d3a0b691e3480b", + "fp8/fp6/e4m3/e4m3/16": "60e6c760e789cf7895e951fb592f662e0bbb931874cdb06954067d52e348e8f9", + "fp8/bf6/e8m0/e5m3/32": "673c1e64cda2c61562dc7625ead9701d560daa32b29b97815aa1c4616043b8d5", + "fp8/bf6/e8m0/e5m3/16": "42ef038f21f8527682d18a9559980cf96311aef5da3ffdb67bee7051b3ed3717", + "fp8/bf6/e8m0/e4m3/32": "5ec743742361b41886c0b476caaab679144532312857794364c3b90b7e934421", + "fp8/bf6/e8m0/e4m3/16": "94bd2a008ce634a17a79a9f5f6ec5aecf130dfd28fa517d5eb6c82f278bfa972", + "fp8/bf6/e5m3/e8m0/32": "03f8a57534b7a8c0616c4308836fb4cda15af8cda808a84d47ea26d120719171", + "fp8/bf6/e5m3/e8m0/16": "062a9fde78a343d4cc22cabcb8ace71559c1d38720ab6dd5293615f9f1df8bfc", + "fp8/bf6/e5m3/e5m3/32": "8765bfc38712f6f64f4e5d0bc23c830f24f577f3e24f180b18bc092b04f997a6", + "fp8/bf6/e5m3/e5m3/16": "2089430ec1bfcedd52446fb9727d646518d2ad97064c4a97648b75def078c9a0", + "fp8/bf6/e5m3/e4m3/32": "27c54f3df7c3649921d3909cdebb74c62fbce099420635ef773bc62dee50d8dd", + "fp8/bf6/e5m3/e4m3/16": "4f9184db0f2f9935eea39d0f15cb0149164859c31cf21040f78b646434d28314", + "fp8/bf6/e4m3/e8m0/32": "159e32591bde664d37482d491443be573085ba1b6eb9b3d0d8d98513b65ad589", + "fp8/bf6/e4m3/e8m0/16": "d5dfa14c4b096813faa6789696721e866a15d428edd4dd1b47b01b2ae33f4d38", + "fp8/bf6/e4m3/e5m3/32": "272043b8c153389ad91162a0eb54a83a13b8753d359cce74daf57fb31e43479b", + "fp8/bf6/e4m3/e5m3/16": "0baf2836d5276765bc686c19dac3f59994c767c4a9b1b9337e57d9f0747f4f68", + "fp8/bf6/e4m3/e4m3/32": "42ecfab98d0244eebd0da4867921fc7df6a1762a6508ed62eb40b971d5e7e03f", + "fp8/bf6/e4m3/e4m3/16": "994d3154164df61e1550a4b4a05e767f0fefbf3b0fe9ea2d86850dec19427ea6", + "fp8/fp4/e5m3/e8m0/32": "f92b362b4a3a7cf0be55963828e130b7a859cac23607879117e30ad15eacd308", + "fp8/fp4/e5m3/e8m0/16": "1374f9e0a34040fe321893dd8a6126b9bfa23c31b72e0e4f2b7d8c2105dfea0e", + "fp8/fp4/e5m3/e5m3/32": "c1e330bfd49f73eeba60afe606834ae162573d230b5bc369888c1ad32ad659c2", + "fp8/fp4/e5m3/e5m3/16": "0106be64f01459d4a7a5bd9465e5abbf7295be6f47402cc73f4e758c15ad12cf", + "fp8/fp4/e5m3/e4m3/32": "15fde1f2748a3707479953117622e9134deaf5a91561343dd71677893a21cca0", + "fp8/fp4/e5m3/e4m3/16": "7d59eb54e807e1003d44f3d5361c66335a7579531443d7397a523a3c866dfedf", + "fp8/fp4/e4m3/e8m0/32": "4344f204f496fc87e6e80b9af2ad3780eeb2eef941faf7394406eda8e2a5a400", + "fp8/fp4/e4m3/e8m0/16": "591459a4f964cde8bf37e2c24ae094a99aa44ea290cb253e27931d1c191b5551", + "fp8/fp4/e4m3/e5m3/32": "410c90816872eeb5c1f9afa253e9973b7287077ddbe7a11594071636365fe2e8", + "fp8/fp4/e4m3/e5m3/16": "1e4668a283b1b6028c981023c44a34d678ab77209434c65d7a0b183ad0262fea", + "fp8/fp4/e4m3/e4m3/32": "2178e9f0caa117ba57c5b26c1ea2a55e6773fd417bd51e903c400bfae9b90168", + "fp8/fp4/e4m3/e4m3/16": "04b29350c2cb555a91ff21b1cc4428a0ec2c562076fb7855ed29ac9a09df0034", + "bf8/fp8/e8m0/e5m3/32": "e11cb4bc37eadd457348522ce2f167f10d619fae33e89f7e3c75e6b1cef2adf8", + "bf8/fp8/e8m0/e5m3/16": "ec52ac7258864c641f3e60670d9719edb290194c20083d933ca071769ff498b6", + "bf8/fp8/e8m0/e4m3/32": "48ce41b3de4bcba3ccd07d45e68f6f52c308eadfc5903fc7faef818fcc00fa3b", + "bf8/fp8/e8m0/e4m3/16": "044e25ec11bcb582389efccb111765fbdb323131500cdb5eecabb7e1f2599460", + "bf8/fp8/e5m3/e8m0/32": "a85330207dccd9fe29f796bdd333c08d23c87ed41bd92e7831b59d1a65fb1d56", + "bf8/fp8/e5m3/e8m0/16": "fe19a27cc91bbbbe5d5f387c0c5309c93233c4306a3329188d27bac918505bff", + "bf8/fp8/e5m3/e5m3/32": "8251a946c1f24d38a24c9d9e4e472e28d672cd762fc4ba1d7df42542508e880f", + "bf8/fp8/e5m3/e5m3/16": "d4a90e80fa26a5690f2c14d49ba1e46c34234fe2f42dfb92b1da34e36f589b3c", + "bf8/fp8/e5m3/e4m3/32": "388f620742bb0bd22cf481f21fb75f6f6e5c9cbb32a6408af5bbf8fba5484a6c", + "bf8/fp8/e5m3/e4m3/16": "338ae4efd137b57de6dffb5dad83d05b76250afbe4a848b9b9db030348e15ea4", + "bf8/fp8/e4m3/e8m0/32": "fe435af741d3dc6ae50e63586fd9ba60837068dff4fd1b54939ca7f636899c52", + "bf8/fp8/e4m3/e8m0/16": "c150a339a8e87cd73919466c57ed9e137d7f522206b8a3dafde36d784837ad3d", + "bf8/fp8/e4m3/e5m3/32": "2d8688a84d87c30b7ac5df45b751a335aef80766dd4084ad16fb11c1f9bf6411", + "bf8/fp8/e4m3/e5m3/16": "e12f8b6706dcf87a4bee965b9a62013b105a419a9d036290e207173d84cfaa15", + "bf8/fp8/e4m3/e4m3/32": "7cde85a8463048e39d7549280695d891728400bf090a7e58e385ad5bbacc3745", + "bf8/fp8/e4m3/e4m3/16": "fce9aeb6bedf5cb1e86f85504721d003861a6b36041eb97738b607e20728bb79", + "bf8/bf8/e8m0/e5m3/32": "4ce92942ba7b0fa107d8c6d5e9d9d824f1749af050d0a39e89f04f46a395da02", + "bf8/bf8/e8m0/e5m3/16": "5c0afd62befb519374045af72bd4b77e5e75c4a085c00a9025bee435f54c40ec", + "bf8/bf8/e8m0/e4m3/32": "bc9a4ecd5089272d7b26e972b448077e1d2690ae14cc673ca74e243b22b812d6", + "bf8/bf8/e8m0/e4m3/16": "31bde40571ae1b1e47dfe8ad098c1e7dddb9f6d66dc3f7d67ff4dd8956d4a04d", + "bf8/bf8/e5m3/e8m0/32": "0f2e97511fcdb0ae17c89ebb4141f873d4b6350d4f6de9f763d6b2eda855295a", + "bf8/bf8/e5m3/e8m0/16": "beaedb14b05e9fb393e2e2a21718617665731f49ce65ce1e0f7f4446a0e54234", + "bf8/bf8/e5m3/e5m3/32": "226bc1339e8ade1c2e25e47b89a650cb4f68d30c35fdc77ddc289faa9a227617", + "bf8/bf8/e5m3/e5m3/16": "3ffd58442dc026043d0ea36e90d65a1f0e6b4a1fff9c6b34d87076319ba2d963", + "bf8/bf8/e5m3/e4m3/32": "095324f00210afedd6819e69a926ce6f089580efe7bd08cdead05c50d9c59ec7", + "bf8/bf8/e5m3/e4m3/16": "9b63e03b1126411aae2fae7f939a2b8d8065cfbf539f8dad0155b6eef2f1a521", + "bf8/bf8/e4m3/e8m0/32": "9a12f815df269fc8ec985a58af4375a53a90cdcca759025261f6832be6d7dfd4", + "bf8/bf8/e4m3/e8m0/16": "763d4504ea575af80b17d28d83b5132417dbc3f7addda1536f9dd916e7855d27", + "bf8/bf8/e4m3/e5m3/32": "3f758c978459d126a18b1ca0542ac7f260ac2d9951ac83d35e5905828236b6d5", + "bf8/bf8/e4m3/e5m3/16": "c95e3876329747ef8e2980e9354e63b20eae9321c8daff99a518db2a89885f34", + "bf8/bf8/e4m3/e4m3/32": "4fe913efc699cbd96adc318c94673d3a069e0e739a021a797e1b1ca68d634a01", + "bf8/bf8/e4m3/e4m3/16": "1fd3796a074ddeaf514b3b139d763ae18a32c876db81a4671f59dd63a43147ab", + "bf8/fp6/e8m0/e5m3/32": "6f9e0abdc2382c745c87d827b88b4455758e712ab9adf0c438c8214eca385630", + "bf8/fp6/e8m0/e5m3/16": "79709e3e5be85a8e1cb211bd019883c902ffa13a5ec67a55a63d42dcd1dc9d7d", + "bf8/fp6/e8m0/e4m3/32": "03e8ace4cc325ecac8aeca4454e65b0695bb4d8511013f4a37d820d6f68a94f8", + "bf8/fp6/e8m0/e4m3/16": "c572e96d040275154b17df85b1dfa5d601ab20de7f24bacc6b57f24814237498", + "bf8/fp6/e5m3/e8m0/32": "442263b39cf5e1943c2a06d0f2420234c0f07d0150a4cf396fa0aa697c634506", + "bf8/fp6/e5m3/e8m0/16": "a05d1b3200bcc4db907df428424e8ea4af63f10ea8ab1251d0276d5c9f05c42d", + "bf8/fp6/e5m3/e5m3/32": "7f7ca686636084704be39976f943387fe686df8349d31710ea304a5478464b91", + "bf8/fp6/e5m3/e5m3/16": "b0dd47c0506e2b12c35dd66c27a9bcc05e5520bade06aa5983833edd3a3bb4b1", + "bf8/fp6/e5m3/e4m3/32": "cc1ee5dbf9575fffa18dabbf2803cc7fa5fb60c9e3e57b792d34c8efe2c2fff8", + "bf8/fp6/e5m3/e4m3/16": "eb596c463ddddd3ef96b26b84a0123817609199e8b538f6da8fd81f34ebeb684", + "bf8/fp6/e4m3/e8m0/32": "509325bd4916807d5a2363d1064e3e0f145f4cecdac89fac13d339f79d621fa9", + "bf8/fp6/e4m3/e8m0/16": "89ddbab05b007646522263d6d7beb8d5fc5d819303c78b9e75bc5869289ede15", + "bf8/fp6/e4m3/e5m3/32": "969e620fe328e1672fb4b09e8da9c92f138c33bc02c0ca29a031faee82b59bcf", + "bf8/fp6/e4m3/e5m3/16": "7076bd5b1b3030b0deb91418d593b7148294d65c19551f080b7a0e458bf9d035", + "bf8/fp6/e4m3/e4m3/32": "2c5e26b4b974e45bbcdc7a8fa14526f512696f9bb8d0ecf772286d017251375f", + "bf8/fp6/e4m3/e4m3/16": "a5f7be080cad581212f8a4747345a36f0d4058cb43b38843e748ac79b11a7901", + "bf8/bf6/e8m0/e5m3/32": "4d8c42c921d1791316e1fc6c799cb1aaf8945d9022b4cfb5b5e1b1bb07a8463e", + "bf8/bf6/e8m0/e5m3/16": "dc187d5dc0c837b74814f292d54bf82804424c7551d9ac390ad4f369c628d6e6", + "bf8/bf6/e8m0/e4m3/32": "ab0056577a857b77b84b14856ec588edd34c98b27e158d72db16cb6bd90381c9", + "bf8/bf6/e8m0/e4m3/16": "5fec446d4e8055abac6b49e34b42e2384a8a79866dd002ba3bc97dc0bb6883d3", + "bf8/bf6/e5m3/e8m0/32": "6924570e1b69546832b3bb5c20d4a23153ec9d4a99b344ce496bae5bd67c3281", + "bf8/bf6/e5m3/e8m0/16": "d0350a4ced88a12e6a534ac990957088a39eb12a9b2c72646b5c3bd192ee4d82", + "bf8/bf6/e5m3/e5m3/32": "b24ee77f10d8f404ce61362cea341a7e054fd8fec2595e685d4f388969f1dc6c", + "bf8/bf6/e5m3/e5m3/16": "18229bdde379841a09724ca123f606c6a2357148e64c1a88535e2cd5e52dfcb9", + "bf8/bf6/e5m3/e4m3/32": "acebe7aa67f8b442fb4d73d51e16a7df1a0716b078b92dc53fc72cc33e215697", + "bf8/bf6/e5m3/e4m3/16": "ba4063caec4e0769897a131f41d71979f9b7a27513ff590698f37ac358427af9", + "bf8/bf6/e4m3/e8m0/32": "c92a9100eca08ad01603a6dfae321258ccde817141a3fa88772b6e84ffed9ac4", + "bf8/bf6/e4m3/e8m0/16": "6059dc8d4685b6f8c18c92d0edef0330cefc8876b2af05a5c322a678739a903d", + "bf8/bf6/e4m3/e5m3/32": "5c190742123c4524fb5c7902add5605131f8b66850660fd3231cbe60fc5b1041", + "bf8/bf6/e4m3/e5m3/16": "64143c43e1784faef1203cfc95938db9d2a6c8362aad8e241bcb9c757164a2c8", + "bf8/bf6/e4m3/e4m3/32": "3c1241a21463263a76178a837ed40a470ea57388e89c5fe8b7a6ccf0fa864a1e", + "bf8/bf6/e4m3/e4m3/16": "5377f1f0d7a8c3e3179f0043b84ac2f38d0d6dd8ef7ed38e62d4c8eab3b50c1c", + "bf8/fp4/e5m3/e8m0/32": "fbf4e8354f55df92972b0e4783b4bb299131317e800a5ac391aac2c5453a3696", + "bf8/fp4/e5m3/e8m0/16": "ac162516eb094a87d8437e5b7505711ddfd0385da7995f64cde16a9caa745209", + "bf8/fp4/e5m3/e5m3/32": "1843f2c15a4e29c45ff045ce38b6bf52aa1d183b5dafc09b6f632acc917b6da8", + "bf8/fp4/e5m3/e5m3/16": "926434b4498d80f2fbfce7711868d9dbde84eb88c501a4a2ae63dfac54156886", + "bf8/fp4/e5m3/e4m3/32": "97bf8f32858be904fe91234e583b28daa31a765542cace3f9277d252a70dffd8", + "bf8/fp4/e5m3/e4m3/16": "83faea00befaf286a8cda22e516e32e78e00dd1c63b4caf488c5bf271ab57f15", + "bf8/fp4/e4m3/e8m0/32": "58680391730f047e4022c05059738c716af65ee45c6df57864fbbda1d2df7958", + "bf8/fp4/e4m3/e8m0/16": "f5ec2e991e29dae40eee1b651849e79becc38dbc09698fbbb848ce0c1659cdb4", + "bf8/fp4/e4m3/e5m3/32": "e80bdf5cc96ce5e01650daf42fdd13ba8b6211d4256d27b1c60f23b0ff1e59e3", + "bf8/fp4/e4m3/e5m3/16": "62029fb2acf8849a0af6fad0e169cb9eb3d07be320adfde7ae3590b7815bec61", + "bf8/fp4/e4m3/e4m3/32": "ab2e08ab3902a34a6de565662ba042ebfaaa405c00572e9cbc51a940d8c2d4b3", + "bf8/fp4/e4m3/e4m3/16": "ffce5de9d33a21e610bac1e8bf37eaf296361659adc7dca47f39cbb1fca91ed1", + "fp6/fp8/e8m0/e5m3/32": "03816060815a4c0a19fd53fd0b6dba826b87dbc71f8631fdba41b0bdc880e5ae", + "fp6/fp8/e8m0/e5m3/16": "d1b6dfd6ef7e319c512fee90903aff7f7d74ebf2d0b75837d753c81835f52758", + "fp6/fp8/e8m0/e4m3/32": "f72e15c42946feac252785f4e7f81ce400cac894d8395f5588f3de81fe6b06c2", + "fp6/fp8/e8m0/e4m3/16": "a85fc9cee6c2e4773547f3ae741748e4fc9f4098d3a974a18664788bbcf7a8cd", + "fp6/fp8/e5m3/e8m0/32": "cc2ca6b1f5d8007a0df575c65ab0b7339f54273bbd59006d6ee0512ee8c90fe0", + "fp6/fp8/e5m3/e8m0/16": "bc9d7bbf2037e9f7ba06a41a7abc43b2203bad41a5062918c02a81a3b59106c8", + "fp6/fp8/e5m3/e5m3/32": "a8b8316a337f3a6d7680dff26ed010d16e1083aab2511952b68454f018b9df44", + "fp6/fp8/e5m3/e5m3/16": "0a3ebc42eb8b6c721b91aeb7ba599e1346f637b9ea5b5193171ee5359053d1ce", + "fp6/fp8/e5m3/e4m3/32": "8f434338e315e85826f759090bfdeec267e554f978e4917c0e046aab5f648b90", + "fp6/fp8/e5m3/e4m3/16": "4608cf7188164e9351d41cf91e0de261cc66aee33a4af4b511936ae985b297d2", + "fp6/fp8/e4m3/e8m0/32": "5693da683ebf0b76054d646c1148d21a682bff22c8a1d8e3afdb71cf73034059", + "fp6/fp8/e4m3/e8m0/16": "3701a1066a696ac4188a5381613510c6119659f36e443c37b27455521dd00278", + "fp6/fp8/e4m3/e5m3/32": "05a9ae5da9c61af8d3321f4e18aa43c1be7eed995ebb83c5c36f101d76d6b226", + "fp6/fp8/e4m3/e5m3/16": "f9ded0874dd3ae03bb05afde2e83c8f424de6864bfe23e0ebf778c0137231f48", + "fp6/fp8/e4m3/e4m3/32": "dd219a11f7c479e1af40caf6d5cba7eea55154e28523adcf80b02d19b2cb8c92", + "fp6/fp8/e4m3/e4m3/16": "cf77ed532b9a3e5317c7e1f1c2bee3f34d4ecc8fb8914397cd83ad87f6d6e228", + "fp6/bf8/e8m0/e5m3/32": "3722e0b7d696c0a30fb0ab329b17cf9dd930741d336a8d05ddca02d3bd34d3ff", + "fp6/bf8/e8m0/e5m3/16": "ed0112c6be381a62d406558af7d37f38306e08cfba1c2413cadb7bf3cea81ed8", + "fp6/bf8/e8m0/e4m3/32": "ee9786b6cf4538ece59ec5eb68fb149673bbf434dcc4f96bfbec7045850176a3", + "fp6/bf8/e8m0/e4m3/16": "eac8398d9e85320ba2c43b6f7074459a357361a3b09951d4595b8c6e37438947", + "fp6/bf8/e5m3/e8m0/32": "42d19f2a7d228e2bcb564e4c8768c5b53c3a7bf9ed25f90551258e1a8a74486e", + "fp6/bf8/e5m3/e8m0/16": "bf26178604eb4ac39d7ba906c9dcb2c8e1eb35649e05ce24460137010c2f8831", + "fp6/bf8/e5m3/e5m3/32": "f728cb07588d21391468afefa65f251e15c23de2e4646841cb9cd9d1e8cf12db", + "fp6/bf8/e5m3/e5m3/16": "3974c59cc8b3b7362f3d18f6c908d4132a587463eaf6df926aff44f430f662c8", + "fp6/bf8/e5m3/e4m3/32": "50a2e3db8bcd66dfa6d2d6c001a78536439b9ee763d7e97887dd954d489940a7", + "fp6/bf8/e5m3/e4m3/16": "c5a7cdb63207deab4f41b6cd0f081b239ac5f422b53ae75d2f1829d4fa0490f0", + "fp6/bf8/e4m3/e8m0/32": "819a883eccdf83d2fe4e182c2ca776f4c3769770b5b3d38779c35ba36ab79584", + "fp6/bf8/e4m3/e8m0/16": "6fbf7fc13e7b922e9a247c57b6b5c90011db06649137673e0dc6f22393d25b9a", + "fp6/bf8/e4m3/e5m3/32": "7e65d74ae1240e9c8067f00dcd53d0907b39f3f8fb1b5619ec19c3286440aa2c", + "fp6/bf8/e4m3/e5m3/16": "0e76804563085d38139e105c89e0b4a5df4c6b30ba2adf9dabd3e61511ea3d2f", + "fp6/bf8/e4m3/e4m3/32": "7935094b6d42f684510f9d24cbbbe50bae94a5e6a39412e688e792acc8f22129", + "fp6/bf8/e4m3/e4m3/16": "e1e69aa540f8a5169dc8fc773a6dc4917bcbf91c9fc23c2c36f32aeeb3493c99", + "fp6/fp6/e8m0/e5m3/32": "7f99d0c4df85319067e83094052424c5dcc20d7feeb7d0beca8de2ba653089c8", + "fp6/fp6/e8m0/e5m3/16": "a30202e2e3801f91c41aa7175ff8a63cea15e537ba2c918a0438abdaa4317d8b", + "fp6/fp6/e8m0/e4m3/32": "e4af5cda8447e5cc9e72b8a1cf2409718b5e1b03d6199200b0798c02c6c56137", + "fp6/fp6/e8m0/e4m3/16": "31136793649b7dfa54964f7df0dffc42c8e0dbb4c63a08685be786b40e797249", + "fp6/fp6/e5m3/e8m0/32": "7ffbf5ee4c7ad63c76cd44d4d0ee3a9416f689a068a46165f728059eb07c8dde", + "fp6/fp6/e5m3/e8m0/16": "3487c39e6547670e0dc1f0bab7f87564f54e278b37d7f2b030d9a63355cfdcc1", + "fp6/fp6/e5m3/e5m3/32": "f8f6c109db242f9521091144498199d99d792392dc8451fb4a7b1f542a959d35", + "fp6/fp6/e5m3/e5m3/16": "9169bba294e3c0df7845e0436d7ccd346a1f457e61beb67c24383324ee64a98c", + "fp6/fp6/e5m3/e4m3/32": "5898eae101249d8f3e1e2c749acaf04eaea7f5a81949224b64c843caeee16b47", + "fp6/fp6/e5m3/e4m3/16": "f5ae46a0076af03f4e8e31720cfed9d0396befcaa7c1c5e366610aace4317dec", + "fp6/fp6/e4m3/e8m0/32": "d9c4e47cfe4ce8d255f15b565f7ecbc3b42e18aee4dcf0714696bed4e824e859", + "fp6/fp6/e4m3/e8m0/16": "6144dbd1ad8f4640f579b31129754449001ea47eeec554ec67685dfba75c424a", + "fp6/fp6/e4m3/e5m3/32": "ee395430e11f584fc1dde4993afabd3640ae32ea3ee09f26dffe0e03a616692c", + "fp6/fp6/e4m3/e5m3/16": "90ca3f03545cec8d51312be343d13b22bf5067c421228290d4db769af472f767", + "fp6/fp6/e4m3/e4m3/32": "0753297fe26006452f308ae9245125cfba0de8b3465605327402097204fc7d93", + "fp6/fp6/e4m3/e4m3/16": "2a62670cd685844712153180cf379482aec7a964d9e6892d5354e01fbdc8f2b2", + "fp6/bf6/e8m0/e5m3/32": "105348c2e78b82a4527ef1ac90c71499258b317b74ace2352d390739ea005a0e", + "fp6/bf6/e8m0/e5m3/16": "31713bbd9b445cfe7f6bd346701ddac7f9dc925db55a6a40cf50c2cd7c5b8cda", + "fp6/bf6/e8m0/e4m3/32": "c8b73a6878378701abe14c87f130dff9ce05c950a54da4a9094322c490b0e95f", + "fp6/bf6/e8m0/e4m3/16": "cc5d92ba276315961bdf4bf7dce52429f621855cc6df48f37d88a40c4da01adf", + "fp6/bf6/e5m3/e8m0/32": "ddacc734ad139b1512ab3d6b0b71e41615375b4df23f14fd85d808d81ea7a5b4", + "fp6/bf6/e5m3/e8m0/16": "a7c46f2e6b333888d4cec2443b5c3fdb3346e9495a8bf561f6c3f932af2c8ab1", + "fp6/bf6/e5m3/e5m3/32": "9810a711a39b9e28567362157c0f930e7dd41840929a3b58b77315d7927e36b2", + "fp6/bf6/e5m3/e5m3/16": "f9104ac1d0149b41b419501856ba1843e6f1cca4c308594c45bec0c8ae363642", + "fp6/bf6/e5m3/e4m3/32": "bbf45b765be001271d1ab603bb7b11f575146e17749bf2bb133b0564b06f05fd", + "fp6/bf6/e5m3/e4m3/16": "05b7a59fa31a3fa1aeea427afb90442e3c78d6cb4616252d729e7ce04bb1647c", + "fp6/bf6/e4m3/e8m0/32": "5d543f34f98f02e7312e91ee07816570d2191e7a4646e9ca08a7f1a8a7123916", + "fp6/bf6/e4m3/e8m0/16": "cbf800ca968fc1eff36ead048c0dcd1484def3b5c95693ceb3780fb3b8fb7419", + "fp6/bf6/e4m3/e5m3/32": "1e25496d4d633099945a593599708e0d19ef73bbc7120b2bc8e9bda88602d656", + "fp6/bf6/e4m3/e5m3/16": "68ce52e9a257a3ef6f4bdcf63570935a2050af336e049e61a875465d5341b082", + "fp6/bf6/e4m3/e4m3/32": "d511b206187727267a09ef36965abe8d239a84c01e114d683728ba1154d20533", + "fp6/bf6/e4m3/e4m3/16": "9105abdf342ed0f0af7240b1eec27d84d2eec50647f3f9a4724c56a0003c8a02", + "fp6/fp4/e5m3/e8m0/32": "faab9d36134ffdb8f062c7f7c93952752505fecdbf7f9da88134fbcc7029e17c", + "fp6/fp4/e5m3/e8m0/16": "d5ab4c07a20bf2627af636cf9794388bc9addc6871a65c5282300934a5661f4f", + "fp6/fp4/e5m3/e5m3/32": "bc32296a57e0ec7cc62ef9061aca229919cb0e00cdfee3091bb24640871b2037", + "fp6/fp4/e5m3/e5m3/16": "fb54df7c84d95a8aec6723dae30346c4d9bfda81675d3684966b5507a751049a", + "fp6/fp4/e5m3/e4m3/32": "925a151731a02c2e774400ccb71dc4efa42cf785c65322e54a96d3f7c0596d78", + "fp6/fp4/e5m3/e4m3/16": "6c055810b436a0b0eabaaf67f7f3fc9c4bc0ffa7d6cfb6646b4da98fe8949a3c", + "fp6/fp4/e4m3/e8m0/32": "24cc1afd74062aa8eb7f0309e64c18831cb6fcf198c204274c0000bee25bd785", + "fp6/fp4/e4m3/e8m0/16": "bbacca550b876e1e3af0f18af6e45d76b69094fb7f8c135bdb3d39b6f1fa6913", + "fp6/fp4/e4m3/e5m3/32": "dd1c68c367424ffc073609aa7fab8104f4b1ccae8865a2f6c69a177c70f59e80", + "fp6/fp4/e4m3/e5m3/16": "026c2b41ff0a21c31781ed985d0059ad631eaa269f554053c574c77798dfe038", + "fp6/fp4/e4m3/e4m3/32": "273cba267eeb719cab6f11dbf9d4b66be170e8a81df8f2c8090183398ddb09a2", + "fp6/fp4/e4m3/e4m3/16": "e409615a9fcef589d3e749ea20f39603b2a51c3d0c41c5e2cdb74a8926d82ba0", + "bf6/fp8/e8m0/e5m3/32": "eeb24b60744ef3886b940d40b00362cacc6f0c956690d1af202861c496d8e4c6", + "bf6/fp8/e8m0/e5m3/16": "f516ad9a2e2c1f14445e887314e0312f4e561bf7433f0deaef472aa78cee4ebd", + "bf6/fp8/e8m0/e4m3/32": "3f36880529be07fa8c83ee6f3cb8ff74c65337415038b98e98a8882c18295c0c", + "bf6/fp8/e8m0/e4m3/16": "1caf5d5d51279b059f212ddccb79b68aada0d107076c73b30bd1ef54bc656abe", + "bf6/fp8/e5m3/e8m0/32": "0c831457ec4ff89c6540f2a420138531c46169d947dc6088bcc07c9b9763ac37", + "bf6/fp8/e5m3/e8m0/16": "58d38bd0b44697fef32d985eafbac4884e0baa560d508be66c529398ec638d1f", + "bf6/fp8/e5m3/e5m3/32": "155f5e0ef1ef634974dd8f3007cba8883c18a5d640179594e33027145c161465", + "bf6/fp8/e5m3/e5m3/16": "af529718d9672089cf24114236a39f41eb0881db22808e20cc818a0f0e6fe367", + "bf6/fp8/e5m3/e4m3/32": "aef3375f13c97957c05b82ee29d6ef3a9ec1d711496571caca89bd8cd3717413", + "bf6/fp8/e5m3/e4m3/16": "71a115567c55c940fab3c10000a88b099786627b9f001521c535913c79381498", + "bf6/fp8/e4m3/e8m0/32": "1d677d86029a35f708b48171adf09194008d781ed1aa75e81161f66ddfc5de5a", + "bf6/fp8/e4m3/e8m0/16": "903026f2636a14a756a91c1becc1c40ecdb82c63ec62015831d15f575e77c0b0", + "bf6/fp8/e4m3/e5m3/32": "8e914dbf52426e3cfe5dcf0205481662403ff296d2d3329a72988bb4d1ed8279", + "bf6/fp8/e4m3/e5m3/16": "ac12968ef9db085b93a8543e61a8f7494ca79dbf4c572e717d9b8f3792fa776c", + "bf6/fp8/e4m3/e4m3/32": "01e938f675cf4ce9a7cab3fb76a7e44274926b0f11dc5b04e331d2cd4fb046ee", + "bf6/fp8/e4m3/e4m3/16": "f1f7f8f3cb504786e3da327b0c80134af48a178a36b880360a864015ec0aecb1", + "bf6/bf8/e8m0/e5m3/32": "283b7f069a4b2339d928f16de5733ec0f339a2a24e443f4b5bf76982c5f88dba", + "bf6/bf8/e8m0/e5m3/16": "01fc8e783ec1e1f975a1bed6db5ff2bdc7586560c1d727425e1440077224b1c4", + "bf6/bf8/e8m0/e4m3/32": "2f91197fafa4bf501f5478bcbfda2b7b7e36d97e86b22eebdd70d82c866941c5", + "bf6/bf8/e8m0/e4m3/16": "471550447702ff2d41c4d156aa5e1142daa29e8a939be2989ac2953df892d825", + "bf6/bf8/e5m3/e8m0/32": "b0a49d3f7c9dd9690165d7147cdadbcdae873e8331ff569621fc440772f9544f", + "bf6/bf8/e5m3/e8m0/16": "54a7acd085554ebed111c34af46ef0a6272cc42a3a188494a8d5515c52ae205d", + "bf6/bf8/e5m3/e5m3/32": "e5be573db446851b799ae6f52ed2ad72e1731efd02edc25af9acbce383e42c10", + "bf6/bf8/e5m3/e5m3/16": "a78b606408072246d52a0d0a03b726c580ea245342f1e7cc2ccae81c6d8242aa", + "bf6/bf8/e5m3/e4m3/32": "4f383aab8ffe0c23ae5e9abebfae8170313890a827a6ec42e2314ab60db87d5a", + "bf6/bf8/e5m3/e4m3/16": "4217ee75bf0bb42616f613569767ebd677b330029b71e9a98dd7ad1e44acf397", + "bf6/bf8/e4m3/e8m0/32": "c2471b517cdf8d91eaea0e49fd9db260b661c6469f7fe9bebee74025fe056e9c", + "bf6/bf8/e4m3/e8m0/16": "2f7ff8d556bfdc149f10949f3d1c242ff531c78cac4448f7cb27a6d808024b69", + "bf6/bf8/e4m3/e5m3/32": "3f00e79a55a77d35ea8bea3bf597cd48070c9c70f966f4752126be6141ac5e22", + "bf6/bf8/e4m3/e5m3/16": "df990786fd04718612772f90ce6c2619129c0e23122e534da16374ad03af9d7d", + "bf6/bf8/e4m3/e4m3/32": "b99fbd5ceab2fa4af4d70e28032448be5616aac9c5baa7805723f0eae0518ed4", + "bf6/bf8/e4m3/e4m3/16": "d68fae154927b102eaed126ba5dffba8f09b6a4edafe49a401ca6fa282ec2261", + "bf6/fp6/e8m0/e5m3/32": "23b022b264b2c4ad2d6e1338eb3ac24d064d2d5dbea86c218433c6edb1e62fa9", + "bf6/fp6/e8m0/e5m3/16": "4a3a441cd8381977530837ecca2de7e3d55679cca7ef47acb7da04ab0db8d29c", + "bf6/fp6/e8m0/e4m3/32": "721d7f1343ca1301325760e76f04e552177844ae9e8b0e091325d125b4fd046b", + "bf6/fp6/e8m0/e4m3/16": "050bc4fc8dd184d0ebf6ea11c77b4167eb2007d0186b3596de1e2d3ae4b6a414", + "bf6/fp6/e5m3/e8m0/32": "45823a850e02ee3dc46ea4974fc0e7bf276740b9f6cb877ef19a6f3cfdc8b6ea", + "bf6/fp6/e5m3/e8m0/16": "be1a956a3b5f5bedcbc47589e09291e3680948e1e7aa73e63853695659f18f31", + "bf6/fp6/e5m3/e5m3/32": "47d76f4e7c61edf67d79d8d6091caff0fcd85e07806a87923fab99018aeed947", + "bf6/fp6/e5m3/e5m3/16": "3ce7c53b6dd7dc815fd54d6e0a1a2b536f77aacf41ff58058ebe941d2101288d", + "bf6/fp6/e5m3/e4m3/32": "7dd71953b0ebe38a7bdffa231e97dc59108044a58e72afefdcb40d30977b3d7f", + "bf6/fp6/e5m3/e4m3/16": "327ddaa1b6fa62a0d19176199e7e25dcb01db6a4f9102275104c22c44620475a", + "bf6/fp6/e4m3/e8m0/32": "6b641131ffdd005bcb19b6e76c138e0cc56ae70f8affcbfe3c5cf4aa9b515789", + "bf6/fp6/e4m3/e8m0/16": "36dfce46ed4e631e6ed3118aa590728fdf732919cdb0f1f233d4626cab605850", + "bf6/fp6/e4m3/e5m3/32": "f27b757fc913e6ebdb5821b70335fe403e938903ea4afb9b9e4acea9d052a5e8", + "bf6/fp6/e4m3/e5m3/16": "471fb451e4405b0ab7b38c9b6f77ec619378fdf2a85420671091a11f00593a13", + "bf6/fp6/e4m3/e4m3/32": "72a2394e356b94ccf75514fff67a7e7a3e94b994357d92e1620816108560fa23", + "bf6/fp6/e4m3/e4m3/16": "b0e9c029ad5957632ca73edde0118c7ccf393c91d56368c15554cd9f684750e2", + "bf6/bf6/e8m0/e5m3/32": "92f4cb18395a5d0cd11d7345f17578c6c419c9b77acb79914f6b841d67e317ed", + "bf6/bf6/e8m0/e5m3/16": "4957edf77119d0e4d336ffa1688675b3ccf7f5e7d131d19b6a0591afe5c33106", + "bf6/bf6/e8m0/e4m3/32": "76153849c501734d6b1f2b76f32dec78c84cd163fbba8e3a88907d3247044947", + "bf6/bf6/e8m0/e4m3/16": "91c14aa795da944d6f288d558e186bf39b4fd00131cab82c89569166f6b3580c", + "bf6/bf6/e5m3/e8m0/32": "7fee78a45c19db932da84cbb2dc1282394596f77cf05c6b93bb78a2349382134", + "bf6/bf6/e5m3/e8m0/16": "91fea0ddc932f845088c7f28b9fd598b067505fcc27f6f35d34251ff5f9132be", + "bf6/bf6/e5m3/e5m3/32": "7e08adee9a0a5b7c9ebbaf7a30936b5820d6ee25df621771a1ae037521b6c310", + "bf6/bf6/e5m3/e5m3/16": "1b8b1ad21310ffc755338dc635825a6e83ce02e034688e6145416494bd406d77", + "bf6/bf6/e5m3/e4m3/32": "9577aebc348be9f0a6c79444bc613f4a8b9276035fb2b008e5782be3430c5464", + "bf6/bf6/e5m3/e4m3/16": "d226a95b9b36edaa7117b981325d4bb082e53377203879dfd1c33aee624690db", + "bf6/bf6/e4m3/e8m0/32": "fb13ade64fb4b6c286f94cb63a53a5e771239bc6edeada159b736680acea1f85", + "bf6/bf6/e4m3/e8m0/16": "53399c7c4951ef9c6d1f083870ad59fb734b43f545bea3494d2f4b9bd6b5dadd", + "bf6/bf6/e4m3/e5m3/32": "99fbfae3492310e7f3e64de5d821a17a9ed14f65838e237467a019a46f9a6a6e", + "bf6/bf6/e4m3/e5m3/16": "1d1e08819200db18bb703c91b53c435b1345b5d3195b5b8ff01d04c5b3102116", + "bf6/bf6/e4m3/e4m3/32": "0844309de13ab3757efac82731e452dc55effc4f742136309e10a78d45a2c035", + "bf6/bf6/e4m3/e4m3/16": "253f0b81f65b560aa751cea6ed80e384ef8df5166f034544a173599c9c9b825b", + "bf6/fp4/e5m3/e8m0/32": "b0530d58dd9ddacfa3c497d892ad560ede969078ae3ccd2e2521f3219396999c", + "bf6/fp4/e5m3/e8m0/16": "b149fc0b32f1b1d9adf2552acd14c6603e9d8263ba33400e2e9a53652ba74e9f", + "bf6/fp4/e5m3/e5m3/32": "9b1ddf0c531112ab1d6f2380ce34dc77d72f8181e098ea09761c99bc3a0f1fd8", + "bf6/fp4/e5m3/e5m3/16": "12eaa4fe79c78ae238e677b0fdd9912a333ab0cb349af322719defc7b864e0a2", + "bf6/fp4/e5m3/e4m3/32": "fb25dc4354828a7e510798c99493df66d86b8061a8e4536485b6fc7a7ae52b3d", + "bf6/fp4/e5m3/e4m3/16": "16b0827b1fc5ada714071aa85d03a0ec2bae0700ce2652a15fc23339213f4b63", + "bf6/fp4/e4m3/e8m0/32": "09c3012ee25b9a9f3cc03dc9ead3e35db6eb38c4f00c01b3de73cc11bdc2dd34", + "bf6/fp4/e4m3/e8m0/16": "6bc5e5035b8260a0f1eb741c05783ec9c1f3830449f2387a887d54c54b5096ee", + "bf6/fp4/e4m3/e5m3/32": "10d280ce5fef41c5ace6ada641416e9f8633be556ce5eb36235417576165b4e4", + "bf6/fp4/e4m3/e5m3/16": "fb370fd72e5a1dfa337139b4ac25c6b18044e10694edc370d81b0e2fb3940c89", + "bf6/fp4/e4m3/e4m3/32": "8235b69fc70877ca1229fe5397d50f2569e58a7f6ccbf7f98fe6c2d93ffd55fc", + "bf6/fp4/e4m3/e4m3/16": "9729c52d5b9f148a50749e16cb26c8c875c29e883d603e25398dc0fc58c50c3a", + "fp4/fp8/e8m0/e5m3/32": "b06729d6f01be089b4d012df7e69618c16291a9123c9863763441139d3f7be5a", + "fp4/fp8/e8m0/e5m3/16": "a8f0db0c4c4d447bf01bf8ad2ed8ba2b49cbeb1b314bd17f7ca67fe73dfa4743", + "fp4/fp8/e8m0/e4m3/32": "5efb47f2ad6f196f195ee4791530ad086bd97b5c5b4da1b1972b1a60637d6ecb", + "fp4/fp8/e8m0/e4m3/16": "f2a4020e74d184e9c89a576ed090e92702feae04b52046d13d984ab61563762d", + "fp4/fp8/e5m3/e5m3/32": "4866c2929146dae0c3bc11fc698f99e5051cd9bbca786494773c59fb21e32b3e", + "fp4/fp8/e5m3/e5m3/16": "6bebc8bb6d9fb425651d94be96820ed9415a4e743344c082673a1f6e27751369", + "fp4/fp8/e5m3/e4m3/32": "3f51d1ff30fbcdf8fcefe5a4765998d3acbe5be698bb082695991ce43c0d7c6b", + "fp4/fp8/e5m3/e4m3/16": "642ad05d1ac3aff7fe30695d5bead353f481ebc8789407c84073b23c47a8860e", + "fp4/fp8/e4m3/e5m3/32": "41f6c622c0fd24b0ec082031899f40ee1b46aabee668a2686fef437727c5fe62", + "fp4/fp8/e4m3/e5m3/16": "e8ee2ea5860d92b20f2b801eaffb6bb7c3b88c1802baa9458bd92a131766f1e6", + "fp4/fp8/e4m3/e4m3/32": "458e78d06e293be53149b4e76e57bab55f4e24a5bfc8dbf87aa7a2073b22a07b", + "fp4/fp8/e4m3/e4m3/16": "195a82f9c0c72c2538e4b92493007a16bd1def8ae80ff4d93425c2bcf2f658ab", + "fp4/bf8/e8m0/e5m3/32": "fafc4dc0034112ddf6e1e34fc5c132b2444dc713b43fc815b7467c068e78487e", + "fp4/bf8/e8m0/e5m3/16": "a3fff2dedb6e65c99050423eb5c8d7c6bf1d5b1c359f09c2b1e3e35fe43f37f0", + "fp4/bf8/e8m0/e4m3/32": "cc0595ed0cde391623693504dd132ebebc6daf03e096b2305a4749a007c4e852", + "fp4/bf8/e8m0/e4m3/16": "84466c20fa688db1926d19c38380f904fdf15540ff74c3d393def973c7e939df", + "fp4/bf8/e5m3/e5m3/32": "f5b642cf633d10aeaae17ddcc0e07f9c6e5901eda5b1500cc709c6e7f4f0b2ab", + "fp4/bf8/e5m3/e5m3/16": "554b9470ff7e1fb79b7beca0ab8942026c446a15a5a1c21bffaed056149bd7cb", + "fp4/bf8/e5m3/e4m3/32": "d09b6e7613bdcf43595f5391624fae51ce3c352a539a681e5e327ffabfe28852", + "fp4/bf8/e5m3/e4m3/16": "81e8e798235d4279163e0a1d706aa91e67c7289abaeb9f2439a7414ce6d5c023", + "fp4/bf8/e4m3/e5m3/32": "6b0597a2ad2117630edc45142fc9478e42feaeb26880d97fe2e5d0b015af2125", + "fp4/bf8/e4m3/e5m3/16": "b575d04bafc6e43d8d61f797c32ab00f873c2dddb5e25133d43e1361af5a400f", + "fp4/bf8/e4m3/e4m3/32": "994637afd07c7c523bb1cf9c270f51a384a6e3ad702b154de64d7149c292d089", + "fp4/bf8/e4m3/e4m3/16": "94159f433473f2ea83bb675318495816c09ccc629e36cff62fb8a01000663c86", + "fp4/fp6/e8m0/e5m3/32": "670684d268b5df752aa6becfab6c67d6f049f8bdf11e42fbb8f5193855d3e532", + "fp4/fp6/e8m0/e5m3/16": "e5f5514079be9f92b9b44c98e417c792bf1813c2c0f2d15472da2890d7db73d6", + "fp4/fp6/e8m0/e4m3/32": "6e1130a41c5fd8f2b5018fdc733f33894b93813b06a0c653a80fdec4b0a48381", + "fp4/fp6/e8m0/e4m3/16": "cd87c515a19ac15e63920aebfdc6b867b063829fad30ec5daa83b248b7667d65", + "fp4/fp6/e5m3/e5m3/32": "8466f703260278bbe5fc7e814089f0cbe23ea0e54945192f0643b53871344746", + "fp4/fp6/e5m3/e5m3/16": "589a06ca2b5000a4d594991677833b6c6be78ca36b6a69aeeba3c6a2e7500e29", + "fp4/fp6/e5m3/e4m3/32": "a27f48d294dd42c876be5a3a9bc418409f29f739e48165e682059f2bc1812571", + "fp4/fp6/e5m3/e4m3/16": "929e418e6cfc39d6ccb3b2a069eef3787a14ca700f312d78c4a664c7dcede475", + "fp4/fp6/e4m3/e5m3/32": "af95a99193325f03dc5fd43e6bfa21ada00f3f3d26c9da02c65c4019f0c313f0", + "fp4/fp6/e4m3/e5m3/16": "e4a5165940b5df77951c93d38428fc41f561d8b16ad33548a6e9b4e7b591d4f5", + "fp4/fp6/e4m3/e4m3/32": "97da994159532f3245f81f571badb66eed85d19c4fd1253e17e5aca0c7becc36", + "fp4/fp6/e4m3/e4m3/16": "a8afaf72967253f57b53502b430fdad74157274530c8b88c8c86a44cbab0c575", + "fp4/bf6/e8m0/e5m3/32": "b83c11fb4d0e3c3303aaa0ce531c3f8a9de0012c44a1a4fdaa1b3147472ed86a", + "fp4/bf6/e8m0/e5m3/16": "123418e5c59dc24fed54a603c8888056a069ecec30bb48bdf4b21bf2991c0623", + "fp4/bf6/e8m0/e4m3/32": "348cc241c497e67305fd146d9947318ff5db28f6d316eb4c13d92da7d3328663", + "fp4/bf6/e8m0/e4m3/16": "bff00f28e8db5b7200e0785d0e88766e63bb12a90f83eb93213f1675f26b67a8", + "fp4/bf6/e5m3/e5m3/32": "042065ec2f4a53c30126bb0c12c6f677e6e00c2ce9e2b487ec0129619ba604f2", + "fp4/bf6/e5m3/e5m3/16": "6530823c8b17dee4b574f79dbcb53f804a5172435ca553b4cddbcc5163135ef7", + "fp4/bf6/e5m3/e4m3/32": "60c5b248ab017506e79747f8d93d67f2945931f56d1138e02c8d1e0a7b09dc9b", + "fp4/bf6/e5m3/e4m3/16": "f3f74b4617fd516353fee9c6bb85df32d98b6101ef45b5ccfd2b99a914087727", + "fp4/bf6/e4m3/e5m3/32": "50634c3edd88023a58a499b2ef5f6571fb809702d8ec271861ec8b7ee12271cd", + "fp4/bf6/e4m3/e5m3/16": "eddf0fd2b857cd1a5139487136f4f66b8dd0d401185f022586c8362b48d56c9d", + "fp4/bf6/e4m3/e4m3/32": "1c3e10354b0219b9631767fce1e946923c9d2df37aae8f68d802b4903b544b2a", + "fp4/bf6/e4m3/e4m3/16": "079ea7fc5761df17e487e9a9c6234b97bd10f28c5c0d61ee2403cd8f0e40c5b6", + "fp4/fp4/e8m0/e5m3/32": "873db2230629fda89e5504fa3607af3721c0790819b79d307ad2bc07c8104856", + "fp4/fp4/e8m0/e5m3/16": "614688537f04af76cec6ff46a8bcd8cd6dfac3c162e83b011ff495b5c8aff2bd", + "fp4/fp4/e8m0/e4m3/32": "337c9a1c8f85c34a8d9b31930daafe4479fe302a3353515625de0f73e93a81d6", + "fp4/fp4/e8m0/e4m3/16": "04425faa0c5d5f1fae0867c044ea6cef40765f052c43c2feef9e03945f6935e8", + "fp4/fp4/e5m3/e8m0/32": "f06925802cba1da41bab46e07402b49aefb7227c2c109c60b9fb92cf2a89f9ac", + "fp4/fp4/e5m3/e8m0/16": "191c5a67b93b673b43ff0c15159d707e9f1054ea82db40ee225ed76cf3906e86", + "fp4/fp4/e5m3/e4m3/32": "f4fce23e6412f9e1bdbebd273830b04060a4274d5bdf75e675546db623d13a68", + "fp4/fp4/e5m3/e4m3/16": "de5183c1cf53a9c802875ead562ea73b0b6d2c1fd323890597ac8310adde4617", + "fp4/fp4/e4m3/e8m0/32": "807be10ab14d7e8ab572f26e8d6eef6375fca00f2bf2ab8d95a3601a5839adcf", + "fp4/fp4/e4m3/e8m0/16": "bd6da3b9449059b0e59a72d319d2542ba81c24d78fdf629c0608e74dc05ff73f", + "fp4/fp4/e4m3/e5m3/32": "4aab2c3dc6614104d423b193ed22007b1880bc0247ecee5a1aeb8777b4d0a353", + "fp4/fp4/e4m3/e5m3/16": "a547dcdaa82408dc8f7bc3d5467e5e584b8f02b1df24502a446e4b2de582438d" } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py index 3df2d4600ef5..8a95ebf22278 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py @@ -223,8 +223,8 @@ def test_e5m2_is_not_a_scale_dtype_alias(dtype, field): def test_scaled_catalog_identity_and_backend_contract(): catalog = ArchTarget.from_gfx("gfx1250").mma rows = [row for row in catalog.ops if row.family == "wmma_scaled"] - assert len(rows) == 86 - assert len({row.op_id for row in rows}) == 86 + assert len(rows) == 450 + assert len({row.op_id for row in rows}) == 450 for row in rows: tokens = { "fp8e4m3": "fp8", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py index 030c715ac2a0..5b82bfdefeec 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py @@ -109,7 +109,7 @@ def test_mixed_atoms_do_not_leak_to_other_targets(a, b): @pytest.mark.parametrize("scales", [("e4m3", "e8m0", 32), ("e5m3", "e8m0", 16)]) -def test_mixed_atoms_do_not_admit_other_scale_contracts(scales): +def test_mixed_atoms_admit_independent_scale_contracts(scales): assert ( ArchTarget.from_gfx("gfx1250").mma.op_for_shape( family="wmma_scaled", @@ -121,7 +121,7 @@ def test_mixed_atoms_do_not_admit_other_scale_contracts(scales): k=128, scales=scales, ) - is None + is not None ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py index b006dcda7dfc..829fad2a2134 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py @@ -30,11 +30,11 @@ FORMATS = ("fp8", "bf8", "fp6", "bf6", "fp4") SCALES = ("e8m0", "e5m3", "e4m3") -PAIRS = ( - [(d, "fp4", "e8m0", s) for d in FORMATS[:-1] for s in SCALES[1:]] - + [("fp4", d, s, "e8m0") for d in FORMATS[:-1] for s in SCALES[1:]] - + [("fp4", "fp4", s, s) for s in SCALES[1:]] -) +PAIRS = [ + (a, b, sa, sb) + for a, b, sa, sb in product(FORMATS, FORMATS, SCALES, SCALES) + if (sa, sb) != ("e8m0", "e8m0") +] def spec_for(a="fp4", b="fp4", sa="e4m3", sb="e4m3", block_k=32, **kwargs): @@ -57,11 +57,6 @@ def spec_for(a="fp4", b="fp4", sa="e4m3", sb="e4m3", block_k=32, **kwargs): @pytest.mark.parametrize("a,b,sa,sb", list(product(FORMATS, FORMATS, SCALES, SCALES))) @pytest.mark.parametrize("block_k", [32, 16]) def test_catalog_and_builder_agree_on_legal_scale_combinations(a, b, sa, sb, block_k): - expected = ( - (sa == "e8m0" or a == "fp4") - and (sb == "e8m0" or b == "fp4") - and (a != "fp4" or b != "fp4" or sa == sb) - ) atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( family="wmma_scaled", a_dtype=a, @@ -72,8 +67,8 @@ def test_catalog_and_builder_agree_on_legal_scale_combinations(a, b, sa, sb, blo k=128, scales=(sa, sb, block_k), ) - assert (atom is not None) == expected - assert is_valid_spec(spec_for(a, b, sa, sb, block_k))[0] == expected + assert atom is not None + assert is_valid_spec(spec_for(a, b, sa, sb, block_k))[0] @pytest.mark.parametrize("a,b,sa,sb", PAIRS) @@ -215,25 +210,23 @@ def test_scale_formats_golden(case, expected_sha): assert hashlib.sha256(llvm.encode()).hexdigest() == expected_sha -@pytest.mark.parametrize( - "a,b,sa,sb", [("fp8", "fp4", "e4m3", "e8m0"), ("fp4", "fp4", "e4m3", "e5m3")] -) +@pytest.mark.parametrize("field,value", [("k", 64), ("c_dtype", "fp16")]) def test_backend_rejects_invalid_contract_even_if_catalog_injected( - monkeypatch, a, b, sa, sb + monkeypatch, field, value ): from rocke.core.arch.target import MmaCatalog atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( family="wmma_scaled", - a_dtype=a, - b_dtype=b, + a_dtype="fp6", + b_dtype="fp8", c_dtype="fp32", m=16, n=16, k=128, - scales=("e8m0", "e8m0", 32), + scales=("e4m3", "e5m3", 32), ) - invalid = replace(atom, a_scale_dtype=sa, b_scale_dtype=sb) + invalid = replace(atom, **{field: value}) monkeypatch.setattr(MmaCatalog, "by_op_id", lambda self, op_id: invalid) with pytest.raises(ValueError, match="unsupported scaled WMMA backend contract"): gfx1250_scaled_wmma(atom.op_id) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py index b16f8d5d5d37..342d7a09a8f3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py @@ -61,16 +61,19 @@ def _build_scaled_atom(*, scale16: bool): class TestGfx1250ScaledWmma(unittest.TestCase): - def test_backend_rejects_unsupported_scale_type_on_either_input(self): + def test_backend_selects_scale_type_independently_on_either_input(self): catalog = ArchTarget.from_gfx("gfx1250").mma - for field in ("a_scale_dtype", "b_scale_dtype"): - atom = replace(_scaled_atom(), **{field: "e4m3"}) - with ( - self.subTest(field=field), - mock.patch.object(catalog, "by_op_id", return_value=atom), - self.assertRaisesRegex(ValueError, "unsupported scaled WMMA"), - ): - gfx1250_scaled_wmma(atom.op_id) + for operand, field in enumerate(("a_scale_dtype", "b_scale_dtype")): + for selector, dtype in enumerate(("e8m0", "e5m3", "e4m3")): + atom = replace(_scaled_atom(), **{field: dtype}) + with ( + self.subTest(field=field, dtype=dtype), + mock.patch.object(catalog, "by_op_id", return_value=atom), + ): + spec = gfx1250_scaled_wmma(atom.op_id) + expected = [0, 0] + expected[operand] = selector + self.assertEqual(spec.scale_formats, tuple(expected)) def test_scale_fragment_counts_are_checked_for_each_source(self): catalog = ArchTarget.from_gfx("gfx1250").mma diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp index e898d5041246..5c9c2b384f01 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp @@ -2,18 +2,19 @@ // SPDX-License-Identifier: MIT /* Exercise public catalog queries against stored keys, without normalizing * away malformed table entries. Alias queries must find the same atom. */ +#include +#include + +#include +#include +#include + #include "mma_family_index.h" #include "rocke/arch_target.h" #include "rocke/error.hpp" #include "rocke/helper_rocke.core.arch.h" #include "rocke/wmma_scale_internal.h" -#include -#include -#include -#include -#include - static const char* short_dtype(const char* dtype) { const char* aliases[][2] = {{"fp8e4m3", "fp8"}, @@ -173,7 +174,7 @@ static int test_scale_contracts() for(int input = 0; input < 2; ++input) { auto unsupported = *base; - (input == 0 ? unsupported.a_scale_dtype : unsupported.b_scale_dtype) = "e4m3"; + (input == 0 ? unsupported.a_scale_dtype : unsupported.b_scale_dtype) = "e5m2"; CHECK(rejects_query([&] { rocke_scaled_wmma_contract(&unsupported); })); } int count = 1; @@ -304,22 +305,16 @@ static int test_scale_contracts() op.scale_block_k); CHECK(strcmp(id, op.op_id) == 0); CHECK(rocke_mma_catalog_by_op_id(&arch->mma, id) == &op); - if(strcmp(op.a_dtype, "fp4e2m1") != 0) - { - auto invalid = op; - invalid.a_scale_dtype = "e4m3"; - CHECK(rejects_query([&] { rocke_scaled_wmma_contract(&invalid); })); - } - if(strcmp(op.a_dtype, "fp4e2m1") == 0 && strcmp(op.b_dtype, "fp4e2m1") == 0) - { - auto invalid = op; - invalid.b_scale_dtype = strcmp(op.a_scale_dtype, "e4m3") == 0 ? "e5m3" : "e4m3"; - CHECK(rejects_query([&] { rocke_scaled_wmma_contract(&invalid); })); - } + const auto contract = rocke_scaled_wmma_contract(&op); + CHECK(contract.scale_formats[0] >= 0 && contract.scale_formats[0] <= 2); + CHECK(contract.scale_formats[1] >= 0 && contract.scale_formats[1] <= 2); + auto invalid = op; + invalid.k = 64; + CHECK(rejects_query([&] { rocke_scaled_wmma_contract(&invalid); })); ++scaled_rows; } } - CHECK(packed_rows == 4 && scaled_rows == 86); + CHECK(packed_rows == 4 && scaled_rows == 450); return 0; } @@ -388,7 +383,7 @@ static int test_scale_layouts_and_families() } } } - CHECK(scaled == 86); + CHECK(scaled == 450); return 0; } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/run_diff.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/run_diff.py index 5525b25934e4..7fae29416d88 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/run_diff.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/run_diff.py @@ -88,7 +88,8 @@ def collect_ref_shas(results): return out -MAX_CFG = 128 # hard cap on config enumeration per family +MAX_CFG = 128 # default cap on config enumeration per family +FAMILY_MAX_CFG = {"gfx1250_lowering": 512} TIMEOUT = 120 # seconds per emitter invocation _CANON = None @@ -327,7 +328,7 @@ def run_family(name, archive, mode, canonical=False, src_dir=None): } configs = [] range_drift = None - for idx in range(MAX_CFG): + for idx in range(FAMILY_MAX_CFG.get(name, MAX_CFG)): cr, co, ce = run_c(binpath, idx, mode) pr, po, pe = run_py(name, idx, mode, src_dir=src_dir) # mode not supported by this emitter -> stop trying this mode diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py index a1ed6a972f79..fad536f6e0cf 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py @@ -342,3 +342,15 @@ def test_discovering_no_families_exits_nonzero(self): if __name__ == "__main__": unittest.main() + + +def test_gfx1250_scale_catalog_configs_are_not_truncated(): + import importlib.util + + source = Path(__file__).resolve().parents[1] / "parity" / "gfx1250_lowering_emit.py" + spec = importlib.util.spec_from_file_location("gfx1250_lowering_configs", source) + emitter = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "path", [str(source.parent), *sys.path]): + spec.loader.exec_module(emitter) + assert len(emitter.CONFIGS) == 472 + assert run_diff.FAMILY_MAX_CFG["gfx1250_lowering"] > len(emitter.CONFIGS) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c index a9bc6a129b51..339cb61dae30 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c @@ -306,8 +306,12 @@ static void build_async_store(rocke_ir_builder_t* b) rocke_value_t* lds_ptr = rocke_b_smem_addr_of(b, smem); const int widths[] = {1, 4, 8, 16}; for(int i = 0; i < 4; ++i) - rocke_b_global_store_async_from_lds( - b, out, lds_ptr, widths[i], widths[i], /*cachepolicy=*/3); + rocke_b_global_store_async_from_lds(b, + out, + lds_ptr, + widths[i], + widths[i], + /*cachepolicy=*/3); rocke_b_ret(b); } @@ -518,6 +522,370 @@ static const config_t CONFIGS[] = { {NULL, "gfx1250", "fp4", true, "bf6", "e4m3", "e8m0"}, {NULL, "gfx1250", "fp4", true, "fp4", "e5m3", "e5m3"}, {NULL, "gfx1250", "fp4", true, "fp4", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp8", false, "fp8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp8", false, "fp8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp8", false, "fp8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp8", false, "fp8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp8", false, "fp8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp8", false, "fp8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp8", false, "fp8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp8", false, "fp8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp8", false, "bf8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp8", false, "bf8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp8", false, "bf8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp8", false, "bf8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp8", false, "bf8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp8", false, "bf8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp8", false, "bf8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp8", false, "bf8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp8", false, "fp6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp8", false, "fp6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp8", false, "fp6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp8", false, "fp6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp8", false, "fp6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp8", false, "fp6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp8", false, "fp6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp8", false, "fp6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp8", false, "bf6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp8", false, "bf6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp8", false, "bf6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp8", false, "bf6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp8", false, "bf6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp8", false, "bf6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp8", false, "bf6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp8", false, "bf6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp8", false, "fp4", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp8", false, "fp4", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp8", false, "fp4", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp8", false, "fp4", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp8", false, "fp4", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp8", false, "fp4", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf8", false, "fp8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf8", false, "fp8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf8", false, "fp8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf8", false, "fp8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf8", false, "fp8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf8", false, "fp8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf8", false, "fp8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf8", false, "fp8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf8", false, "bf8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf8", false, "bf8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf8", false, "bf8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf8", false, "bf8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf8", false, "bf8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf8", false, "bf8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf8", false, "bf8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf8", false, "bf8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf8", false, "fp6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf8", false, "fp6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf8", false, "fp6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf8", false, "fp6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf8", false, "fp6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf8", false, "fp6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf8", false, "fp6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf8", false, "fp6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf8", false, "bf6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf8", false, "bf6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf8", false, "bf6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf8", false, "bf6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf8", false, "bf6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf8", false, "bf6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf8", false, "bf6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf8", false, "bf6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf8", false, "fp4", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf8", false, "fp4", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf8", false, "fp4", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf8", false, "fp4", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf8", false, "fp4", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf8", false, "fp4", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp6", false, "fp8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp6", false, "fp8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp6", false, "fp8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp6", false, "fp8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp6", false, "fp8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp6", false, "fp8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp6", false, "fp8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp6", false, "fp8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp6", false, "bf8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp6", false, "bf8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp6", false, "bf8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp6", false, "bf8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp6", false, "bf8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp6", false, "bf8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp6", false, "bf8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp6", false, "bf8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp6", false, "fp6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp6", false, "fp6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp6", false, "fp6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp6", false, "fp6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp6", false, "fp6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp6", false, "fp6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp6", false, "fp6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp6", false, "fp6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp6", false, "bf6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp6", false, "bf6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp6", false, "bf6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp6", false, "bf6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp6", false, "bf6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp6", false, "bf6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp6", false, "bf6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp6", false, "bf6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp6", false, "fp4", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp6", false, "fp4", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp6", false, "fp4", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp6", false, "fp4", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp6", false, "fp4", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp6", false, "fp4", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf6", false, "fp8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf6", false, "fp8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf6", false, "fp8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf6", false, "fp8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf6", false, "fp8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf6", false, "fp8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf6", false, "fp8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf6", false, "fp8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf6", false, "bf8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf6", false, "bf8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf6", false, "bf8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf6", false, "bf8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf6", false, "bf8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf6", false, "bf8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf6", false, "bf8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf6", false, "bf8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf6", false, "fp6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf6", false, "fp6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf6", false, "fp6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf6", false, "fp6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf6", false, "fp6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf6", false, "fp6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf6", false, "fp6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf6", false, "fp6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf6", false, "bf6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf6", false, "bf6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf6", false, "bf6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf6", false, "bf6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf6", false, "bf6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf6", false, "bf6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf6", false, "bf6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf6", false, "bf6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf6", false, "fp4", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf6", false, "fp4", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf6", false, "fp4", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf6", false, "fp4", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf6", false, "fp4", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf6", false, "fp4", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "fp8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp4", false, "fp8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "fp8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp4", false, "fp8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "fp8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp4", false, "fp8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "bf8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp4", false, "bf8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "bf8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp4", false, "bf8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "bf8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp4", false, "bf8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "fp6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp4", false, "fp6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "fp6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp4", false, "fp6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "fp6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp4", false, "fp6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "bf6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp4", false, "bf6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "bf6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp4", false, "bf6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "bf6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp4", false, "bf6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "fp4", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp4", false, "fp4", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "fp4", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp4", false, "fp4", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp4", false, "fp4", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp4", false, "fp4", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "fp8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "fp8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp8", true, "fp8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp8", true, "fp8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "fp8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp8", true, "fp8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp8", true, "fp8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "fp8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp8", true, "bf8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "bf8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp8", true, "bf8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp8", true, "bf8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "bf8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp8", true, "bf8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp8", true, "bf8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "bf8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp8", true, "fp6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "fp6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp8", true, "fp6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp8", true, "fp6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "fp6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp8", true, "fp6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp8", true, "fp6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "fp6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp8", true, "bf6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "bf6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp8", true, "bf6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp8", true, "bf6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "bf6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp8", true, "bf6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp8", true, "bf6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "bf6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp8", true, "fp4", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp8", true, "fp4", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "fp4", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp8", true, "fp4", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp8", true, "fp4", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp8", true, "fp4", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf8", true, "fp8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf8", true, "fp8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf8", true, "fp8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf8", true, "fp8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf8", true, "fp8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf8", true, "fp8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf8", true, "fp8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf8", true, "fp8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf8", true, "bf8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf8", true, "bf8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf8", true, "bf8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf8", true, "bf8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf8", true, "bf8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf8", true, "bf8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf8", true, "bf8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf8", true, "bf8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf8", true, "fp6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf8", true, "fp6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf8", true, "fp6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf8", true, "fp6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf8", true, "fp6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf8", true, "fp6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf8", true, "fp6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf8", true, "fp6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf8", true, "bf6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf8", true, "bf6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf8", true, "bf6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf8", true, "bf6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf8", true, "bf6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf8", true, "bf6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf8", true, "bf6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf8", true, "bf6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf8", true, "fp4", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf8", true, "fp4", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf8", true, "fp4", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf8", true, "fp4", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf8", true, "fp4", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf8", true, "fp4", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp6", true, "fp8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp6", true, "fp8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp6", true, "fp8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp6", true, "fp8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp6", true, "fp8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp6", true, "fp8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp6", true, "fp8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp6", true, "fp8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp6", true, "bf8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp6", true, "bf8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp6", true, "bf8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp6", true, "bf8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp6", true, "bf8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp6", true, "bf8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp6", true, "bf8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp6", true, "bf8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp6", true, "fp6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp6", true, "fp6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp6", true, "fp6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp6", true, "fp6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp6", true, "fp6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp6", true, "fp6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp6", true, "fp6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp6", true, "fp6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp6", true, "bf6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp6", true, "bf6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp6", true, "bf6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp6", true, "bf6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp6", true, "bf6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp6", true, "bf6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp6", true, "bf6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp6", true, "bf6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp6", true, "fp4", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp6", true, "fp4", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp6", true, "fp4", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp6", true, "fp4", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp6", true, "fp4", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp6", true, "fp4", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf6", true, "fp8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf6", true, "fp8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf6", true, "fp8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf6", true, "fp8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf6", true, "fp8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf6", true, "fp8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf6", true, "fp8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf6", true, "fp8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf6", true, "bf8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf6", true, "bf8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf6", true, "bf8", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf6", true, "bf8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf6", true, "bf8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf6", true, "bf8", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf6", true, "bf8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf6", true, "bf8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf6", true, "fp6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf6", true, "fp6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf6", true, "fp6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf6", true, "fp6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf6", true, "fp6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf6", true, "fp6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf6", true, "fp6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf6", true, "fp6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf6", true, "bf6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "bf6", true, "bf6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "bf6", true, "bf6", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf6", true, "bf6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf6", true, "bf6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf6", true, "bf6", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf6", true, "bf6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf6", true, "bf6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "bf6", true, "fp4", "e5m3", "e8m0"}, + {NULL, "gfx1250", "bf6", true, "fp4", "e5m3", "e5m3"}, + {NULL, "gfx1250", "bf6", true, "fp4", "e5m3", "e4m3"}, + {NULL, "gfx1250", "bf6", true, "fp4", "e4m3", "e8m0"}, + {NULL, "gfx1250", "bf6", true, "fp4", "e4m3", "e5m3"}, + {NULL, "gfx1250", "bf6", true, "fp4", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "fp8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp4", true, "fp8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "fp8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp4", true, "fp8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "fp8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp4", true, "fp8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "bf8", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp4", true, "bf8", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "bf8", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp4", true, "bf8", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "bf8", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp4", true, "bf8", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "fp6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp4", true, "fp6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "fp6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp4", true, "fp6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "fp6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp4", true, "fp6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "bf6", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp4", true, "bf6", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "bf6", "e5m3", "e5m3"}, + {NULL, "gfx1250", "fp4", true, "bf6", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "bf6", "e4m3", "e5m3"}, + {NULL, "gfx1250", "fp4", true, "bf6", "e4m3", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "fp4", "e8m0", "e5m3"}, + {NULL, "gfx1250", "fp4", true, "fp4", "e8m0", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "fp4", "e5m3", "e8m0"}, + {NULL, "gfx1250", "fp4", true, "fp4", "e5m3", "e4m3"}, + {NULL, "gfx1250", "fp4", true, "fp4", "e4m3", "e8m0"}, + {NULL, "gfx1250", "fp4", true, "fp4", "e4m3", "e5m3"}, }; static const int NUM_CONFIGS = (int)(sizeof(CONFIGS) / sizeof(CONFIGS[0])); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py index e2b4ddd16522..768076b5beea 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py @@ -347,6 +347,196 @@ def build(b): ) +CONFIGS.extend( + (_wmma_scaled(a, b, mode, sa, sb), "gfx1250") + for mode in ("scale", "scale16") + for a, b, sa, sb in [ + ("fp8", "fp8", "e8m0", "e5m3"), + ("fp8", "fp8", "e8m0", "e4m3"), + ("fp8", "fp8", "e5m3", "e8m0"), + ("fp8", "fp8", "e5m3", "e5m3"), + ("fp8", "fp8", "e5m3", "e4m3"), + ("fp8", "fp8", "e4m3", "e8m0"), + ("fp8", "fp8", "e4m3", "e5m3"), + ("fp8", "fp8", "e4m3", "e4m3"), + ("fp8", "bf8", "e8m0", "e5m3"), + ("fp8", "bf8", "e8m0", "e4m3"), + ("fp8", "bf8", "e5m3", "e8m0"), + ("fp8", "bf8", "e5m3", "e5m3"), + ("fp8", "bf8", "e5m3", "e4m3"), + ("fp8", "bf8", "e4m3", "e8m0"), + ("fp8", "bf8", "e4m3", "e5m3"), + ("fp8", "bf8", "e4m3", "e4m3"), + ("fp8", "fp6", "e8m0", "e5m3"), + ("fp8", "fp6", "e8m0", "e4m3"), + ("fp8", "fp6", "e5m3", "e8m0"), + ("fp8", "fp6", "e5m3", "e5m3"), + ("fp8", "fp6", "e5m3", "e4m3"), + ("fp8", "fp6", "e4m3", "e8m0"), + ("fp8", "fp6", "e4m3", "e5m3"), + ("fp8", "fp6", "e4m3", "e4m3"), + ("fp8", "bf6", "e8m0", "e5m3"), + ("fp8", "bf6", "e8m0", "e4m3"), + ("fp8", "bf6", "e5m3", "e8m0"), + ("fp8", "bf6", "e5m3", "e5m3"), + ("fp8", "bf6", "e5m3", "e4m3"), + ("fp8", "bf6", "e4m3", "e8m0"), + ("fp8", "bf6", "e4m3", "e5m3"), + ("fp8", "bf6", "e4m3", "e4m3"), + ("fp8", "fp4", "e5m3", "e8m0"), + ("fp8", "fp4", "e5m3", "e5m3"), + ("fp8", "fp4", "e5m3", "e4m3"), + ("fp8", "fp4", "e4m3", "e8m0"), + ("fp8", "fp4", "e4m3", "e5m3"), + ("fp8", "fp4", "e4m3", "e4m3"), + ("bf8", "fp8", "e8m0", "e5m3"), + ("bf8", "fp8", "e8m0", "e4m3"), + ("bf8", "fp8", "e5m3", "e8m0"), + ("bf8", "fp8", "e5m3", "e5m3"), + ("bf8", "fp8", "e5m3", "e4m3"), + ("bf8", "fp8", "e4m3", "e8m0"), + ("bf8", "fp8", "e4m3", "e5m3"), + ("bf8", "fp8", "e4m3", "e4m3"), + ("bf8", "bf8", "e8m0", "e5m3"), + ("bf8", "bf8", "e8m0", "e4m3"), + ("bf8", "bf8", "e5m3", "e8m0"), + ("bf8", "bf8", "e5m3", "e5m3"), + ("bf8", "bf8", "e5m3", "e4m3"), + ("bf8", "bf8", "e4m3", "e8m0"), + ("bf8", "bf8", "e4m3", "e5m3"), + ("bf8", "bf8", "e4m3", "e4m3"), + ("bf8", "fp6", "e8m0", "e5m3"), + ("bf8", "fp6", "e8m0", "e4m3"), + ("bf8", "fp6", "e5m3", "e8m0"), + ("bf8", "fp6", "e5m3", "e5m3"), + ("bf8", "fp6", "e5m3", "e4m3"), + ("bf8", "fp6", "e4m3", "e8m0"), + ("bf8", "fp6", "e4m3", "e5m3"), + ("bf8", "fp6", "e4m3", "e4m3"), + ("bf8", "bf6", "e8m0", "e5m3"), + ("bf8", "bf6", "e8m0", "e4m3"), + ("bf8", "bf6", "e5m3", "e8m0"), + ("bf8", "bf6", "e5m3", "e5m3"), + ("bf8", "bf6", "e5m3", "e4m3"), + ("bf8", "bf6", "e4m3", "e8m0"), + ("bf8", "bf6", "e4m3", "e5m3"), + ("bf8", "bf6", "e4m3", "e4m3"), + ("bf8", "fp4", "e5m3", "e8m0"), + ("bf8", "fp4", "e5m3", "e5m3"), + ("bf8", "fp4", "e5m3", "e4m3"), + ("bf8", "fp4", "e4m3", "e8m0"), + ("bf8", "fp4", "e4m3", "e5m3"), + ("bf8", "fp4", "e4m3", "e4m3"), + ("fp6", "fp8", "e8m0", "e5m3"), + ("fp6", "fp8", "e8m0", "e4m3"), + ("fp6", "fp8", "e5m3", "e8m0"), + ("fp6", "fp8", "e5m3", "e5m3"), + ("fp6", "fp8", "e5m3", "e4m3"), + ("fp6", "fp8", "e4m3", "e8m0"), + ("fp6", "fp8", "e4m3", "e5m3"), + ("fp6", "fp8", "e4m3", "e4m3"), + ("fp6", "bf8", "e8m0", "e5m3"), + ("fp6", "bf8", "e8m0", "e4m3"), + ("fp6", "bf8", "e5m3", "e8m0"), + ("fp6", "bf8", "e5m3", "e5m3"), + ("fp6", "bf8", "e5m3", "e4m3"), + ("fp6", "bf8", "e4m3", "e8m0"), + ("fp6", "bf8", "e4m3", "e5m3"), + ("fp6", "bf8", "e4m3", "e4m3"), + ("fp6", "fp6", "e8m0", "e5m3"), + ("fp6", "fp6", "e8m0", "e4m3"), + ("fp6", "fp6", "e5m3", "e8m0"), + ("fp6", "fp6", "e5m3", "e5m3"), + ("fp6", "fp6", "e5m3", "e4m3"), + ("fp6", "fp6", "e4m3", "e8m0"), + ("fp6", "fp6", "e4m3", "e5m3"), + ("fp6", "fp6", "e4m3", "e4m3"), + ("fp6", "bf6", "e8m0", "e5m3"), + ("fp6", "bf6", "e8m0", "e4m3"), + ("fp6", "bf6", "e5m3", "e8m0"), + ("fp6", "bf6", "e5m3", "e5m3"), + ("fp6", "bf6", "e5m3", "e4m3"), + ("fp6", "bf6", "e4m3", "e8m0"), + ("fp6", "bf6", "e4m3", "e5m3"), + ("fp6", "bf6", "e4m3", "e4m3"), + ("fp6", "fp4", "e5m3", "e8m0"), + ("fp6", "fp4", "e5m3", "e5m3"), + ("fp6", "fp4", "e5m3", "e4m3"), + ("fp6", "fp4", "e4m3", "e8m0"), + ("fp6", "fp4", "e4m3", "e5m3"), + ("fp6", "fp4", "e4m3", "e4m3"), + ("bf6", "fp8", "e8m0", "e5m3"), + ("bf6", "fp8", "e8m0", "e4m3"), + ("bf6", "fp8", "e5m3", "e8m0"), + ("bf6", "fp8", "e5m3", "e5m3"), + ("bf6", "fp8", "e5m3", "e4m3"), + ("bf6", "fp8", "e4m3", "e8m0"), + ("bf6", "fp8", "e4m3", "e5m3"), + ("bf6", "fp8", "e4m3", "e4m3"), + ("bf6", "bf8", "e8m0", "e5m3"), + ("bf6", "bf8", "e8m0", "e4m3"), + ("bf6", "bf8", "e5m3", "e8m0"), + ("bf6", "bf8", "e5m3", "e5m3"), + ("bf6", "bf8", "e5m3", "e4m3"), + ("bf6", "bf8", "e4m3", "e8m0"), + ("bf6", "bf8", "e4m3", "e5m3"), + ("bf6", "bf8", "e4m3", "e4m3"), + ("bf6", "fp6", "e8m0", "e5m3"), + ("bf6", "fp6", "e8m0", "e4m3"), + ("bf6", "fp6", "e5m3", "e8m0"), + ("bf6", "fp6", "e5m3", "e5m3"), + ("bf6", "fp6", "e5m3", "e4m3"), + ("bf6", "fp6", "e4m3", "e8m0"), + ("bf6", "fp6", "e4m3", "e5m3"), + ("bf6", "fp6", "e4m3", "e4m3"), + ("bf6", "bf6", "e8m0", "e5m3"), + ("bf6", "bf6", "e8m0", "e4m3"), + ("bf6", "bf6", "e5m3", "e8m0"), + ("bf6", "bf6", "e5m3", "e5m3"), + ("bf6", "bf6", "e5m3", "e4m3"), + ("bf6", "bf6", "e4m3", "e8m0"), + ("bf6", "bf6", "e4m3", "e5m3"), + ("bf6", "bf6", "e4m3", "e4m3"), + ("bf6", "fp4", "e5m3", "e8m0"), + ("bf6", "fp4", "e5m3", "e5m3"), + ("bf6", "fp4", "e5m3", "e4m3"), + ("bf6", "fp4", "e4m3", "e8m0"), + ("bf6", "fp4", "e4m3", "e5m3"), + ("bf6", "fp4", "e4m3", "e4m3"), + ("fp4", "fp8", "e8m0", "e5m3"), + ("fp4", "fp8", "e8m0", "e4m3"), + ("fp4", "fp8", "e5m3", "e5m3"), + ("fp4", "fp8", "e5m3", "e4m3"), + ("fp4", "fp8", "e4m3", "e5m3"), + ("fp4", "fp8", "e4m3", "e4m3"), + ("fp4", "bf8", "e8m0", "e5m3"), + ("fp4", "bf8", "e8m0", "e4m3"), + ("fp4", "bf8", "e5m3", "e5m3"), + ("fp4", "bf8", "e5m3", "e4m3"), + ("fp4", "bf8", "e4m3", "e5m3"), + ("fp4", "bf8", "e4m3", "e4m3"), + ("fp4", "fp6", "e8m0", "e5m3"), + ("fp4", "fp6", "e8m0", "e4m3"), + ("fp4", "fp6", "e5m3", "e5m3"), + ("fp4", "fp6", "e5m3", "e4m3"), + ("fp4", "fp6", "e4m3", "e5m3"), + ("fp4", "fp6", "e4m3", "e4m3"), + ("fp4", "bf6", "e8m0", "e5m3"), + ("fp4", "bf6", "e8m0", "e4m3"), + ("fp4", "bf6", "e5m3", "e5m3"), + ("fp4", "bf6", "e5m3", "e4m3"), + ("fp4", "bf6", "e4m3", "e5m3"), + ("fp4", "bf6", "e4m3", "e4m3"), + ("fp4", "fp4", "e8m0", "e5m3"), + ("fp4", "fp4", "e8m0", "e4m3"), + ("fp4", "fp4", "e5m3", "e8m0"), + ("fp4", "fp4", "e5m3", "e4m3"), + ("fp4", "fp4", "e4m3", "e8m0"), + ("fp4", "fp4", "e4m3", "e5m3"), + ] +) + + def _spec(idx: int): """Config selector: the (builder, arch) pair the shared driver expects.""" if not 0 <= idx < len(CONFIGS): diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py index a3501500790c..543be13acfa7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py @@ -14,7 +14,7 @@ @pytest.mark.parametrize( - "a,b,sa,sb", [("fp4", "fp4", "e4m3", "e4m3"), ("fp6", "fp4", "e8m0", "e5m3")] + "a,b,sa,sb", [("fp4", "fp4", "e4m3", "e5m3"), ("fp6", "fp8", "e4m3", "e5m3")] ) @pytest.mark.parametrize("matrix_path", ["wmma_scale", "wmma_scale16"]) @pytest.mark.parametrize("output_dtype", ["bf16", "fp16"]) @@ -134,3 +134,21 @@ def test_scale_formats_gemm_cli_rejects_invalid_spec_before_verify( err = capsys.readouterr().err assert "M/N/K must be positive" in err assert "Traceback" not in err + + +@pytest.mark.parametrize("dtype", ["fp8", "bf8", "fp6", "bf6", "fp4"]) +@pytest.mark.parametrize("operand", ["a", "b"]) +def test_scale_formats_gemm_cli_accepts_alternate_scales(monkeypatch, dtype, operand): + from rocke.examples.gfx1250.gemm import scale_formats_gemm as example + + calls = [] + + def verify(spec, args): + assert is_valid_spec(spec)[0] + calls.append(spec) + return 0 + + monkeypatch.setattr(example, "verify", verify) + assert example.main([f"--dtype-{operand}", dtype]) == 0 + assert len(calls) == 1 + assert calls[0].resolved_scale_dtypes() == ("e4m3", "e4m3") diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py index 268a150e7e2c..10e6688fb3de 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py @@ -1,7 +1,8 @@ # Copyright (c) Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT -"""gfx1250 independent-scale numeric fixtures; distinct from the planned probe.""" +"""gfx1250 independent-scale fixtures and fixed-byte selector checks.""" +from itertools import product import subprocess import sys @@ -9,13 +10,15 @@ _MODULE = "rocke.examples.gfx1250.gemm.scale_formats_gemm" -# Additional scale encodings through COMGR, with representative HIP cases. +# Independent scale encodings through both COMGR and HIP. _FORMATS = ("fp8e4m3", "bf8e5m2", "fp6", "bf6", "fp4") -_MIXED = ( - [(a, "fp4", "e8m0", scale) for a in _FORMATS[:-1] for scale in ("e5m3", "e4m3")] - + [("fp4", b, scale, "e8m0") for b in _FORMATS[:-1] for scale in ("e5m3", "e4m3")] - + [("fp4", "fp4", scale, scale) for scale in ("e5m3", "e4m3")] -) +_MIXED = [ + (a, b, sa, sb) + for a, b, sa, sb in product( + _FORMATS, _FORMATS, ("e8m0", "e5m3", "e4m3"), ("e8m0", "e5m3", "e4m3") + ) + if (sa, sb) != ("e8m0", "e8m0") +] @pytest.mark.parametrize("matrix_path", ["wmma_scale", "wmma_scale16"]) @@ -69,3 +72,81 @@ def test_scale_format_wmma_numeric( assert f"PASS: verified {count} cases" in output, output assert output.count("bad=0") == count, output print(output, end="") + + +@pytest.mark.parametrize("dtype", _FORMATS) +@pytest.mark.parametrize("operand", ["a", "b"]) +@pytest.mark.parametrize("block_k", [32, 16]) +@pytest.mark.parametrize("route", ["comgr", "hip"]) +def test_scale_selector_fixed_bytes_numeric(gpu_env, dtype, operand, block_k, route): + """Hold encoded inputs fixed while changing the tested operand's selector.""" + import numpy as np + import ml_dtypes + + from rocke.examples.gfx1250.gemm.block_scaled_gemm_verify import ( + _launch, + check_result, + pack_fp6_codes, + ) + from rocke.helpers import compile_kernel + from rocke.helpers.compile import compile_kernel_via_hipcc + from rocke.instances.gfx1250.block_scaled_gemm import ( + BlockScaledGemmSpec, + build_block_scaled_gemm, + ) + from rocke.runtime.hip_module import Runtime + + unity = {"e8m0": 0x7F, "e5m3": 0x78, "e4m3": 0x38} + # Literal predictions avoid the production and verifier scale decoders. + predictions = { + 0x3C: {"e8m0": 2.0**-67, "e5m3": 3 / 512, "e4m3": 1.5}, + 0x7C: {"e8m0": 0.125, "e5m3": 1.5, "e4m3": 384.0}, + } + positions = (0, 15, 16, 31, 32, 63, 64, 95, 96, 127) + + def matrix(kind, k): + codes = np.zeros((16, 128), dtype=np.uint8) + if kind == "fp4": + codes[:, k] = 2 + return codes[:, ::2] | (codes[:, 1::2] << 4) + if kind in ("fp6", "bf6"): + codes[:, k] = 8 if kind == "fp6" else 12 + return pack_fp6_codes(codes) + codes[:, k] = 0x38 if kind == "fp8e4m3" else 0x3C + return codes + + rt = Runtime() + for scale in unity: + da, db = (dtype, "fp4") if operand == "a" else ("fp4", dtype) + sa, sb = (scale, "e8m0") if operand == "a" else ("e8m0", scale) + spec = BlockScaledGemmSpec( + name="fixed_scale_bytes", + M=16, + N=16, + K=128, + dtype_a=da, + dtype_b=db, + scale_dtype="e8m0", + scale_dtype_a=sa, + scale_dtype_b=sb, + block_k=block_k, + matrix_path="wmma_scale16" if block_k == 16 else "wmma_scale", + ) + compiler = compile_kernel if route == "comgr" else compile_kernel_via_hipcc + artifact = compiler(build_block_scaled_gemm(spec), arch="gfx1250") + module = rt.load_module(artifact.hsaco) + try: + fn = module.get_function(artifact.kernel_name) + for k, code in product(positions, predictions): + a, b = matrix(da, k), matrix(db, k) + a_scales = np.full((16, 128 // block_k), unity[sa], dtype=np.uint8) + b_scales = np.full((128 // block_k, 16), unity[sb], dtype=np.uint8) + # Every tested scale byte is identical across selector variants. + (a_scales if operand == "a" else b_scales).fill(code) + got = _launch(rt, fn, spec, (a, b, a_scales, b_scales)) + expected = np.full( + (16, 16), predictions[code][scale], dtype=ml_dtypes.bfloat16 + ).astype(np.float32) + check_result(got, expected, exact=True) + finally: + module.unload() From 738b6ff6f0a5ae963e32ba992e27b9332a1ccc8b Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:18:57 -0400 Subject: [PATCH 26/32] fix(rocke): share differential configuration limits --- .../rocke/examples/gfx1250/gemm/README.md | 2 +- .../differential/ir_artifact_diff.py | 3 +- .../tests/instances/differential/run_diff.py | 8 +- .../differential/test_gate_verdict.py | 109 ++++++++++++++++-- 4 files changed, 107 insertions(+), 15 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md index 402dca910c00..34197e630dcc 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md @@ -15,7 +15,7 @@ LLVM 23 ROCm libraries, NumPy, and `ml_dtypes`. Torch is optional. | `mxfp6_gemm` | Packed FP6 E2M3 or E3M2 on both operands | E8M0 | | `mxfp4_gemm` | Packed FP4 E2M1 on both operands | E8M0 | | [`mixed_scaled_gemm`](mixed_scaled_gemm.py) | Different A/B formats from FP8, BF8, FP6, BF6, and FP4 | E8M0 | -| [`scale_formats_gemm`](scale_formats_gemm.py) | FP4 alone or mixed with FP8/BF8/FP6/BF6 | E8M0, E4M3, or E5M3, subject to operand legality | +| [`scale_formats_gemm`](scale_formats_gemm.py) | Any A/B pair drawn from FP8/BF8/FP6/BF6/FP4 | Independent E8M0, E4M3, or E5M3 scales for each operand | ```sh ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.mxfp8_gemm diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/ir_artifact_diff.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/ir_artifact_diff.py index 8e884bbaa691..db53e9f5ce24 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/ir_artifact_diff.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/ir_artifact_diff.py @@ -42,7 +42,6 @@ PARITY = run_diff.PARITY PYROOT = run_diff.PYROOT -MAX_CFG = run_diff.MAX_CFG TIMEOUT = run_diff.TIMEOUT # Arch derivation. A family whose name starts with one of these prefixes lowers @@ -206,7 +205,7 @@ def lower_artifact(cli, family, ir_text, ll_ref): def run_family(cli, family): rows = [] - for idx in range(MAX_CFG): + for idx in range(run_diff.config_limit(family)): pr_ir, ir_text, ir_err = run_py(family, idx, "ir") pr_ll, ll_ref, ll_err = run_py(family, idx, "ll") diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/run_diff.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/run_diff.py index 7fae29416d88..fd1484273224 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/run_diff.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/run_diff.py @@ -92,6 +92,12 @@ def collect_ref_shas(results): FAMILY_MAX_CFG = {"gfx1250_lowering": 512} TIMEOUT = 120 # seconds per emitter invocation + +def config_limit(family: str) -> int: + """Share enumeration caps so every differential path covers the same configs.""" + return FAMILY_MAX_CFG.get(family, MAX_CFG) + + _CANON = None @@ -328,7 +334,7 @@ def run_family(name, archive, mode, canonical=False, src_dir=None): } configs = [] range_drift = None - for idx in range(FAMILY_MAX_CFG.get(name, MAX_CFG)): + for idx in range(config_limit(name)): cr, co, ce = run_c(binpath, idx, mode) pr, po, pe = run_py(name, idx, mode, src_dir=src_dir) # mode not supported by this emitter -> stop trying this mode diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py index fad536f6e0cf..8206980716a2 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py @@ -33,6 +33,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import run_diff # noqa: E402 -- after sys.path shim +import ir_artifact_diff # noqa: E402 from run_diff import ( # noqa: E402 DRIFT_VERDICTS, GATE_FAIL, @@ -340,17 +341,103 @@ def test_discovering_no_families_exits_nonzero(self): self.assertEqual(self.drive_main(None, Path(d)), 1) -if __name__ == "__main__": - unittest.main() +class TestConfigEnumeration(unittest.TestCase): + """Both differential paths must visit the expanded catalog and its end.""" + + @classmethod + def setUpClass(cls): + import importlib.util + + source = ( + Path(__file__).resolve().parents[1] / "parity" / "gfx1250_lowering_emit.py" + ) + spec = importlib.util.spec_from_file_location( + "gfx1250_lowering_configs", source + ) + emitter = importlib.util.module_from_spec(spec) + with mock.patch.object(sys, "path", [str(source.parent), *sys.path]): + spec.loader.exec_module(emitter) + cls.config_count = len(emitter.CONFIGS) + + def drive_family(self, driver, mismatch=None): + def emit(idx, mode, native=False): + if idx >= self.config_count: + return 1, b"", "unknown config index" + prefix = "ir" if mode == "ir" else "ll" + if native and idx == mismatch: + prefix = "different" + return 0, f"{prefix}-{idx}".encode(), "" + + if driver == "builders": + with ( + mock.patch.object( + run_diff, "compile_c", return_value=(Path("emit"), "") + ), + mock.patch.object( + run_diff, + "run_c", + side_effect=lambda binary, idx, mode: emit(idx, mode, True), + ), + mock.patch.object( + run_diff, + "run_py", + side_effect=lambda family, idx, mode, **kw: emit(idx, mode), + ) as py, + ): + result = run_diff.run_family("gfx1250_lowering", Path("archive"), "ll") + return ( + result["configs"], + result["status"], + [c.args[1] for c in py.call_args_list], + ) + def lower(cli, family, ir, ref): + idx = int(ir.split(b"-")[1]) + out = b"different" if idx == mismatch else ref + return 0, out, "", "gfx1250" + + with ( + mock.patch.object( + ir_artifact_diff, + "run_py", + side_effect=lambda family, idx, mode: emit(idx, mode), + ) as py, + mock.patch.object(ir_artifact_diff, "lower_artifact", side_effect=lower), + ): + rows = ir_artifact_diff.run_family(Path("cli"), "gfx1250_lowering") + indices = [c.args[1] for c in py.call_args_list if c.args[2] == "ll"] + return rows, ir_artifact_diff.family_status(rows)[0], indices + + def test_both_drivers_visit_all_configs_and_stop_at_end(self): + self.assertEqual(self.config_count, 472) + for driver, status in (("builders", "GREEN"), ("artifacts", "IDENTICAL")): + with self.subTest(driver=driver): + rows, actual_status, indices = self.drive_family(driver) + self.assertEqual(len(rows), self.config_count) + self.assertEqual(indices, list(range(self.config_count + 1))) + self.assertEqual(actual_status, status) + + def test_both_drivers_detect_drift_beyond_default_cap(self): + late = run_diff.MAX_CFG + for driver in ("builders", "artifacts"): + with self.subTest(driver=driver): + rows, status, _ = self.drive_family(driver, mismatch=late) + self.assertEqual(status, "DRIFT") + self.assertIn(rows[late]["verdict"], ("MISMATCH", "DRIFT")) + + def test_default_cap_would_miss_late_drift(self): + self.assertEqual(run_diff.config_limit("gemm"), 128) + for driver, status in (("builders", "GREEN"), ("artifacts", "IDENTICAL")): + with ( + self.subTest(driver=driver), + mock.patch.object(run_diff, "FAMILY_MAX_CFG", {}), + ): + rows, actual_status, _ = self.drive_family( + driver, mismatch=run_diff.MAX_CFG + ) + self.assertEqual(len(rows), run_diff.MAX_CFG) + self.assertEqual(actual_status, status) -def test_gfx1250_scale_catalog_configs_are_not_truncated(): - import importlib.util - source = Path(__file__).resolve().parents[1] / "parity" / "gfx1250_lowering_emit.py" - spec = importlib.util.spec_from_file_location("gfx1250_lowering_configs", source) - emitter = importlib.util.module_from_spec(spec) - with mock.patch.object(sys, "path", [str(source.parent), *sys.path]): - spec.loader.exec_module(emitter) - assert len(emitter.CONFIGS) == 472 - assert run_diff.FAMILY_MAX_CFG["gfx1250_lowering"] > len(emitter.CONFIGS) +if __name__ == "__main__": + unittest.main() From 0bba82653c932c374c286212340fb7c70988904e Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:43:09 -0400 Subject: [PATCH 27/32] refactor(rocke): generate scale-format parity configurations --- .../differential/test_gate_verdict.py | 18 + .../instances/parity/gfx1250_lowering_emit.c | 477 +++--------------- .../instances/parity/gfx1250_lowering_emit.py | 226 +-------- 3 files changed, 101 insertions(+), 620 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py index 8206980716a2..3e836b70b7bf 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py @@ -21,6 +21,7 @@ from __future__ import annotations +import hashlib import json import sys import unittest @@ -357,8 +358,25 @@ def setUpClass(cls): emitter = importlib.util.module_from_spec(spec) with mock.patch.object(sys, "path", [str(source.parent), *sys.path]): spec.loader.exec_module(emitter) + cls.emitter = emitter cls.config_count = len(emitter.CONFIGS) + def test_scale_pair_order_matches_explicit_tables(self): + # Fingerprint captured from the explicit tuples before loop generation. + # The order is observable: both emitters select kernels by case index. + legacy = self.emitter._LEGACY_SCALE_PAIRS + remaining = self.emitter._OTHER_SCALE_PAIRS + self.assertEqual((len(legacy), len(remaining)), (18, 182)) + pairs = legacy + remaining + self.assertEqual(len(set(pairs)), 200) + fingerprint = hashlib.sha256( + json.dumps(pairs, separators=(",", ":")).encode() + ).hexdigest() + self.assertEqual( + fingerprint, + "2e67cba9ddce17e2d13ff93f25a8492085b1eac54d9daa464d1931f48392f9a4", + ) + def drive_family(self, driver, mismatch=None): def emit(idx, mode, native=False): if idx >= self.config_count: diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c index 339cb61dae30..25b87dee4ffb 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c @@ -188,7 +188,8 @@ static void build_wmma_scale16_fp4(rocke_ir_builder_t* b) /* ds_read_b128_tr_b16. gfx950 has one type-agnostic opcode returning * <8 x i16> that the handler reinterprets; gfx1250 has per-element-type - * opcodes (.v8f16 / .v8bf16) that land in the right type with no reinterpret. */ + * opcodes (.v8f16 / .v8bf16) that land in the right type with no reinterpret. + */ static void tr16_b128(rocke_ir_builder_t* b, const rocke_type_t* elem) { rocke_param_opts_t o; @@ -413,7 +414,7 @@ typedef struct config /* Each gfx1250 config that tests a *choice* of encoding is followed by its * gfx950 twin, so the pair pins both branches. Index-for-index with the Python * emitter's CONFIGS list. */ -static const config_t CONFIGS[] = { +static const config_t BASE_CONFIGS[] = { {build_wmma_k32_f16, "gfx1250"}, {build_wmma_k32_bf16, "gfx1250"}, {build_wmma_k64_fp8_fp8, "gfx1250"}, @@ -486,409 +487,60 @@ static const config_t CONFIGS[] = { {NULL, "gfx1250", "fp4", true, "bf8"}, {NULL, "gfx1250", "fp4", true, "fp6"}, {NULL, "gfx1250", "fp4", true, "bf6"}, - {NULL, "gfx1250", "fp8", false, "fp4", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp8", false, "fp4", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf8", false, "fp4", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf8", false, "fp4", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp6", false, "fp4", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp6", false, "fp4", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf6", false, "fp4", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf6", false, "fp4", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "fp8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp4", false, "fp8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp4", false, "bf8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp4", false, "bf8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp4", false, "fp6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp4", false, "fp6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp4", false, "bf6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp4", false, "bf6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp4", false, "fp4", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp4", false, "fp4", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp8", true, "fp4", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "fp4", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf8", true, "fp4", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf8", true, "fp4", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp6", true, "fp4", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp6", true, "fp4", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf6", true, "fp4", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf6", true, "fp4", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "fp8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp4", true, "fp8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp4", true, "bf8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp4", true, "bf8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp4", true, "fp6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp4", true, "fp6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp4", true, "bf6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp4", true, "bf6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp4", true, "fp4", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp4", true, "fp4", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp8", false, "fp8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp8", false, "fp8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp8", false, "fp8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp8", false, "fp8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp8", false, "fp8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp8", false, "fp8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp8", false, "fp8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp8", false, "fp8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp8", false, "bf8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp8", false, "bf8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp8", false, "bf8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp8", false, "bf8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp8", false, "bf8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp8", false, "bf8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp8", false, "bf8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp8", false, "bf8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp8", false, "fp6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp8", false, "fp6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp8", false, "fp6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp8", false, "fp6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp8", false, "fp6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp8", false, "fp6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp8", false, "fp6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp8", false, "fp6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp8", false, "bf6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp8", false, "bf6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp8", false, "bf6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp8", false, "bf6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp8", false, "bf6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp8", false, "bf6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp8", false, "bf6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp8", false, "bf6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp8", false, "fp4", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp8", false, "fp4", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp8", false, "fp4", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp8", false, "fp4", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp8", false, "fp4", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp8", false, "fp4", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf8", false, "fp8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf8", false, "fp8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf8", false, "fp8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf8", false, "fp8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf8", false, "fp8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf8", false, "fp8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf8", false, "fp8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf8", false, "fp8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf8", false, "bf8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf8", false, "bf8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf8", false, "bf8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf8", false, "bf8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf8", false, "bf8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf8", false, "bf8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf8", false, "bf8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf8", false, "bf8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf8", false, "fp6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf8", false, "fp6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf8", false, "fp6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf8", false, "fp6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf8", false, "fp6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf8", false, "fp6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf8", false, "fp6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf8", false, "fp6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf8", false, "bf6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf8", false, "bf6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf8", false, "bf6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf8", false, "bf6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf8", false, "bf6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf8", false, "bf6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf8", false, "bf6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf8", false, "bf6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf8", false, "fp4", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf8", false, "fp4", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf8", false, "fp4", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf8", false, "fp4", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf8", false, "fp4", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf8", false, "fp4", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp6", false, "fp8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp6", false, "fp8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp6", false, "fp8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp6", false, "fp8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp6", false, "fp8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp6", false, "fp8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp6", false, "fp8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp6", false, "fp8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp6", false, "bf8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp6", false, "bf8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp6", false, "bf8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp6", false, "bf8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp6", false, "bf8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp6", false, "bf8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp6", false, "bf8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp6", false, "bf8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp6", false, "fp6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp6", false, "fp6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp6", false, "fp6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp6", false, "fp6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp6", false, "fp6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp6", false, "fp6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp6", false, "fp6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp6", false, "fp6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp6", false, "bf6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp6", false, "bf6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp6", false, "bf6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp6", false, "bf6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp6", false, "bf6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp6", false, "bf6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp6", false, "bf6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp6", false, "bf6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp6", false, "fp4", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp6", false, "fp4", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp6", false, "fp4", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp6", false, "fp4", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp6", false, "fp4", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp6", false, "fp4", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf6", false, "fp8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf6", false, "fp8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf6", false, "fp8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf6", false, "fp8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf6", false, "fp8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf6", false, "fp8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf6", false, "fp8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf6", false, "fp8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf6", false, "bf8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf6", false, "bf8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf6", false, "bf8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf6", false, "bf8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf6", false, "bf8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf6", false, "bf8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf6", false, "bf8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf6", false, "bf8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf6", false, "fp6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf6", false, "fp6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf6", false, "fp6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf6", false, "fp6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf6", false, "fp6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf6", false, "fp6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf6", false, "fp6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf6", false, "fp6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf6", false, "bf6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf6", false, "bf6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf6", false, "bf6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf6", false, "bf6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf6", false, "bf6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf6", false, "bf6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf6", false, "bf6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf6", false, "bf6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf6", false, "fp4", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf6", false, "fp4", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf6", false, "fp4", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf6", false, "fp4", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf6", false, "fp4", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf6", false, "fp4", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "fp8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp4", false, "fp8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "fp8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp4", false, "fp8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "fp8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp4", false, "fp8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "bf8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp4", false, "bf8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "bf8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp4", false, "bf8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "bf8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp4", false, "bf8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "fp6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp4", false, "fp6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "fp6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp4", false, "fp6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "fp6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp4", false, "fp6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "bf6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp4", false, "bf6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "bf6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp4", false, "bf6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "bf6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp4", false, "bf6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "fp4", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp4", false, "fp4", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "fp4", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp4", false, "fp4", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp4", false, "fp4", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp4", false, "fp4", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "fp8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "fp8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp8", true, "fp8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp8", true, "fp8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "fp8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp8", true, "fp8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp8", true, "fp8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "fp8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp8", true, "bf8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "bf8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp8", true, "bf8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp8", true, "bf8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "bf8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp8", true, "bf8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp8", true, "bf8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "bf8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp8", true, "fp6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "fp6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp8", true, "fp6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp8", true, "fp6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "fp6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp8", true, "fp6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp8", true, "fp6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "fp6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp8", true, "bf6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "bf6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp8", true, "bf6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp8", true, "bf6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "bf6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp8", true, "bf6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp8", true, "bf6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "bf6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp8", true, "fp4", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp8", true, "fp4", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "fp4", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp8", true, "fp4", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp8", true, "fp4", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp8", true, "fp4", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf8", true, "fp8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf8", true, "fp8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf8", true, "fp8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf8", true, "fp8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf8", true, "fp8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf8", true, "fp8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf8", true, "fp8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf8", true, "fp8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf8", true, "bf8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf8", true, "bf8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf8", true, "bf8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf8", true, "bf8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf8", true, "bf8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf8", true, "bf8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf8", true, "bf8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf8", true, "bf8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf8", true, "fp6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf8", true, "fp6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf8", true, "fp6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf8", true, "fp6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf8", true, "fp6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf8", true, "fp6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf8", true, "fp6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf8", true, "fp6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf8", true, "bf6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf8", true, "bf6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf8", true, "bf6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf8", true, "bf6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf8", true, "bf6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf8", true, "bf6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf8", true, "bf6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf8", true, "bf6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf8", true, "fp4", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf8", true, "fp4", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf8", true, "fp4", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf8", true, "fp4", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf8", true, "fp4", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf8", true, "fp4", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp6", true, "fp8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp6", true, "fp8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp6", true, "fp8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp6", true, "fp8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp6", true, "fp8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp6", true, "fp8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp6", true, "fp8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp6", true, "fp8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp6", true, "bf8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp6", true, "bf8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp6", true, "bf8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp6", true, "bf8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp6", true, "bf8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp6", true, "bf8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp6", true, "bf8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp6", true, "bf8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp6", true, "fp6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp6", true, "fp6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp6", true, "fp6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp6", true, "fp6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp6", true, "fp6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp6", true, "fp6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp6", true, "fp6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp6", true, "fp6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp6", true, "bf6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp6", true, "bf6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp6", true, "bf6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp6", true, "bf6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp6", true, "bf6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp6", true, "bf6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp6", true, "bf6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp6", true, "bf6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp6", true, "fp4", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp6", true, "fp4", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp6", true, "fp4", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp6", true, "fp4", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp6", true, "fp4", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp6", true, "fp4", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf6", true, "fp8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf6", true, "fp8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf6", true, "fp8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf6", true, "fp8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf6", true, "fp8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf6", true, "fp8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf6", true, "fp8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf6", true, "fp8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf6", true, "bf8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf6", true, "bf8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf6", true, "bf8", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf6", true, "bf8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf6", true, "bf8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf6", true, "bf8", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf6", true, "bf8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf6", true, "bf8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf6", true, "fp6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf6", true, "fp6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf6", true, "fp6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf6", true, "fp6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf6", true, "fp6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf6", true, "fp6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf6", true, "fp6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf6", true, "fp6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf6", true, "bf6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "bf6", true, "bf6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "bf6", true, "bf6", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf6", true, "bf6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf6", true, "bf6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf6", true, "bf6", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf6", true, "bf6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf6", true, "bf6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "bf6", true, "fp4", "e5m3", "e8m0"}, - {NULL, "gfx1250", "bf6", true, "fp4", "e5m3", "e5m3"}, - {NULL, "gfx1250", "bf6", true, "fp4", "e5m3", "e4m3"}, - {NULL, "gfx1250", "bf6", true, "fp4", "e4m3", "e8m0"}, - {NULL, "gfx1250", "bf6", true, "fp4", "e4m3", "e5m3"}, - {NULL, "gfx1250", "bf6", true, "fp4", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "fp8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp4", true, "fp8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "fp8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp4", true, "fp8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "fp8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp4", true, "fp8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "bf8", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp4", true, "bf8", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "bf8", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp4", true, "bf8", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "bf8", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp4", true, "bf8", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "fp6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp4", true, "fp6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "fp6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp4", true, "fp6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "fp6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp4", true, "fp6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "bf6", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp4", true, "bf6", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "bf6", "e5m3", "e5m3"}, - {NULL, "gfx1250", "fp4", true, "bf6", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "bf6", "e4m3", "e5m3"}, - {NULL, "gfx1250", "fp4", true, "bf6", "e4m3", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "fp4", "e8m0", "e5m3"}, - {NULL, "gfx1250", "fp4", true, "fp4", "e8m0", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "fp4", "e5m3", "e8m0"}, - {NULL, "gfx1250", "fp4", true, "fp4", "e5m3", "e4m3"}, - {NULL, "gfx1250", "fp4", true, "fp4", "e4m3", "e8m0"}, - {NULL, "gfx1250", "fp4", true, "fp4", "e4m3", "e5m3"}, }; -static const int NUM_CONFIGS = (int)(sizeof(CONFIGS) / sizeof(CONFIGS[0])); +static const int NUM_BASE_CONFIGS = (int)(sizeof(BASE_CONFIGS) / sizeof(BASE_CONFIGS[0])); +static const char* const MATRIX_DTYPES[] = {"fp8", "bf8", "fp6", "bf6", "fp4"}; +static const char* const SCALE_DTYPES[] = {"e8m0", "e5m3", "e4m3"}; +static const int NUM_MATRIX_DTYPES = (int)(sizeof(MATRIX_DTYPES) / sizeof(MATRIX_DTYPES[0])); +static const int NUM_SCALE_DTYPES = (int)(sizeof(SCALE_DTYPES) / sizeof(SCALE_DTYPES[0])); +static const int FP4_INDEX = NUM_MATRIX_DTYPES - 1; +static const int NUM_CONFIGS + = NUM_BASE_CONFIGS + + 2 * NUM_MATRIX_DTYPES * NUM_MATRIX_DTYPES * (NUM_SCALE_DTYPES * NUM_SCALE_DTYPES - 1); + +/* Mirror the Python emitter's legacy FP4 cases, then its remaining scale pairs. + * Selecting by index keeps the table immutable and preserves every case index. + */ +static bool select_config(int idx, config_t* out) +{ + if(idx < 0 || idx >= NUM_CONFIGS) + return false; + if(idx < NUM_BASE_CONFIGS) + { + *out = BASE_CONFIGS[idx]; + return true; + } + idx -= NUM_BASE_CONFIGS; + for(int phase = 0; phase < 2; ++phase) + for(int mode = 0; mode < 2; ++mode) + for(int a = 0; a < NUM_MATRIX_DTYPES; ++a) + for(int b = 0; b < NUM_MATRIX_DTYPES; ++b) + for(int sa = 0; sa < NUM_SCALE_DTYPES; ++sa) + for(int sb = 0; sb < NUM_SCALE_DTYPES; ++sb) + { + if(sa == 0 && sb == 0) + continue; + bool legacy + = (a != FP4_INDEX && b == FP4_INDEX && sa == 0 && sb != 0) + || (a == FP4_INDEX && b != FP4_INDEX && sa != 0 && sb == 0) + || (a == FP4_INDEX && b == FP4_INDEX && sa == sb); + if(legacy != (phase == 0)) + continue; + if(idx-- == 0) + { + *out = (config_t){NULL, + "gfx1250", + MATRIX_DTYPES[a], + mode != 0, + MATRIX_DTYPES[b], + SCALE_DTYPES[sa], + SCALE_DTYPES[sb]}; + return true; + } + } + return false; +} int main(int argc, char** argv) { @@ -906,7 +558,8 @@ int main(int argc, char** argv) fprintf(stderr, "unknown mode %s\n", mode); return 2; } - if(idx < 0 || idx >= NUM_CONFIGS) + config_t config; + if(!select_config(idx, &config)) { fprintf(stderr, "unknown config index %d\n", idx); return 2; @@ -920,18 +573,18 @@ int main(int argc, char** argv) } /* Python: b.kernel.attrs["max_workgroup_size"] = 64 */ rocke_attr_set_int(&b, &b.kernel->attrs, "max_workgroup_size", 64); - if(CONFIGS[idx].build) + if(config.build) { - CONFIGS[idx].build(&b); + config.build(&b); } else { wmma_scaled(&b, - CONFIGS[idx].scale16, - CONFIGS[idx].dtype, - CONFIGS[idx].dtype_b ? CONFIGS[idx].dtype_b : CONFIGS[idx].dtype, - CONFIGS[idx].scale_a ? CONFIGS[idx].scale_a : "e8m0", - CONFIGS[idx].scale_b ? CONFIGS[idx].scale_b : "e8m0"); + config.scale16, + config.dtype, + config.dtype_b ? config.dtype_b : config.dtype, + config.scale_a ? config.scale_a : "e8m0", + config.scale_b ? config.scale_b : "e8m0"); } if(!rocke_ir_builder_ok(&b)) @@ -948,7 +601,7 @@ int main(int argc, char** argv) char err[ROCKE_ERR_MSG_CAP]; err[0] = 0; rocke_status_t st = rocke_lower_kernel_to_llvm_ex( - kernel, ROCKE_LLVM_FLAVOR_AUTO, CONFIGS[idx].arch, &llvm_text, err, sizeof err); + kernel, ROCKE_LLVM_FLAVOR_AUTO, config.arch, &llvm_text, err, sizeof err); if(st != ROCKE_OK || !llvm_text) { fprintf(stderr, "lower failed: status=%d err=%s\n", (int)st, err); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py index 768076b5beea..d06da6304d1c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py @@ -321,219 +321,29 @@ def build(b): ) -CONFIGS.extend( - (_wmma_scaled(a, b, mode, sa, sb), "gfx1250") - for mode in ("scale", "scale16") - for a, b, sa, sb in [ - ("fp8", "fp4", "e8m0", "e5m3"), - ("fp8", "fp4", "e8m0", "e4m3"), - ("bf8", "fp4", "e8m0", "e5m3"), - ("bf8", "fp4", "e8m0", "e4m3"), - ("fp6", "fp4", "e8m0", "e5m3"), - ("fp6", "fp4", "e8m0", "e4m3"), - ("bf6", "fp4", "e8m0", "e5m3"), - ("bf6", "fp4", "e8m0", "e4m3"), - ("fp4", "fp8", "e5m3", "e8m0"), - ("fp4", "fp8", "e4m3", "e8m0"), - ("fp4", "bf8", "e5m3", "e8m0"), - ("fp4", "bf8", "e4m3", "e8m0"), - ("fp4", "fp6", "e5m3", "e8m0"), - ("fp4", "fp6", "e4m3", "e8m0"), - ("fp4", "bf6", "e5m3", "e8m0"), - ("fp4", "bf6", "e4m3", "e8m0"), - ("fp4", "fp4", "e5m3", "e5m3"), - ("fp4", "fp4", "e4m3", "e4m3"), - ] +# Keep the original FP4 scale cases first so configuration indices remain stable. +_MATRIX_DTYPES = ("fp8", "bf8", "fp6", "bf6", "fp4") +_SCALE_DTYPES = ("e8m0", "e5m3", "e4m3") +_LEGACY_SCALE_PAIRS = ( + [(a, "fp4", "e8m0", s) for a in _MATRIX_DTYPES[:-1] for s in _SCALE_DTYPES[1:]] + + [("fp4", b, s, "e8m0") for b in _MATRIX_DTYPES[:-1] for s in _SCALE_DTYPES[1:]] + + [("fp4", "fp4", s, s) for s in _SCALE_DTYPES[1:]] ) - +_LEGACY_SCALE_PAIR_SET = set(_LEGACY_SCALE_PAIRS) +_OTHER_SCALE_PAIRS = [ + (a, b, sa, sb) + for a in _MATRIX_DTYPES + for b in _MATRIX_DTYPES + for sa in _SCALE_DTYPES + for sb in _SCALE_DTYPES + if (sa, sb) != ("e8m0", "e8m0") and (a, b, sa, sb) not in _LEGACY_SCALE_PAIR_SET +] CONFIGS.extend( (_wmma_scaled(a, b, mode, sa, sb), "gfx1250") + for pairs in (_LEGACY_SCALE_PAIRS, _OTHER_SCALE_PAIRS) for mode in ("scale", "scale16") - for a, b, sa, sb in [ - ("fp8", "fp8", "e8m0", "e5m3"), - ("fp8", "fp8", "e8m0", "e4m3"), - ("fp8", "fp8", "e5m3", "e8m0"), - ("fp8", "fp8", "e5m3", "e5m3"), - ("fp8", "fp8", "e5m3", "e4m3"), - ("fp8", "fp8", "e4m3", "e8m0"), - ("fp8", "fp8", "e4m3", "e5m3"), - ("fp8", "fp8", "e4m3", "e4m3"), - ("fp8", "bf8", "e8m0", "e5m3"), - ("fp8", "bf8", "e8m0", "e4m3"), - ("fp8", "bf8", "e5m3", "e8m0"), - ("fp8", "bf8", "e5m3", "e5m3"), - ("fp8", "bf8", "e5m3", "e4m3"), - ("fp8", "bf8", "e4m3", "e8m0"), - ("fp8", "bf8", "e4m3", "e5m3"), - ("fp8", "bf8", "e4m3", "e4m3"), - ("fp8", "fp6", "e8m0", "e5m3"), - ("fp8", "fp6", "e8m0", "e4m3"), - ("fp8", "fp6", "e5m3", "e8m0"), - ("fp8", "fp6", "e5m3", "e5m3"), - ("fp8", "fp6", "e5m3", "e4m3"), - ("fp8", "fp6", "e4m3", "e8m0"), - ("fp8", "fp6", "e4m3", "e5m3"), - ("fp8", "fp6", "e4m3", "e4m3"), - ("fp8", "bf6", "e8m0", "e5m3"), - ("fp8", "bf6", "e8m0", "e4m3"), - ("fp8", "bf6", "e5m3", "e8m0"), - ("fp8", "bf6", "e5m3", "e5m3"), - ("fp8", "bf6", "e5m3", "e4m3"), - ("fp8", "bf6", "e4m3", "e8m0"), - ("fp8", "bf6", "e4m3", "e5m3"), - ("fp8", "bf6", "e4m3", "e4m3"), - ("fp8", "fp4", "e5m3", "e8m0"), - ("fp8", "fp4", "e5m3", "e5m3"), - ("fp8", "fp4", "e5m3", "e4m3"), - ("fp8", "fp4", "e4m3", "e8m0"), - ("fp8", "fp4", "e4m3", "e5m3"), - ("fp8", "fp4", "e4m3", "e4m3"), - ("bf8", "fp8", "e8m0", "e5m3"), - ("bf8", "fp8", "e8m0", "e4m3"), - ("bf8", "fp8", "e5m3", "e8m0"), - ("bf8", "fp8", "e5m3", "e5m3"), - ("bf8", "fp8", "e5m3", "e4m3"), - ("bf8", "fp8", "e4m3", "e8m0"), - ("bf8", "fp8", "e4m3", "e5m3"), - ("bf8", "fp8", "e4m3", "e4m3"), - ("bf8", "bf8", "e8m0", "e5m3"), - ("bf8", "bf8", "e8m0", "e4m3"), - ("bf8", "bf8", "e5m3", "e8m0"), - ("bf8", "bf8", "e5m3", "e5m3"), - ("bf8", "bf8", "e5m3", "e4m3"), - ("bf8", "bf8", "e4m3", "e8m0"), - ("bf8", "bf8", "e4m3", "e5m3"), - ("bf8", "bf8", "e4m3", "e4m3"), - ("bf8", "fp6", "e8m0", "e5m3"), - ("bf8", "fp6", "e8m0", "e4m3"), - ("bf8", "fp6", "e5m3", "e8m0"), - ("bf8", "fp6", "e5m3", "e5m3"), - ("bf8", "fp6", "e5m3", "e4m3"), - ("bf8", "fp6", "e4m3", "e8m0"), - ("bf8", "fp6", "e4m3", "e5m3"), - ("bf8", "fp6", "e4m3", "e4m3"), - ("bf8", "bf6", "e8m0", "e5m3"), - ("bf8", "bf6", "e8m0", "e4m3"), - ("bf8", "bf6", "e5m3", "e8m0"), - ("bf8", "bf6", "e5m3", "e5m3"), - ("bf8", "bf6", "e5m3", "e4m3"), - ("bf8", "bf6", "e4m3", "e8m0"), - ("bf8", "bf6", "e4m3", "e5m3"), - ("bf8", "bf6", "e4m3", "e4m3"), - ("bf8", "fp4", "e5m3", "e8m0"), - ("bf8", "fp4", "e5m3", "e5m3"), - ("bf8", "fp4", "e5m3", "e4m3"), - ("bf8", "fp4", "e4m3", "e8m0"), - ("bf8", "fp4", "e4m3", "e5m3"), - ("bf8", "fp4", "e4m3", "e4m3"), - ("fp6", "fp8", "e8m0", "e5m3"), - ("fp6", "fp8", "e8m0", "e4m3"), - ("fp6", "fp8", "e5m3", "e8m0"), - ("fp6", "fp8", "e5m3", "e5m3"), - ("fp6", "fp8", "e5m3", "e4m3"), - ("fp6", "fp8", "e4m3", "e8m0"), - ("fp6", "fp8", "e4m3", "e5m3"), - ("fp6", "fp8", "e4m3", "e4m3"), - ("fp6", "bf8", "e8m0", "e5m3"), - ("fp6", "bf8", "e8m0", "e4m3"), - ("fp6", "bf8", "e5m3", "e8m0"), - ("fp6", "bf8", "e5m3", "e5m3"), - ("fp6", "bf8", "e5m3", "e4m3"), - ("fp6", "bf8", "e4m3", "e8m0"), - ("fp6", "bf8", "e4m3", "e5m3"), - ("fp6", "bf8", "e4m3", "e4m3"), - ("fp6", "fp6", "e8m0", "e5m3"), - ("fp6", "fp6", "e8m0", "e4m3"), - ("fp6", "fp6", "e5m3", "e8m0"), - ("fp6", "fp6", "e5m3", "e5m3"), - ("fp6", "fp6", "e5m3", "e4m3"), - ("fp6", "fp6", "e4m3", "e8m0"), - ("fp6", "fp6", "e4m3", "e5m3"), - ("fp6", "fp6", "e4m3", "e4m3"), - ("fp6", "bf6", "e8m0", "e5m3"), - ("fp6", "bf6", "e8m0", "e4m3"), - ("fp6", "bf6", "e5m3", "e8m0"), - ("fp6", "bf6", "e5m3", "e5m3"), - ("fp6", "bf6", "e5m3", "e4m3"), - ("fp6", "bf6", "e4m3", "e8m0"), - ("fp6", "bf6", "e4m3", "e5m3"), - ("fp6", "bf6", "e4m3", "e4m3"), - ("fp6", "fp4", "e5m3", "e8m0"), - ("fp6", "fp4", "e5m3", "e5m3"), - ("fp6", "fp4", "e5m3", "e4m3"), - ("fp6", "fp4", "e4m3", "e8m0"), - ("fp6", "fp4", "e4m3", "e5m3"), - ("fp6", "fp4", "e4m3", "e4m3"), - ("bf6", "fp8", "e8m0", "e5m3"), - ("bf6", "fp8", "e8m0", "e4m3"), - ("bf6", "fp8", "e5m3", "e8m0"), - ("bf6", "fp8", "e5m3", "e5m3"), - ("bf6", "fp8", "e5m3", "e4m3"), - ("bf6", "fp8", "e4m3", "e8m0"), - ("bf6", "fp8", "e4m3", "e5m3"), - ("bf6", "fp8", "e4m3", "e4m3"), - ("bf6", "bf8", "e8m0", "e5m3"), - ("bf6", "bf8", "e8m0", "e4m3"), - ("bf6", "bf8", "e5m3", "e8m0"), - ("bf6", "bf8", "e5m3", "e5m3"), - ("bf6", "bf8", "e5m3", "e4m3"), - ("bf6", "bf8", "e4m3", "e8m0"), - ("bf6", "bf8", "e4m3", "e5m3"), - ("bf6", "bf8", "e4m3", "e4m3"), - ("bf6", "fp6", "e8m0", "e5m3"), - ("bf6", "fp6", "e8m0", "e4m3"), - ("bf6", "fp6", "e5m3", "e8m0"), - ("bf6", "fp6", "e5m3", "e5m3"), - ("bf6", "fp6", "e5m3", "e4m3"), - ("bf6", "fp6", "e4m3", "e8m0"), - ("bf6", "fp6", "e4m3", "e5m3"), - ("bf6", "fp6", "e4m3", "e4m3"), - ("bf6", "bf6", "e8m0", "e5m3"), - ("bf6", "bf6", "e8m0", "e4m3"), - ("bf6", "bf6", "e5m3", "e8m0"), - ("bf6", "bf6", "e5m3", "e5m3"), - ("bf6", "bf6", "e5m3", "e4m3"), - ("bf6", "bf6", "e4m3", "e8m0"), - ("bf6", "bf6", "e4m3", "e5m3"), - ("bf6", "bf6", "e4m3", "e4m3"), - ("bf6", "fp4", "e5m3", "e8m0"), - ("bf6", "fp4", "e5m3", "e5m3"), - ("bf6", "fp4", "e5m3", "e4m3"), - ("bf6", "fp4", "e4m3", "e8m0"), - ("bf6", "fp4", "e4m3", "e5m3"), - ("bf6", "fp4", "e4m3", "e4m3"), - ("fp4", "fp8", "e8m0", "e5m3"), - ("fp4", "fp8", "e8m0", "e4m3"), - ("fp4", "fp8", "e5m3", "e5m3"), - ("fp4", "fp8", "e5m3", "e4m3"), - ("fp4", "fp8", "e4m3", "e5m3"), - ("fp4", "fp8", "e4m3", "e4m3"), - ("fp4", "bf8", "e8m0", "e5m3"), - ("fp4", "bf8", "e8m0", "e4m3"), - ("fp4", "bf8", "e5m3", "e5m3"), - ("fp4", "bf8", "e5m3", "e4m3"), - ("fp4", "bf8", "e4m3", "e5m3"), - ("fp4", "bf8", "e4m3", "e4m3"), - ("fp4", "fp6", "e8m0", "e5m3"), - ("fp4", "fp6", "e8m0", "e4m3"), - ("fp4", "fp6", "e5m3", "e5m3"), - ("fp4", "fp6", "e5m3", "e4m3"), - ("fp4", "fp6", "e4m3", "e5m3"), - ("fp4", "fp6", "e4m3", "e4m3"), - ("fp4", "bf6", "e8m0", "e5m3"), - ("fp4", "bf6", "e8m0", "e4m3"), - ("fp4", "bf6", "e5m3", "e5m3"), - ("fp4", "bf6", "e5m3", "e4m3"), - ("fp4", "bf6", "e4m3", "e5m3"), - ("fp4", "bf6", "e4m3", "e4m3"), - ("fp4", "fp4", "e8m0", "e5m3"), - ("fp4", "fp4", "e8m0", "e4m3"), - ("fp4", "fp4", "e5m3", "e8m0"), - ("fp4", "fp4", "e5m3", "e4m3"), - ("fp4", "fp4", "e4m3", "e8m0"), - ("fp4", "fp4", "e4m3", "e5m3"), - ] + for a, b, sa, sb in pairs ) From d67410ee65d7d4c7c91c1ac22ee0e3eb2048cc09 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:17:05 -0400 Subject: [PATCH 28/32] refactor(rocke): generate scaled WMMA catalogs from dtype loops --- .../rocke/platform/cpp/core/arch/data.cpp | 10549 +--------------- .../dsl_docs/architecture/adding_an_arch.md | 20 + .../rocke/core/arch/data/arch_specs.json | 476 +- .../platform/python/rocke/core/arch/target.py | 315 +- .../platform/tests/core/mma_frag_ssot.cpp | 15 +- .../platform/tests/core/test_arch_mma_ssot.py | 68 + .../tests/core/test_mma_catalog_lookup.cpp | 15 + .../tests/core/test_mma_frag_tables.py | 21 +- .../tests/core/test_mma_scale_layouts.py | 23 + 9 files changed, 504 insertions(+), 10998 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp index d7afd829fcaf..a258fa374fca 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp @@ -914,8 +914,8 @@ static const rocke_layout_map_t lm_wmma_gfx1250_32_b * This is the arch SSOT that rocke.core.ir.IRBuilder.mma (rocke_b_mma) consults * to size a tile.mma result vector from a bare op_id string; the ir bucket keeps * no private copy. It mirrors _MMA_FRAGMENT_INFO's c_frag_len column exactly. - * Only atoms this engine can emit are listed: this table mirrors the Python SSOT - * including the gfx1250 WMMA atoms. + * Unscaled atoms are listed here; scaled gfx1250 atoms are resolved from the + * generated catalog below, avoiding a second copy of their IDs. * * `ati` = arch-target internal (see arch_target_internal.h): the naming prefix * reserved for storage private to this arch-target port, kept out of the public @@ -994,475 +994,8 @@ static const rocke_ati_mma_frag_row_t rocke_ati_mma_frag[] = { {"wmma_gfx1250_f32_16x16x64_fp8_bf8", 8}, {"wmma_gfx1250_f32_16x16x64_bf8_fp8", 8}, {"wmma_gfx1250_f32_16x16x64_bf8_bf8", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e8m0_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e5m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e4m3_k32", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e8m0_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e4m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e5m3_k16", 8}, - {"wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e4m3_k16", 8}, }; -int rocke_arch_mma_c_frag_len(const char* op_id) -{ - size_t i; - if(!op_id) - { - return 0; - } - for(i = 0; i < sizeof(rocke_ati_mma_frag) / sizeof(rocke_ati_mma_frag[0]); ++i) - { - if(strcmp(rocke_ati_mma_frag[i].op_id, op_id) == 0) - { - return rocke_ati_mma_frag[i].c_frag_len; - } - } - return 0; /* unknown atom: the zero-length _frag_info fallback */ -} - /* ========================================================================= * Per-arch rocke_mma_op_t[] catalogs (the embedded arch_specs.json mma rows, * enriched with frag lengths + layout-map pointers, mirroring _build_mma_op). @@ -2150,7 +1683,7 @@ static const rocke_mma_op_t k_mma_gfx11_generic[] = { * pending the gfx1250 ISA backend port (rocke_ll_backend_for); the accumulator * reuses lm_wmma_gfx12_c (same 16x16 tile layout). FP8/BF8 K=64 atoms carry * frag_len=8 (8 i32 per lane) with NULL A/B maps. */ -static const rocke_mma_op_t k_mma_gfx1250[] = { +static constexpr rocke_mma_op_t k_mma_gfx1250_unscaled[] = { {"wmma", "fp32", "fp32", @@ -2256,9908 +1789,182 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { NULL, NULL, &lm_wmma_gfx12_c}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e8m0_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e5m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e4m3_k32", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K32, - 4, - 4, - &lm_wmma_scale_k32_a, - &lm_wmma_scale_k32_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp8e4m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "bf8e5m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e2m3", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e8m0_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e8m0", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp6e3m2", - "fp4e2m1", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp8e4m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "bf8e5m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e2m3", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e8m0", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e5m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e5m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e5m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, - {"wmma_scaled", - "fp4e2m1", - "fp6e3m2", - "fp32", - 16, - 16, - 128, - "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e4m3_k16", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c, - "e4m3", - "e4m3", - ROCKE_MMA_SCALE_K16, - 8, - 8, - &lm_wmma_scale_k16_a, - &lm_wmma_scale_k16_b}, }; +/* Private PODs evaluated at compile time. IDs have separate final static + * storage: copying the descriptor array cannot leave pointers into temporaries. + * Keep the same staged order as _gfx1250_scaled_rows; no runtime initialization, + * allocation, STL, JSON parsing, or Python dependency. */ +namespace +{ +constexpr int k_scaled_count = 450; +constexpr int k_unscaled_count = 7; +constexpr const char* k_inputs[] = {"fp8", "bf8", "fp6", "bf6", "fp4"}; +constexpr const char* k_input_dtypes[] = {"fp8e4m3", "bf8e5m2", "fp6e2m3", "fp6e3m2", "fp4e2m1"}; +constexpr const char* k_scales[] = {"e8m0", "e5m3", "e4m3"}; +constexpr int k_blocks[] = {32, 16}; +// Native's original E8M0 seed order predates the JSON order; retain both. +constexpr int k_native_e8m0[][2] + = {{0, 0}, {4, 0}, {2, 0}, {3, 0}, {1, 0}, {0, 1}, {4, 1}, {2, 1}, {3, 1}, {1, 1}}; +constexpr int k_homogeneous[][2] + = {{0, 0}, {2, 0}, {3, 0}, {1, 0}, {0, 1}, {2, 1}, {3, 1}, {4, 0}, {4, 1}, {1, 1}}; +struct scaled_key +{ + int a, b, sa, sb, mode; +}; +struct scaled_keys +{ + scaled_key rows[k_scaled_count]; + int count; +}; + +consteval scaled_keys make_scaled_keys() +{ + scaled_keys result{}; + bool seen[2][5][5][3][3] = {}; + auto add = [&](int a, int b, int sa, int sb, int mode) { + if(!seen[mode][a][b][sa][sb]) + { + if(result.count == k_scaled_count) + throw "too many scaled MMA atoms"; + seen[mode][a][b][sa][sb] = true; + result.rows[result.count++] = {a, b, sa, sb, mode}; + } + }; + for(const auto& seed : k_native_e8m0) + add(seed[0], seed[0], 0, 0, seed[1]); + for(int mode = 0; mode < 2; ++mode) + for(int a = 0; a < 5; ++a) + for(int b = 0; b < 5; ++b) + if(a != b) + add(a, b, 0, 0, mode); + for(int mode = 0; mode < 2; ++mode) + { + for(int a = 0; a < 4; ++a) + for(int sb = 1; sb < 3; ++sb) + add(a, 4, 0, sb, mode); + for(int b = 0; b < 4; ++b) + for(int sa = 1; sa < 3; ++sa) + add(4, b, sa, 0, mode); + for(int scale = 1; scale < 3; ++scale) + add(4, 4, scale, scale, mode); + } + for(const auto& seed : k_homogeneous) + for(int sa = 0; sa < 3; ++sa) + for(int sb = 0; sb < 3; ++sb) + add(seed[0], seed[0], sa, sb, seed[1]); + for(int mode = 0; mode < 2; ++mode) + for(int a = 0; a < 5; ++a) + for(int b = 0; b < 5; ++b) + if(a != b) + for(int sa = 0; sa < 3; ++sa) + for(int sb = 0; sb < 3; ++sb) + add(a, b, sa, sb, mode); + return result; +} +constexpr auto k_scaled_keys = make_scaled_keys(); +static_assert(k_scaled_keys.count == k_scaled_count); +static_assert(sizeof(k_mma_gfx1250_unscaled) / sizeof(k_mma_gfx1250_unscaled[0]) + == k_unscaled_count); + +struct scaled_ids +{ + char rows[k_scaled_count][96]; +}; +consteval scaled_ids make_scaled_ids() +{ + scaled_ids result{}; + for(int i = 0; i < k_scaled_count; ++i) + { + const auto& key = k_scaled_keys.rows[i]; + int length = 0; + auto append = [&](const char* text) { + for(int j = 0; text[j]; ++j) + { + if(length >= 95) + throw "scaled MMA ID too long"; + result.rows[i][length++] = text[j]; + } + }; + append("wmma_gfx1250_f32_16x16x128_"); + append(k_inputs[key.a]); + append("_"); + append(k_inputs[key.b]); + append("_scale_"); + append(k_scales[key.sa]); + append("_"); + append(k_scales[key.sb]); + append(key.mode == 0 ? "_k32" : "_k16"); + } + return result; +} +constexpr auto k_scaled_ids = make_scaled_ids(); + +struct gfx1250_catalog +{ + rocke_mma_op_t ops[k_unscaled_count + k_scaled_count]; +}; +consteval gfx1250_catalog make_gfx1250_catalog() +{ + gfx1250_catalog result{}; + for(int i = 0; i < k_unscaled_count; ++i) + result.ops[i] = k_mma_gfx1250_unscaled[i]; + for(int i = 0; i < k_scaled_count; ++i) + { + const auto& key = k_scaled_keys.rows[i]; + const bool k32 = key.mode == 0; + result.ops[k_unscaled_count + i] = { + "wmma_scaled", + k_input_dtypes[key.a], + k_input_dtypes[key.b], + "fp32", + 16, + 16, + 128, + k_scaled_ids.rows[i], + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + k_scales[key.sa], + k_scales[key.sb], + k32 ? ROCKE_MMA_SCALE_K32 : ROCKE_MMA_SCALE_K16, + 128 / k_blocks[key.mode], + 128 / k_blocks[key.mode], + k32 ? &lm_wmma_scale_k32_a : &lm_wmma_scale_k16_a, + k32 ? &lm_wmma_scale_k32_b : &lm_wmma_scale_k16_b, + }; + } + return result; +} +constexpr auto k_mma_gfx1250 = make_gfx1250_catalog(); +} // namespace + +int rocke_arch_mma_c_frag_len(const char* op_id) +{ + size_t i; + if(!op_id) + { + return 0; + } + for(i = 0; i < sizeof(rocke_ati_mma_frag) / sizeof(rocke_ati_mma_frag[0]); ++i) + { + if(strcmp(rocke_ati_mma_frag[i].op_id, op_id) == 0) + { + return rocke_ati_mma_frag[i].c_frag_len; + } + } + for(const auto& op : k_mma_gfx1250.ops) + { + if(strcmp(op.op_id, op_id) == 0) + return op.c_frag_len; + } + return 0; /* unknown atom: the zero-length _frag_info fallback */ +} + /* ========================================================================= * Per-arch rocke_arch_target_t singletons (the embedded arches[gfx] entries). * ========================================================================= @@ -12238,7 +2045,7 @@ static const rocke_arch_target_t k_target_gfx1250 = { 32, 163840, 6, - {k_mma_gfx1250, K_NUM(k_mma_gfx1250)}, + {k_mma_gfx1250.ops, K_NUM(k_mma_gfx1250.ops)}, {false, false, 4, 0}, {1024, 256, 0, 106}, }; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/adding_an_arch.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/adding_an_arch.md index bfcc8f3cf490..9410f00ef531 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/adding_an_arch.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/adding_an_arch.md @@ -115,6 +115,26 @@ Mirror the same facts into the C++ SSOT, static `rocke_arch_target_t` plus its `k_mma_gfxNNNN[]` catalog. The two SSOTs are compared by the byte-identity gate, not by review alone. +The gfx1250 scaled family uses one JSON declaration, `gfx1250_wmma_scaled_v1`, +instead of individual rows. It expands independent A/B matrix types +(`fp8`, `bf8`, `fp6`, `bf6`, `fp4`), independent A/B scale types +(`e8m0`, `e5m3`, `e4m3`), and K-group sizes 32/16 into 450 ordinary +`wmma_scaled` rows with shape 16×16×128 and fp32 accumulators. The loader +validates the complete declaration before expansion and rejects duplicate IDs. +Version 1 fixes both these axes and their historical enumeration order; changing +support requires an explicit loader/native generator update and regression +baseline review. The first ten native E8M0 entries historically use a different +order from Python; each API retains its own pre-change order. Other architectures +retain explicit rows. + +Python generates the scaled fragment dictionary from the expanded catalog. +C++ generates the contiguous catalog at compile time using private POD arrays; +ID strings live in separate immutable static storage. The native lookup remains +libc-only at runtime, with no JSON parser, Python dependency, allocation, or +dynamic initialization. Bare-ID accumulator fragment lookup consults that +catalog rather than maintaining a second scaled-ID table. Compare every +expanded row against the pre-change baseline as well as running engine parity. + ### New matrix atoms An `op_id` that no existing arch lists needs a row in `_MMA_FRAGMENT_INFO` diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json index b9b6836e110a..c8ea2c948eec 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json @@ -186,456 +186,32 @@ {"family": "wmma", "a": "fp8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_fp8_bf8"}, {"family": "wmma", "a": "bf8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_fp8"}, {"family": "wmma", "a": "bf8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_bf8"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e8m0_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e5m3_k32"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e4m3_k32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf8_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp6_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_bf6_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp4_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp8_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp6_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf6_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf8", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_fp4_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp8_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf8_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_bf6_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp6_fp4_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp8_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_bf8_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp6_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e8m0_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "bf6", "b": "fp4", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf6_fp4_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp8_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf8", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf8_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "fp6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_fp6_scale_e4m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e8m0_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e5m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e5m3_e4m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e5m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e5m3_k16"}, - {"family": "wmma_scaled", "a": "fp4", "b": "bf6", "c": "fp32", "a_scale_dtype": "e4m3", "b_scale_dtype": "e4m3", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp4_bf6_scale_e4m3_e4m3_k16"} + { + "generate": "gfx1250_wmma_scaled_v1", + "family": "wmma_scaled", + "input_dtypes": [ + "fp8", + "bf8", + "fp6", + "bf6", + "fp4" + ], + "scale_dtypes": [ + "e8m0", + "e5m3", + "e4m3" + ], + "block_k": [ + 32, + 16 + ], + "shape": [ + 16, + 16, + 128 + ], + "c": "fp32" + } ] }, "gfx11-generic": { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py index 4eb248048167..763fde530330 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py @@ -30,6 +30,130 @@ _DATA_FILE = Path(__file__).parent / "data" / "arch_specs.json" +# A bounded family declaration, rather than a general catalog templating DSL. +# Version 1 freezes the shipped axes and historical enumeration order. +_GFX1250_SCALED_FAMILY = { + "generate": "gfx1250_wmma_scaled_v1", + "family": "wmma_scaled", + "input_dtypes": ["fp8", "bf8", "fp6", "bf6", "fp4"], + "scale_dtypes": ["e8m0", "e5m3", "e4m3"], + "block_k": [32, 16], + "shape": [16, 16, 128], + "c": "fp32", +} + + +def _expand_mma_rows(gfx: str, declarations: list[dict]) -> list[dict]: + """Expand supported families to ordinary rows; reject duplicate identities.""" + if not isinstance(declarations, list): + raise ValueError(f"MMA declarations for {gfx} must be a list") + out = [] + ids = set() + for declaration in declarations: + if not isinstance(declaration, dict) or ( + "generate" not in declaration and "op_id" not in declaration + ): + raise ValueError(f"invalid MMA declaration for {gfx}: {declaration!r}") + if "generate" not in declaration: + rows = [declaration] + else: + # JSON text comparison also rejects bools masquerading as integers. + if gfx != "gfx1250" or json.dumps( + declaration, sort_keys=True + ) != json.dumps(_GFX1250_SCALED_FAMILY, sort_keys=True): + raise ValueError( + f"unsupported MMA family declaration for {gfx}: {declaration!r}" + ) + rows = _gfx1250_scaled_rows(declaration) + for row in rows: + if row["op_id"] in ids: + raise ValueError(f"duplicate MMA op_id for {gfx}: {row['op_id']}") + ids.add(row["op_id"]) + out.append(row) + return out + + +def _gfx1250_scaled_rows(declaration: dict) -> list[dict]: + inputs = declaration["input_dtypes"] + scales = declaration["scale_dtypes"] + blocks = declaration["block_k"] + # Original homogeneous E8M0 atoms precede mixed E8M0, the first FP4 + # alternative encodings, then the remaining homogeneous/mixed encodings. + homogeneous = [ + (0, 32), + (2, 32), + (3, 32), + (1, 32), + (0, 16), + (2, 16), + (3, 16), + (4, 32), + (4, 16), + (1, 16), + ] + keys = [(inputs[i], inputs[i], scales[0], scales[0], k) for i, k in homogeneous] + keys.extend( + (a, b, scales[0], scales[0], k) + for k in blocks + for a in inputs + for b in inputs + if a != b + ) + for k in blocks: + keys.extend( + (a, inputs[4], scales[0], sb, k) for a in inputs[:4] for sb in scales[1:] + ) + keys.extend( + (inputs[4], b, sa, scales[0], k) for b in inputs[:4] for sa in scales[1:] + ) + keys.extend((inputs[4], inputs[4], scale, scale, k) for scale in scales[1:]) + keys.extend( + (inputs[i], inputs[i], sa, sb, k) + for i, k in homogeneous + for sa in scales + for sb in scales + ) + keys.extend( + (a, b, sa, sb, k) + for k in blocks + for a in inputs + for b in inputs + if a != b + for sa in scales + for sb in scales + ) + rows = [] + # Later stages fill the Cartesian product without repeating earlier atoms. + for a, b, sa, sb, block_k in dict.fromkeys(keys): + m, n, k = declaration["shape"] + rows.append( + { + "family": declaration["family"], + "a": a, + "b": b, + "c": declaration["c"], + "a_scale_dtype": sa, + "b_scale_dtype": sb, + "scale_block_k": block_k, + "m": m, + "n": n, + "k": k, + "op_id": f"wmma_gfx1250_f32_{m}x{n}x{k}_{a}_{b}_scale_{sa}_{sb}_k{block_k}", + } + ) + return rows + + +@lru_cache(maxsize=1) +def _load_specs() -> Dict[str, dict]: + # Pin UTF-8 for embedded interpreters whose default codec is ASCII. + with open(_DATA_FILE, encoding="utf-8") as fh: + doc = json.load(fh) + for gfx, spec in doc["arches"].items(): + spec["mma"] = _expand_mma_rows(gfx, spec["mma"]) + return doc["arches"] + + # A callable that, given an :class:`~rocke.core.ir.IRBuilder`, a runtime lane # ``Value`` (0..wave_size-1) and a compile-time fragment slot index, emits the # index arithmetic for the two tile coordinates of that fragment element and @@ -859,141 +983,6 @@ class _FragInfo: None, _wmma_gfx12_acc_16x16, ), - # Native gfx1250 scaled WMMA. FP8/BF8 use 64 bytes per lane. FP6 uses - # 48 packed bytes plus four zero words; FP4 uses 32 bytes plus eight zero - # words. All share the same <16 x i32> ABI. - # SCALE packs four K=32 E8M0 bytes in i32 and SCALE16 packs eight K=16 - # bytes in i64. Both share the gfx12 column-distributed accumulator. - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32": _FragInfo( - 16, - 16, - 8, - 32, - None, - None, - _wmma_gfx12_acc_16x16, - a_scale_frag_len=4, - b_scale_frag_len=4, - a_scale_fn=_wmma_gfx1250_a_scale, - b_scale_fn=_wmma_gfx1250_b_scale, - ), - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e8m0_k32": _FragInfo( - 16, - 16, - 8, - 32, - None, - None, - _wmma_gfx12_acc_16x16, - a_scale_frag_len=4, - b_scale_frag_len=4, - a_scale_fn=_wmma_gfx1250_a_scale, - b_scale_fn=_wmma_gfx1250_b_scale, - ), - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k32": _FragInfo( - 16, - 16, - 8, - 32, - None, - None, - _wmma_gfx12_acc_16x16, - a_scale_frag_len=4, - b_scale_frag_len=4, - a_scale_fn=_wmma_gfx1250_a_scale, - b_scale_fn=_wmma_gfx1250_b_scale, - ), - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k32": _FragInfo( - 16, - 16, - 8, - 32, - None, - None, - _wmma_gfx12_acc_16x16, - a_scale_frag_len=4, - b_scale_frag_len=4, - a_scale_fn=_wmma_gfx1250_a_scale, - b_scale_fn=_wmma_gfx1250_b_scale, - ), - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k32": _FragInfo( - 16, - 16, - 8, - 32, - None, - None, - _wmma_gfx12_acc_16x16, - a_scale_frag_len=4, - b_scale_frag_len=4, - a_scale_fn=_wmma_gfx1250_a_scale, - b_scale_fn=_wmma_gfx1250_b_scale, - ), - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16": _FragInfo( - 16, - 16, - 8, - 32, - None, - None, - _wmma_gfx12_acc_16x16, - a_scale_frag_len=8, - b_scale_frag_len=8, - a_scale_fn=_wmma_gfx1250_a_scale, - b_scale_fn=_wmma_gfx1250_b_scale, - ), - "wmma_gfx1250_f32_16x16x128_fp4_fp4_scale_e8m0_e8m0_k16": _FragInfo( - 16, - 16, - 8, - 32, - None, - None, - _wmma_gfx12_acc_16x16, - a_scale_frag_len=8, - b_scale_frag_len=8, - a_scale_fn=_wmma_gfx1250_a_scale, - b_scale_fn=_wmma_gfx1250_b_scale, - ), - "wmma_gfx1250_f32_16x16x128_fp6_fp6_scale_e8m0_e8m0_k16": _FragInfo( - 16, - 16, - 8, - 32, - None, - None, - _wmma_gfx12_acc_16x16, - a_scale_frag_len=8, - b_scale_frag_len=8, - a_scale_fn=_wmma_gfx1250_a_scale, - b_scale_fn=_wmma_gfx1250_b_scale, - ), - "wmma_gfx1250_f32_16x16x128_bf6_bf6_scale_e8m0_e8m0_k16": _FragInfo( - 16, - 16, - 8, - 32, - None, - None, - _wmma_gfx12_acc_16x16, - a_scale_frag_len=8, - b_scale_frag_len=8, - a_scale_fn=_wmma_gfx1250_a_scale, - b_scale_fn=_wmma_gfx1250_b_scale, - ), - "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16": _FragInfo( - 16, - 16, - 8, - 32, - None, - None, - _wmma_gfx12_acc_16x16, - a_scale_frag_len=8, - b_scale_frag_len=8, - a_scale_fn=_wmma_gfx1250_a_scale, - b_scale_fn=_wmma_gfx1250_b_scale, - ), "wmma_gfx1250_f32_16x16x32_bf16": _FragInfo( 16, 16, @@ -1006,34 +995,25 @@ class _FragInfo: } -# Mixed operands keep the same padded ABI and scale-coordinate maps. -_MMA_FRAGMENT_INFO.update( - { - f"wmma_gfx1250_f32_16x16x128_{a}_{b}_scale_e8m0_e8m0_k{block_k}": _MMA_FRAGMENT_INFO[ - f"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k{block_k}" - ] - for block_k in (32, 16) - for a in ("fp8", "bf8", "fp6", "bf6", "fp4") - for b in ("fp8", "bf8", "fp6", "bf6", "fp4") - if a != b - } -) - - -# Alternative scale encodings retain the same byte carriers and lane maps. -_MMA_FRAGMENT_INFO.update( - { - f"wmma_gfx1250_f32_16x16x128_{a}_{b}_scale_{sa}_{sb}_k{block_k}": _MMA_FRAGMENT_INFO[ - f"wmma_gfx1250_f32_16x16x128_{a}_{b}_scale_e8m0_e8m0_k{block_k}" - ] - for block_k in (32, 16) - for a in ("fp8", "bf8", "fp6", "bf6", "fp4") - for b in ("fp8", "bf8", "fp6", "bf6", "fp4") - for sa in ("e8m0", "e5m3", "e4m3") - for sb in ("e8m0", "e5m3", "e4m3") - if (sa, sb) != ("e8m0", "e8m0") - } -) +# Scaled operands share the padded <16 x i32> ABI, byte-carrier scales, +# and gfx12 accumulator map. Derive every ID from the expanded catalog. +for _row in _load_specs()["gfx1250"]["mma"]: + if _row["family"] == "wmma_scaled": + _scale_frag_len = 128 // _row["scale_block_k"] + _MMA_FRAGMENT_INFO[_row["op_id"]] = _FragInfo( + 16, + 16, + 8, + 32, + None, + None, + _wmma_gfx12_acc_16x16, + a_scale_frag_len=_scale_frag_len, + b_scale_frag_len=_scale_frag_len, + a_scale_fn=_wmma_gfx1250_a_scale, + b_scale_fn=_wmma_gfx1250_b_scale, + ) +del _row, _scale_frag_len def _frag_info(op_id: str) -> _FragInfo: @@ -1279,15 +1259,6 @@ def from_gfx(gfx: str) -> "ArchTarget": # --------------------------------------------------------------------------- -@lru_cache(maxsize=1) -def _load_specs() -> Dict[str, dict]: - # Pin UTF-8: the embedded interpreter defaults to the ASCII codec, and - # arch_specs.json contains non-ASCII bytes. - with open(_DATA_FILE, encoding="utf-8") as fh: - doc = json.load(fh) - return doc["arches"] - - @lru_cache(maxsize=1) def _op_id_c_dtype() -> Dict[str, str]: """``op_id -> normalized accumulator dtype``, aggregated across every arch diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp index 9cad6977a3f5..ee21291b62a8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp @@ -99,10 +99,17 @@ int main(void) CHECK(gfx1250 != NULL, "gfx1250 target"); if(gfx1250) { - check_catalog_fragments(&gfx1250->mma, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32"); - check_catalog_fragments(&gfx1250->mma, - "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16"); + int scaled = 0; + for(int i = 0; i < gfx1250->mma.num_ops; ++i) + { + const auto& op = gfx1250->mma.ops[i]; + if(strcmp(op.family, "wmma_scaled") != 0) + continue; + check_catalog_fragments(&gfx1250->mma, op.op_id); + check_atom(&b, op.op_id, 8, false); + ++scaled; + } + CHECK(scaled == 450, "all scaled atoms must have accumulator metadata"); } /* Unknown op_id must be rejected. The engine's error path either returns diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py index 8a95ebf22278..d6c44b90a5bf 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py @@ -11,8 +11,10 @@ from __future__ import annotations +import hashlib import json import unittest +from copy import deepcopy from dataclasses import asdict, replace from unittest import mock @@ -23,12 +25,78 @@ from rocke.core.arch.target import ( _load_specs, + _expand_mma_rows, + _GFX1250_SCALED_FAMILY, _op_id_c_dtype, _op_id_family, normalize_dtype, ) +def test_expanded_catalog_matches_pre_refactor_order_and_fields(): + # Independent snapshot of all 457 hand-written rows at 0bba8265. A generator + # bug shared by both engines must not pass just because parity stays green. + rows = _load_specs()["gfx1250"]["mma"] + assert len(rows) == 457 + encoded = json.dumps(rows, sort_keys=True, separators=(",", ":")).encode() + assert hashlib.sha256(encoded).hexdigest() == ( + "95ae6cc6ecb4d704eea729eafd291ba2d1d89183d59be28ba50af66a0377590a" + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("generate", "gfx1250_wmma_scaled_v2"), + ("family", "wmma"), + ("input_dtypes", ["fp8", "bf8", "fp6", "bf6", "fp8"]), + ("input_dtypes", ["fp8"]), + ("scale_dtypes", ["e8m0", "e5m2", "e4m3"]), + ("scale_dtypes", ["e8m0", "e4m3", "e5m3"]), + ("block_k", [32, 8]), + ("block_k", [32.0, 16]), + ("shape", [16, 16, 64]), + ("c", "fp16"), + ("unexpected", True), + ], +) +def test_family_declaration_rejects_unsupported_axes(field, value): + declaration = deepcopy(_GFX1250_SCALED_FAMILY) + declaration[field] = value + with pytest.raises(ValueError, match="unsupported MMA family declaration"): + _expand_mma_rows("gfx1250", [declaration]) + + +def test_family_declaration_rejects_wrong_arch_missing_fields_and_duplicates(): + declaration = deepcopy(_GFX1250_SCALED_FAMILY) + with pytest.raises(ValueError, match="unsupported MMA family declaration"): + _expand_mma_rows("gfx950", [declaration]) + for field in declaration: + if field == "generate": + with pytest.raises(ValueError, match="invalid MMA declaration"): + _expand_mma_rows( + "gfx1250", [{k: v for k, v in declaration.items() if k != field}] + ) + continue + missing = {k: v for k, v in declaration.items() if k != field} + with pytest.raises(ValueError, match="unsupported MMA family declaration"): + _expand_mma_rows("gfx1250", [missing]) + rows = _expand_mma_rows("gfx1250", [declaration]) + for declarations in ( + [declaration, declaration], + [rows[0], declaration], + [declaration, rows[-1]], + ): + with pytest.raises(ValueError, match="duplicate MMA op_id"): + _expand_mma_rows("gfx1250", declarations) + + +@pytest.mark.parametrize("declarations", [None, {}, [None], [42], [{}]]) +def test_invalid_family_schema_fails_loudly(declarations): + with pytest.raises(ValueError, match="MMA declaration"): + _expand_mma_rows("gfx1250", declarations) + + class TestOpIdCDtype(unittest.TestCase): def test_matches_catalog_first_hit(self): # Every op_id in the catalog resolves to its normalized accumulator dtype, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp index 5c9c2b384f01..a0244de9ed73 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: MIT /* Exercise public catalog queries against stored keys, without normalizing * away malformed table entries. Alias queries must find the same atom. */ +#include #include #include @@ -314,6 +315,20 @@ static int test_scale_contracts() ++scaled_rows; } } + // Ordered identity fingerprint from the hand-written 0bba8265 catalog. + // Also exercises all generated static ID pointers after repeated queries. + uint64_t fingerprint = UINT64_C(14695981039346656037); + for(int i = 0; i < arch->mma.num_ops; ++i) + { + const char* text = arch->mma.ops[i].op_id; + do + { + fingerprint ^= static_cast(*text); + fingerprint *= UINT64_C(1099511628211); + } while(*text++); + } + CHECK(arch->mma.num_ops == 457); + CHECK(fingerprint == UINT64_C(16714491246104151946)); CHECK(packed_rows == 4 && scaled_rows == 450); return 0; } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py index eeb6fa6b1bf4..7811952e6fa1 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py @@ -26,7 +26,7 @@ import unittest -from rocke.core.arch.target import _MMA_FRAGMENT_INFO, _op_id_c_dtype +from rocke.core.arch.target import _MMA_FRAGMENT_INFO, _op_id_c_dtype, _load_specs from rocke.core.ir import ( _MMA_RESULT_HINT, _mma_c_frag_len, @@ -44,6 +44,25 @@ def _sizable_op_ids(): class TestMmaFragTables(unittest.TestCase): + def test_every_scaled_catalog_atom_has_the_same_padded_abi(self): + rows = [ + row + for row in _load_specs()["gfx1250"]["mma"] + if row["family"] == "wmma_scaled" + ] + self.assertEqual(len(rows), 450) + for row in rows: + info = _MMA_FRAGMENT_INFO[row["op_id"]] + self.assertEqual( + (info.a_frag_len, info.b_frag_len, info.c_frag_len, info.wave_size), + (16, 16, 8, 32), + ) + self.assertEqual(info.a_scale_frag_len, 128 // row["scale_block_k"]) + self.assertEqual(info.b_scale_frag_len, info.a_scale_frag_len) + self.assertEqual(_mma_c_frag_len(row["op_id"]), 8) + with self.assertRaises(ValueError): + _mma_c_frag_len(row["op_id"] + "_unknown") + def test_frag_lengths_are_positive(self): sizable = _sizable_op_ids() self.assertTrue(sizable, "the frag-length SSOT must expose at least one atom") diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_scale_layouts.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_scale_layouts.py index 092db012f353..daefbcdd011e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_scale_layouts.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_scale_layouts.py @@ -90,6 +90,29 @@ def test_scale_coordinates(dtype, block_k, role): layout.coord(b, lane, slot) +def test_every_scaled_catalog_coordinates_match_independent_lane_oracle(): + atoms = [ + op + for op in ArchTarget.from_gfx("gfx1250").mma.ops + if op.family == "wmma_scaled" + ] + assert len(atoms) == 450 + for atom in atoms: + for role in ("a_scale", "b_scale"): + layout = getattr(atom, f"{role}_layout")() + b = IRBuilder("all_scale_coordinates") + lane = b.thread_id_x() + count = 128 // atom.scale_block_k + assert layout.frag_len == count + for slot in range(count): + coords = layout.coord(b, lane, slot) + for l in range(32): + actual = tuple(_evaluate(v, lane=l) for v in coords) + assert actual == ( + (l % 16, slot) if role == "a_scale" else (slot, l % 16) + ) + + def test_scale_defaults_and_unavailable_maps(): for gfx in known_arches(): for atom in ArchTarget.from_gfx(gfx).mma.ops: From 88dde8e1e008e63cdd876c40d859c5c87142c130 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:28:05 -0400 Subject: [PATCH 29/32] fix(rocke): filter unsupported gfx1250 WMMA scale combinations --- .../rocke/platform/cpp/core/arch/data.cpp | 18 +- .../cpp/include/rocke/wmma_scale_internal.h | 14 + .../platform/python/rocke/core/arch/target.py | 37 +- .../python/rocke/core/arch/wmma_scale.py | 6 +- .../examples/gfx1250/gemm/SCALE_FORMATS.md | 29 +- .../gfx1250/gemm/SCALE_FORMAT_EXPERIMENT.md | 15 +- .../core/gfx1250_scale_formats_llvm23.json | 366 +----------------- .../platform/tests/core/mma_frag_ssot.cpp | 2 +- .../platform/tests/core/test_arch_mma_ssot.py | 14 +- .../core/test_gfx1250_mixed_scaled_wmma.py | 2 +- .../tests/core/test_gfx1250_scale_formats.py | 62 ++- .../tests/core/test_gfx1250_scaled_wmma.py | 5 +- .../tests/core/test_mma_catalog_lookup.cpp | 65 +++- .../tests/core/test_mma_frag_tables.py | 2 +- .../tests/core/test_mma_scale_layouts.py | 2 +- .../differential/test_gate_verdict.py | 34 +- .../instances/parity/gfx1250_lowering_emit.c | 53 ++- .../instances/parity/gfx1250_lowering_emit.py | 16 +- .../test_gfx1250_scale_formats_example.py | 32 +- .../test_gfx1250_scale_formats_numeric.py | 11 +- 20 files changed, 284 insertions(+), 501 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp index 5e3b2262cb67..008a20da1b75 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp @@ -27,6 +27,7 @@ #include #include "rocke/arch_target_internal.h" +#include "rocke/wmma_scale_internal.h" /* ========================================================================= * lane-coord emitters (static rocke_lane_coord_fn) @@ -1797,7 +1798,7 @@ static constexpr rocke_mma_op_t k_mma_gfx1250_unscaled[] = { * allocation, STL, JSON parsing, or Python dependency. */ namespace { -constexpr int k_scaled_count = 450; +constexpr int k_scaled_count = 86; constexpr int k_unscaled_count = 7; constexpr const char* k_inputs[] = {"fp8", "bf8", "fp6", "bf6", "fp4"}; constexpr const char* k_input_dtypes[] = {"fp8e4m3", "bf8e5m2", "fp6e2m3", "fp6e3m2", "fp4e2m1"}; @@ -1806,8 +1807,6 @@ constexpr int k_blocks[] = {32, 16}; // Native's original E8M0 seed order predates the JSON order; retain both. constexpr int k_native_e8m0[][2] = {{0, 0}, {4, 0}, {2, 0}, {3, 0}, {1, 0}, {0, 1}, {4, 1}, {2, 1}, {3, 1}, {1, 1}}; -constexpr int k_homogeneous[][2] - = {{0, 0}, {2, 0}, {3, 0}, {1, 0}, {0, 1}, {2, 1}, {3, 1}, {4, 0}, {4, 1}, {1, 1}}; struct scaled_key { int a, b, sa, sb, mode; @@ -1823,6 +1822,8 @@ consteval scaled_keys make_scaled_keys() scaled_keys result{}; bool seen[2][5][5][3][3] = {}; auto add = [&](int a, int b, int sa, int sb, int mode) { + if(!rocke_scaled_wmma_formats_valid(a, b, sa, sb)) + return; if(!seen[mode][a][b][sa][sb]) { if(result.count == k_scaled_count) @@ -1849,17 +1850,6 @@ consteval scaled_keys make_scaled_keys() for(int scale = 1; scale < 3; ++scale) add(4, 4, scale, scale, mode); } - for(const auto& seed : k_homogeneous) - for(int sa = 0; sa < 3; ++sa) - for(int sb = 0; sb < 3; ++sb) - add(seed[0], seed[0], sa, sb, seed[1]); - for(int mode = 0; mode < 2; ++mode) - for(int a = 0; a < 5; ++a) - for(int b = 0; b < 5; ++b) - if(a != b) - for(int sa = 0; sa < 3; ++sa) - for(int sb = 0; sb < 3; ++sb) - add(a, b, sa, sb, mode); return result; } constexpr auto k_scaled_keys = make_scaled_keys(); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h index 469df96e75b8..1b403bdb119c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h @@ -16,6 +16,14 @@ #include "rocke/ir.h" #include "rocke/storage.h" +/* Mirrors target._gfx1250_scale_combination_valid and LLVM's supported pairs: + * https://github.com/llvm/llvm-project/commit/7597c0c938b364306cbee21dd03835ff5b699c74 + * Matrix selectors: FP8=0, BF8=1, FP6=2, BF6=3, FP4=4; E8M0 scale selector=0. */ +static inline constexpr bool rocke_scaled_wmma_formats_valid(int a, int b, int sa, int sb) +{ + return (a == 4 || sa == 0) && (b == 4 || sb == 0) && (a != 4 || b != 4 || sa == sb); +} + typedef struct rocke_scale_packing { /* Eight-bit encoded scales for consecutive K groups, first group in the @@ -125,6 +133,12 @@ static inline rocke_scaled_wmma_op_t rocke_scaled_wmma_contract(const rocke_mma_ ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA scale format"); spec.matrix_words[i] = words[i]; } + if(!rocke_scaled_wmma_formats_valid(spec.matrix_formats[0], + spec.matrix_formats[1], + spec.scale_formats[0], + spec.scale_formats[1])) + ckc::raise_status(ROCKE_ERR_VALUE, + "unsupported scaled WMMA matrix/scale format combination"); if((atom->scale_block_k != 16 && atom->scale_block_k != 32) || strcmp(atom->c_dtype, "fp32") != 0 || atom->m != 16 || atom->n != 16 || atom->k != 128) ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA backend contract"); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py index 872bddc68c83..6cf30216b6ec 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py @@ -43,6 +43,20 @@ } +def _gfx1250_scale_combination_valid(a: str, b: str, sa: str, sb: str) -> bool: + """Supported matrix/scale combinations, independent of compiler version. + + Mirrors LLVM's isValidWMMAScaleFmtCombination: + https://github.com/llvm/llvm-project/commit/7597c0c938b364306cbee21dd03835ff5b699c74 + """ + a, b = normalize_dtype(a), normalize_dtype(b) + return ( + (a == "fp4e2m1" or sa == "e8m0") + and (b == "fp4e2m1" or sb == "e8m0") + and (a != "fp4e2m1" or b != "fp4e2m1" or sa == sb) + ) + + def _expand_mma_rows(gfx: str, declarations: list[dict]) -> list[dict]: """Expand supported families to ordinary rows; reject duplicate identities.""" if not isinstance(declarations, list): @@ -77,8 +91,8 @@ def _gfx1250_scaled_rows(declaration: dict) -> list[dict]: inputs = declaration["input_dtypes"] scales = declaration["scale_dtypes"] blocks = declaration["block_k"] - # Original homogeneous E8M0 atoms precede mixed E8M0, the first FP4 - # alternative encodings, then the remaining homogeneous/mixed encodings. + # Original homogeneous E8M0 atoms precede mixed E8M0 and supported FP4 + # alternative encodings. homogeneous = [ (0, 32), (2, 32), @@ -107,24 +121,11 @@ def _gfx1250_scaled_rows(declaration: dict) -> list[dict]: (inputs[4], b, sa, scales[0], k) for b in inputs[:4] for sa in scales[1:] ) keys.extend((inputs[4], inputs[4], scale, scale, k) for scale in scales[1:]) - keys.extend( - (inputs[i], inputs[i], sa, sb, k) - for i, k in homogeneous - for sa in scales - for sb in scales - ) - keys.extend( - (a, b, sa, sb, k) - for k in blocks - for a in inputs - for b in inputs - if a != b - for sa in scales - for sb in scales - ) rows = [] - # Later stages fill the Cartesian product without repeating earlier atoms. + # Preserve the supported atoms' historical order and reject unsupported pairs. for a, b, sa, sb, block_k in dict.fromkeys(keys): + if not _gfx1250_scale_combination_valid(a, b, sa, sb): + continue m, n, k = declaration["shape"] rows.append( { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py index 2a51db1cb363..8b24fdb98b15 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py @@ -6,7 +6,7 @@ from dataclasses import dataclass -from .target import ArchTarget, MmaOp, MmaScaleDType +from .target import ArchTarget, MmaOp, MmaScaleDType, _gfx1250_scale_combination_valid from ..dtypes import dtype_info from ..storage import BitPacking, FragmentPacking, MatrixFragmentLayout @@ -119,6 +119,10 @@ def gfx1250_scaled_wmma(op_id: str) -> ScaledWmmaOp | None: or atom.shape != (16, 16, 128) ): raise ValueError(f"unsupported scaled WMMA backend contract: {atom.op_id}") + if not _gfx1250_scale_combination_valid( + atom.a_dtype, atom.b_dtype, atom.a_scale_dtype, atom.b_scale_dtype + ): + raise ValueError("unsupported scaled WMMA matrix/scale format combination") counts = (atom.a_scale_frag_len, atom.b_scale_frag_len) if any(count != atom.k // atom.scale_block_k for count in counts): raise ValueError(f"unsupported scaled WMMA scale fragment lengths: {counts}") diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMATS.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMATS.md index 7aa6fa345a03..d35652d37843 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMATS.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMATS.md @@ -17,12 +17,17 @@ encoded bytes; choosing a format does not quantize or convert the buffer. | Matrix operand (A or B) | Scale dtype (independent for A and B) | | --- | --- | -| FP8 / BF8 / FP6 / BF6 / FP4 | E8M0, E5M3, or E4M3 | +| FP8 / BF8 / FP6 / BF6 | E8M0 | +| FP4 | E8M0, E5M3, or E4M3 | Here FP8 and FP6 each include both exponent/mantissa formats. E8M0 has selector 0, E5M3 selector 1, and E4M3 selector 2. The reference decoder interprets E5M3 as an unsigned eight-bit scale format with bias 15, distinct from E5M2. -A and B may use different scale formats, including FP4 x FP4. +A and B may use different scale formats when only one operand is FP4. +FP4 x FP4 requires matching scale formats. Both SCALE and SCALE16 use these +rules, giving 43 combinations per mode and 86 scaled atoms in the catalog. +The filter follows [LLVM's supported-combination validation](https://github.com/llvm/llvm-project/commit/7597c0c938b364306cbee21dd03835ff5b699c74), +including on older compilers that accept unsupported combinations without a diagnostic. Standard MXFP formats use E8M0 scales over K32 blocks; E4M3/E5M3 and K16 variants are broader block-scaled contracts. The `i8` scale spelling remains an alias for E8M0 for existing callers. @@ -36,7 +41,7 @@ spec = BlockScaledGemmSpec( name="mixed_fp6_fp4", M=32, N=48, K=256, dtype_a="fp6e2m3", dtype_b="fp4", matrix_path="wmma_scale16", block_k=16, - scale_dtype="e8m0", scale_dtype_a="e5m3", scale_dtype_b="e4m3", + scale_dtype="e8m0", scale_dtype_a="e8m0", scale_dtype_b="e4m3", ) kernel = build_block_scaled_gemm(spec) ``` @@ -56,16 +61,17 @@ The public [machine-readable ISA](https://gpuopen.com/download/machine-readable- 2026-08-06 snapshot's `amdgpu_isa_cdna5.xml` defines SCALE/SCALE16 operand widths and block sizes, but does not enumerate E5M3 scale decoding or the complete scale legality table. LLVM's `WMMA::MatrixScaleFmt` defines the selector names -and values used here. The table above describes this implementation's accepted -combinations; compiler acceptance alone does not establish numerical behavior. -The expanded FP6/FP8 alternate-scale and mixed-scale combinations have host -admission/lowering coverage; their new on-device numerical campaign is pending. +and values used here. The linked LLVM commit adds the supported-combination +predicate to both backend lowering and the assembler. The implementation uses +that contract for catalog admission and independently validates it before lowering. +Compiler acceptance alone does not establish numerical behavior. Existing mixed-format fixtures cover bounded finite values and compare against an independent host calculation. They do not establish arbitrary-input rounding, scale NaN behavior, or performance. The [scale-format discrimination experiment](SCALE_FORMAT_EXPERIMENT.md) -is staged separately and has not been run. It compares identical raw scale bytes -against literal predictions and verifies the emitted instruction fields. +describes comparisons of identical raw scale bytes against literal predictions. +The fixed-byte numeric tests sweep the three selectors on FP4 with a non-FP4 +E8M0 partner, and keep E8M0 controls for FP8/BF8/FP6/BF6 operands. ```sh ROCKE_LLVM_FLAVOR=llvm23 ROCKE_BACKEND=both ROCKE_CPP_STRICT=1 \ @@ -73,7 +79,8 @@ ROCKE_REQUIRE_GFX1250=1 python -m pytest -q -s \ tests/instances/test_gfx1250_scale_formats_numeric.py ``` -Use a matching gfx1250 device, LLVM 23 COMGR/HIP, NumPy/ml_dtypes, and a freshly +Use a matching gfx1250 device, one release-matched COMGR/HIP stack, NumPy/ml_dtypes, and a freshly built C++ extension. COMGR checks Python/C++ LLVM identity; HIP fixtures compile Python-generated HIP. Choosing a scale format does not quantize or convert input -buffers. +buffers. The rocKE emission flavor is `llvm23`; the consuming compiler may be +LLVM 23 or 24, as in ROCm 10.0 and 10.1 respectively. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMAT_EXPERIMENT.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMAT_EXPERIMENT.md index 75a0021f8d90..cf1d35a9ba9f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMAT_EXPERIMENT.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/SCALE_FORMAT_EXPERIMENT.md @@ -1,7 +1,9 @@ # gfx1250 scale-format discrimination experiment -Status: the expanded matrix/scale campaign is planned and has not been run -on the current source. Existing mixed-format fixtures are separate evidence. +The fixed-byte tests in `test_gfx1250_scale_formats_numeric.py` implement the +finite `0x3c`/`0x7c` subset below for supported matrix/scale combinations. +The `0x84` probe, full finite-code sweep, and special-value experiments remain +follow-ups. Mixed-format fixtures separately exercise varying non-unit scales. ## Contents @@ -81,10 +83,11 @@ as an expected pass value in the main finite-value test. - Run FP4 x FP4 with matching selectors on A and B. Keep the untested operand's scale at that selector's unity encoding. Since unity bytes differ between selectors, report this as a separate control from the fixed-buffer experiment. -- Repeat the selector sweep with FP8/BF8, FP6/BF6, and FP4 as the tested - matrix operand. Cover both A/B positions, homogeneous and mixed matrix pairs, - and independent scale selections, including FP4 x FP4 with differing scales. - Keep the raw matrices and tested scale bytes fixed within each selector triplet. +- Repeat the selector sweep on FP4 in both A/B positions, using a non-FP4 + E8M0 partner. FP8/BF8/FP6/BF6 operands get E8M0 controls only, and FP4 x FP4 + uses matching scale formats. These restrictions follow + [LLVM's supported-combination validation](https://github.com/llvm/llvm-project/commit/7597c0c938b364306cbee21dd03835ff5b699c74). + Keep the raw matrices and tested scale bytes fixed within each FP4 selector triplet. - Validate with COMGR and Python-generated HIP, using the same literal inputs. Require Python/C++ LLVM identity where the C++ backend is available. Do not infer C++ HIP compilation from Python HIP success. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scale_formats_llvm23.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scale_formats_llvm23.json index 6f9799667ff0..c3c784f04926 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scale_formats_llvm23.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_scale_formats_llvm23.json @@ -34,369 +34,5 @@ "fp4/bf6/e5m3/e8m0/16": "03e7e256e3b5e60f13283dfd594442e586615272ad9ceba478dacfb6e4de4cc9", "fp4/bf6/e4m3/e8m0/16": "5964c480fdba4385fd92c18d15b9e2a85e0dfdf9f2f63c2f4758bef9ca3a32e3", "fp4/fp4/e5m3/e5m3/16": "01036c856966384b28eb2bd63673fac0b64fb06deb46aa3ea33973b1933da937", - "fp4/fp4/e4m3/e4m3/16": "e68ccf62c4dbccbe3a20d32399aa3d54e48e40ccd5e72d4fa52a54aa888999f7", - "fp8/fp8/e8m0/e5m3/32": "5ddcd9cdc0d562f34a0c23bdd03e9910590e2521ef94d4e139d3b24fffad3f2d", - "fp8/fp8/e8m0/e5m3/16": "729bbaa07e6c122f5ef8bfda8c699bf76a49d90e7dc08714d2a4ec73892772e5", - "fp8/fp8/e8m0/e4m3/32": "93ec0e6a7d763bf45e80f5c2f1069e5250750c5acfe1160eb76c0f6b7968c108", - "fp8/fp8/e8m0/e4m3/16": "27349a23bae88d7b57e160a587bbf7ff08d78447dc8f0d8830acda4f411b9e0d", - "fp8/fp8/e5m3/e8m0/32": "a3f3a6a947030ce7634553ee3a670270c846c80ed12fc46ce496c9c184200d83", - "fp8/fp8/e5m3/e8m0/16": "2e9248cc4cc55e0a5ca681f87526aa4852b975af93b455405a20c028b984067f", - "fp8/fp8/e5m3/e5m3/32": "c5fa0d358031ab853df0717a75ad210fccb31a0f60a5bd1e392ef02853df3ef0", - "fp8/fp8/e5m3/e5m3/16": "fa43cb8dc0ff683b01c85ea5e0578f64a12e0fe912e10befe8931ea2833a424f", - "fp8/fp8/e5m3/e4m3/32": "5fe9925c36b1aa53f9b345e36d140161b7af4e2686d33b2ded6ff06146e270db", - "fp8/fp8/e5m3/e4m3/16": "dc21d3b9a559df664b5a22b6df8c114854c1e7d745c07ddb2eda77bc6eebd5fe", - "fp8/fp8/e4m3/e8m0/32": "0fdaf2ee8e5d1f8df0b549d526ab5c906e4612f52a22dae3a637eceab3388e48", - "fp8/fp8/e4m3/e8m0/16": "e738cfb7fbddd60be96e2c5191df3e5c8699bfdfd25728cb93f1aade964fa8d7", - "fp8/fp8/e4m3/e5m3/32": "60a668bcc02a3507e8d8cd71206fa4b51a637b14f2169ac78ebe141db1aba768", - "fp8/fp8/e4m3/e5m3/16": "92d6fabbefabf7b0c9c3e1df84b527a93630b73b17b214db2f316b895e817b5a", - "fp8/fp8/e4m3/e4m3/32": "92222e6c3d0a38def2058f81661d588f36cb1b4218052c99898097ad3bb9092f", - "fp8/fp8/e4m3/e4m3/16": "66550fd3d90acf9221fc3fea5adf3206fee1f01830a43c58f15dfdb40b53448e", - "fp8/bf8/e8m0/e5m3/32": "baecc775fa51273e6e3993f31f8dd1a9e873f2cba26c588cc891bf2752f398f1", - "fp8/bf8/e8m0/e5m3/16": "537a720869a0afce1e56a22a5bdaa2150508587d74a378711eac456a7bfbb5ac", - "fp8/bf8/e8m0/e4m3/32": "ef9a3f839f9144f59b9d887a44aa3128995d7d53c86833c26e4017cdc39ff1dd", - "fp8/bf8/e8m0/e4m3/16": "efb0c6f3ac1a966a82b8abd3b3965da9bf284f8382458a52815320a53bd73641", - "fp8/bf8/e5m3/e8m0/32": "a7ab5c9220709abfb3b93218e114f8ac467ab8d3f256ebd0a89d40bf5618eb46", - "fp8/bf8/e5m3/e8m0/16": "cc833efde8cfcc429aa623ab7b3a57712da1c56b078fc27552a2a9fee51f7a76", - "fp8/bf8/e5m3/e5m3/32": "e20f6763bf2756ca0905cec15e2ef71e6aedc95c85eb4a0e9a944f1094dc08a6", - "fp8/bf8/e5m3/e5m3/16": "5d383f520f735aebc98cbd358ebbad92070b6af62d3fd97eb7ce52ce85a88086", - "fp8/bf8/e5m3/e4m3/32": "aa1ab94069e76b7b426c82f9eed7ba8a015ca38d85f8c66aee665bcffb960364", - "fp8/bf8/e5m3/e4m3/16": "8313c5c34de8c4ca292e39130a7158b2ccf79ec4df74c91d2f1fdda2b775bf0d", - "fp8/bf8/e4m3/e8m0/32": "93b2d7c755d307c85912604b8935d3242d5fb41f2a846668bc4985c6b1e66d4a", - "fp8/bf8/e4m3/e8m0/16": "52cb586d2f50b522a48fd138ff72eecfac004ebcf2958e5e17d286e039a722d6", - "fp8/bf8/e4m3/e5m3/32": "dd0e4ee57541076c13f09668fbc9a17d66324351c9e364a64b2c14b1378e2a70", - "fp8/bf8/e4m3/e5m3/16": "7c1525e4c8f6448f9430f913625c2a711e1f9436089738653653f304152bea60", - "fp8/bf8/e4m3/e4m3/32": "6cea2a69a2f3b250024ecdf9d19868fc8a7a3cb15ebb2f678ec69085462b8b9b", - "fp8/bf8/e4m3/e4m3/16": "8687c3de8d5e8a64254a7e9787ac8bcdfbfb2705c26661a0607295c7317bccfb", - "fp8/fp6/e8m0/e5m3/32": "0a6244431d32e516373d8e6568065cedbf43519b04949375403f552ae10dd7af", - "fp8/fp6/e8m0/e5m3/16": "ab5e5613e9a70f7988bdae63fb71dd0daca1b57b2cca4bfe56a1808df5b7af34", - "fp8/fp6/e8m0/e4m3/32": "a44cc7caea604794bacc6e514e82a333da07c1bcdc9508c01998879dab6c5480", - "fp8/fp6/e8m0/e4m3/16": "c6516f22bee494cdb4282aa1df22b9c027fea32744f81d9830ec51cfc52a30a8", - "fp8/fp6/e5m3/e8m0/32": "f7c4594e03eece7299b9ae42e37a3e75b5ad8ff8a6366748eab32f0226eca273", - "fp8/fp6/e5m3/e8m0/16": "0ed60656771791810c66a2788e280c5cb6fa133cd2f53d789ece087270151aca", - "fp8/fp6/e5m3/e5m3/32": "eb29a85078c3402fdd38452b96a4f3d6d71714d12569bd488cf07a369002752b", - "fp8/fp6/e5m3/e5m3/16": "ba6de49079741ca210396b1b8273ef40ee7449d3446c5bdd275df421b8951198", - "fp8/fp6/e5m3/e4m3/32": "e0b19bea3a5c80d7c18c0c36aab334671c89d9d30ada0878cfbeb3a3efdefcec", - "fp8/fp6/e5m3/e4m3/16": "f781c95a9087f8f2ecadc4ce3a43ac07add4047e7a4506e8443e5faff0809b92", - "fp8/fp6/e4m3/e8m0/32": "997a4e38b3ce3c9a39c60ec8f347ae5935256b05957ceee9c904c2333af7d6a9", - "fp8/fp6/e4m3/e8m0/16": "af6d4bd5f08cc97a58fe2c89c7899f6520f3227f49f6c86864f8d64c9370eced", - "fp8/fp6/e4m3/e5m3/32": "204e973eb30d6949003aa4f10e0c4f5889a4b55ec7b98a3efeb427c12223b87e", - "fp8/fp6/e4m3/e5m3/16": "de265db176ed45a95a0e4ea17731f3334072947c06c29f5dbe12c84d1acd3a43", - "fp8/fp6/e4m3/e4m3/32": "12ad1db41fdf4674386ca2a3d989d5d187ed41d87f95474c68d3a0b691e3480b", - "fp8/fp6/e4m3/e4m3/16": "60e6c760e789cf7895e951fb592f662e0bbb931874cdb06954067d52e348e8f9", - "fp8/bf6/e8m0/e5m3/32": "673c1e64cda2c61562dc7625ead9701d560daa32b29b97815aa1c4616043b8d5", - "fp8/bf6/e8m0/e5m3/16": "42ef038f21f8527682d18a9559980cf96311aef5da3ffdb67bee7051b3ed3717", - "fp8/bf6/e8m0/e4m3/32": "5ec743742361b41886c0b476caaab679144532312857794364c3b90b7e934421", - "fp8/bf6/e8m0/e4m3/16": "94bd2a008ce634a17a79a9f5f6ec5aecf130dfd28fa517d5eb6c82f278bfa972", - "fp8/bf6/e5m3/e8m0/32": "03f8a57534b7a8c0616c4308836fb4cda15af8cda808a84d47ea26d120719171", - "fp8/bf6/e5m3/e8m0/16": "062a9fde78a343d4cc22cabcb8ace71559c1d38720ab6dd5293615f9f1df8bfc", - "fp8/bf6/e5m3/e5m3/32": "8765bfc38712f6f64f4e5d0bc23c830f24f577f3e24f180b18bc092b04f997a6", - "fp8/bf6/e5m3/e5m3/16": "2089430ec1bfcedd52446fb9727d646518d2ad97064c4a97648b75def078c9a0", - "fp8/bf6/e5m3/e4m3/32": "27c54f3df7c3649921d3909cdebb74c62fbce099420635ef773bc62dee50d8dd", - "fp8/bf6/e5m3/e4m3/16": "4f9184db0f2f9935eea39d0f15cb0149164859c31cf21040f78b646434d28314", - "fp8/bf6/e4m3/e8m0/32": "159e32591bde664d37482d491443be573085ba1b6eb9b3d0d8d98513b65ad589", - "fp8/bf6/e4m3/e8m0/16": "d5dfa14c4b096813faa6789696721e866a15d428edd4dd1b47b01b2ae33f4d38", - "fp8/bf6/e4m3/e5m3/32": "272043b8c153389ad91162a0eb54a83a13b8753d359cce74daf57fb31e43479b", - "fp8/bf6/e4m3/e5m3/16": "0baf2836d5276765bc686c19dac3f59994c767c4a9b1b9337e57d9f0747f4f68", - "fp8/bf6/e4m3/e4m3/32": "42ecfab98d0244eebd0da4867921fc7df6a1762a6508ed62eb40b971d5e7e03f", - "fp8/bf6/e4m3/e4m3/16": "994d3154164df61e1550a4b4a05e767f0fefbf3b0fe9ea2d86850dec19427ea6", - "fp8/fp4/e5m3/e8m0/32": "f92b362b4a3a7cf0be55963828e130b7a859cac23607879117e30ad15eacd308", - "fp8/fp4/e5m3/e8m0/16": "1374f9e0a34040fe321893dd8a6126b9bfa23c31b72e0e4f2b7d8c2105dfea0e", - "fp8/fp4/e5m3/e5m3/32": "c1e330bfd49f73eeba60afe606834ae162573d230b5bc369888c1ad32ad659c2", - "fp8/fp4/e5m3/e5m3/16": "0106be64f01459d4a7a5bd9465e5abbf7295be6f47402cc73f4e758c15ad12cf", - "fp8/fp4/e5m3/e4m3/32": "15fde1f2748a3707479953117622e9134deaf5a91561343dd71677893a21cca0", - "fp8/fp4/e5m3/e4m3/16": "7d59eb54e807e1003d44f3d5361c66335a7579531443d7397a523a3c866dfedf", - "fp8/fp4/e4m3/e8m0/32": "4344f204f496fc87e6e80b9af2ad3780eeb2eef941faf7394406eda8e2a5a400", - "fp8/fp4/e4m3/e8m0/16": "591459a4f964cde8bf37e2c24ae094a99aa44ea290cb253e27931d1c191b5551", - "fp8/fp4/e4m3/e5m3/32": "410c90816872eeb5c1f9afa253e9973b7287077ddbe7a11594071636365fe2e8", - "fp8/fp4/e4m3/e5m3/16": "1e4668a283b1b6028c981023c44a34d678ab77209434c65d7a0b183ad0262fea", - "fp8/fp4/e4m3/e4m3/32": "2178e9f0caa117ba57c5b26c1ea2a55e6773fd417bd51e903c400bfae9b90168", - "fp8/fp4/e4m3/e4m3/16": "04b29350c2cb555a91ff21b1cc4428a0ec2c562076fb7855ed29ac9a09df0034", - "bf8/fp8/e8m0/e5m3/32": "e11cb4bc37eadd457348522ce2f167f10d619fae33e89f7e3c75e6b1cef2adf8", - "bf8/fp8/e8m0/e5m3/16": "ec52ac7258864c641f3e60670d9719edb290194c20083d933ca071769ff498b6", - "bf8/fp8/e8m0/e4m3/32": "48ce41b3de4bcba3ccd07d45e68f6f52c308eadfc5903fc7faef818fcc00fa3b", - "bf8/fp8/e8m0/e4m3/16": "044e25ec11bcb582389efccb111765fbdb323131500cdb5eecabb7e1f2599460", - "bf8/fp8/e5m3/e8m0/32": "a85330207dccd9fe29f796bdd333c08d23c87ed41bd92e7831b59d1a65fb1d56", - "bf8/fp8/e5m3/e8m0/16": "fe19a27cc91bbbbe5d5f387c0c5309c93233c4306a3329188d27bac918505bff", - "bf8/fp8/e5m3/e5m3/32": "8251a946c1f24d38a24c9d9e4e472e28d672cd762fc4ba1d7df42542508e880f", - "bf8/fp8/e5m3/e5m3/16": "d4a90e80fa26a5690f2c14d49ba1e46c34234fe2f42dfb92b1da34e36f589b3c", - "bf8/fp8/e5m3/e4m3/32": "388f620742bb0bd22cf481f21fb75f6f6e5c9cbb32a6408af5bbf8fba5484a6c", - "bf8/fp8/e5m3/e4m3/16": "338ae4efd137b57de6dffb5dad83d05b76250afbe4a848b9b9db030348e15ea4", - "bf8/fp8/e4m3/e8m0/32": "fe435af741d3dc6ae50e63586fd9ba60837068dff4fd1b54939ca7f636899c52", - "bf8/fp8/e4m3/e8m0/16": "c150a339a8e87cd73919466c57ed9e137d7f522206b8a3dafde36d784837ad3d", - "bf8/fp8/e4m3/e5m3/32": "2d8688a84d87c30b7ac5df45b751a335aef80766dd4084ad16fb11c1f9bf6411", - "bf8/fp8/e4m3/e5m3/16": "e12f8b6706dcf87a4bee965b9a62013b105a419a9d036290e207173d84cfaa15", - "bf8/fp8/e4m3/e4m3/32": "7cde85a8463048e39d7549280695d891728400bf090a7e58e385ad5bbacc3745", - "bf8/fp8/e4m3/e4m3/16": "fce9aeb6bedf5cb1e86f85504721d003861a6b36041eb97738b607e20728bb79", - "bf8/bf8/e8m0/e5m3/32": "4ce92942ba7b0fa107d8c6d5e9d9d824f1749af050d0a39e89f04f46a395da02", - "bf8/bf8/e8m0/e5m3/16": "5c0afd62befb519374045af72bd4b77e5e75c4a085c00a9025bee435f54c40ec", - "bf8/bf8/e8m0/e4m3/32": "bc9a4ecd5089272d7b26e972b448077e1d2690ae14cc673ca74e243b22b812d6", - "bf8/bf8/e8m0/e4m3/16": "31bde40571ae1b1e47dfe8ad098c1e7dddb9f6d66dc3f7d67ff4dd8956d4a04d", - "bf8/bf8/e5m3/e8m0/32": "0f2e97511fcdb0ae17c89ebb4141f873d4b6350d4f6de9f763d6b2eda855295a", - "bf8/bf8/e5m3/e8m0/16": "beaedb14b05e9fb393e2e2a21718617665731f49ce65ce1e0f7f4446a0e54234", - "bf8/bf8/e5m3/e5m3/32": "226bc1339e8ade1c2e25e47b89a650cb4f68d30c35fdc77ddc289faa9a227617", - "bf8/bf8/e5m3/e5m3/16": "3ffd58442dc026043d0ea36e90d65a1f0e6b4a1fff9c6b34d87076319ba2d963", - "bf8/bf8/e5m3/e4m3/32": "095324f00210afedd6819e69a926ce6f089580efe7bd08cdead05c50d9c59ec7", - "bf8/bf8/e5m3/e4m3/16": "9b63e03b1126411aae2fae7f939a2b8d8065cfbf539f8dad0155b6eef2f1a521", - "bf8/bf8/e4m3/e8m0/32": "9a12f815df269fc8ec985a58af4375a53a90cdcca759025261f6832be6d7dfd4", - "bf8/bf8/e4m3/e8m0/16": "763d4504ea575af80b17d28d83b5132417dbc3f7addda1536f9dd916e7855d27", - "bf8/bf8/e4m3/e5m3/32": "3f758c978459d126a18b1ca0542ac7f260ac2d9951ac83d35e5905828236b6d5", - "bf8/bf8/e4m3/e5m3/16": "c95e3876329747ef8e2980e9354e63b20eae9321c8daff99a518db2a89885f34", - "bf8/bf8/e4m3/e4m3/32": "4fe913efc699cbd96adc318c94673d3a069e0e739a021a797e1b1ca68d634a01", - "bf8/bf8/e4m3/e4m3/16": "1fd3796a074ddeaf514b3b139d763ae18a32c876db81a4671f59dd63a43147ab", - "bf8/fp6/e8m0/e5m3/32": "6f9e0abdc2382c745c87d827b88b4455758e712ab9adf0c438c8214eca385630", - "bf8/fp6/e8m0/e5m3/16": "79709e3e5be85a8e1cb211bd019883c902ffa13a5ec67a55a63d42dcd1dc9d7d", - "bf8/fp6/e8m0/e4m3/32": "03e8ace4cc325ecac8aeca4454e65b0695bb4d8511013f4a37d820d6f68a94f8", - "bf8/fp6/e8m0/e4m3/16": "c572e96d040275154b17df85b1dfa5d601ab20de7f24bacc6b57f24814237498", - "bf8/fp6/e5m3/e8m0/32": "442263b39cf5e1943c2a06d0f2420234c0f07d0150a4cf396fa0aa697c634506", - "bf8/fp6/e5m3/e8m0/16": "a05d1b3200bcc4db907df428424e8ea4af63f10ea8ab1251d0276d5c9f05c42d", - "bf8/fp6/e5m3/e5m3/32": "7f7ca686636084704be39976f943387fe686df8349d31710ea304a5478464b91", - "bf8/fp6/e5m3/e5m3/16": "b0dd47c0506e2b12c35dd66c27a9bcc05e5520bade06aa5983833edd3a3bb4b1", - "bf8/fp6/e5m3/e4m3/32": "cc1ee5dbf9575fffa18dabbf2803cc7fa5fb60c9e3e57b792d34c8efe2c2fff8", - "bf8/fp6/e5m3/e4m3/16": "eb596c463ddddd3ef96b26b84a0123817609199e8b538f6da8fd81f34ebeb684", - "bf8/fp6/e4m3/e8m0/32": "509325bd4916807d5a2363d1064e3e0f145f4cecdac89fac13d339f79d621fa9", - "bf8/fp6/e4m3/e8m0/16": "89ddbab05b007646522263d6d7beb8d5fc5d819303c78b9e75bc5869289ede15", - "bf8/fp6/e4m3/e5m3/32": "969e620fe328e1672fb4b09e8da9c92f138c33bc02c0ca29a031faee82b59bcf", - "bf8/fp6/e4m3/e5m3/16": "7076bd5b1b3030b0deb91418d593b7148294d65c19551f080b7a0e458bf9d035", - "bf8/fp6/e4m3/e4m3/32": "2c5e26b4b974e45bbcdc7a8fa14526f512696f9bb8d0ecf772286d017251375f", - "bf8/fp6/e4m3/e4m3/16": "a5f7be080cad581212f8a4747345a36f0d4058cb43b38843e748ac79b11a7901", - "bf8/bf6/e8m0/e5m3/32": "4d8c42c921d1791316e1fc6c799cb1aaf8945d9022b4cfb5b5e1b1bb07a8463e", - "bf8/bf6/e8m0/e5m3/16": "dc187d5dc0c837b74814f292d54bf82804424c7551d9ac390ad4f369c628d6e6", - "bf8/bf6/e8m0/e4m3/32": "ab0056577a857b77b84b14856ec588edd34c98b27e158d72db16cb6bd90381c9", - "bf8/bf6/e8m0/e4m3/16": "5fec446d4e8055abac6b49e34b42e2384a8a79866dd002ba3bc97dc0bb6883d3", - "bf8/bf6/e5m3/e8m0/32": "6924570e1b69546832b3bb5c20d4a23153ec9d4a99b344ce496bae5bd67c3281", - "bf8/bf6/e5m3/e8m0/16": "d0350a4ced88a12e6a534ac990957088a39eb12a9b2c72646b5c3bd192ee4d82", - "bf8/bf6/e5m3/e5m3/32": "b24ee77f10d8f404ce61362cea341a7e054fd8fec2595e685d4f388969f1dc6c", - "bf8/bf6/e5m3/e5m3/16": "18229bdde379841a09724ca123f606c6a2357148e64c1a88535e2cd5e52dfcb9", - "bf8/bf6/e5m3/e4m3/32": "acebe7aa67f8b442fb4d73d51e16a7df1a0716b078b92dc53fc72cc33e215697", - "bf8/bf6/e5m3/e4m3/16": "ba4063caec4e0769897a131f41d71979f9b7a27513ff590698f37ac358427af9", - "bf8/bf6/e4m3/e8m0/32": "c92a9100eca08ad01603a6dfae321258ccde817141a3fa88772b6e84ffed9ac4", - "bf8/bf6/e4m3/e8m0/16": "6059dc8d4685b6f8c18c92d0edef0330cefc8876b2af05a5c322a678739a903d", - "bf8/bf6/e4m3/e5m3/32": "5c190742123c4524fb5c7902add5605131f8b66850660fd3231cbe60fc5b1041", - "bf8/bf6/e4m3/e5m3/16": "64143c43e1784faef1203cfc95938db9d2a6c8362aad8e241bcb9c757164a2c8", - "bf8/bf6/e4m3/e4m3/32": "3c1241a21463263a76178a837ed40a470ea57388e89c5fe8b7a6ccf0fa864a1e", - "bf8/bf6/e4m3/e4m3/16": "5377f1f0d7a8c3e3179f0043b84ac2f38d0d6dd8ef7ed38e62d4c8eab3b50c1c", - "bf8/fp4/e5m3/e8m0/32": "fbf4e8354f55df92972b0e4783b4bb299131317e800a5ac391aac2c5453a3696", - "bf8/fp4/e5m3/e8m0/16": "ac162516eb094a87d8437e5b7505711ddfd0385da7995f64cde16a9caa745209", - "bf8/fp4/e5m3/e5m3/32": "1843f2c15a4e29c45ff045ce38b6bf52aa1d183b5dafc09b6f632acc917b6da8", - "bf8/fp4/e5m3/e5m3/16": "926434b4498d80f2fbfce7711868d9dbde84eb88c501a4a2ae63dfac54156886", - "bf8/fp4/e5m3/e4m3/32": "97bf8f32858be904fe91234e583b28daa31a765542cace3f9277d252a70dffd8", - "bf8/fp4/e5m3/e4m3/16": "83faea00befaf286a8cda22e516e32e78e00dd1c63b4caf488c5bf271ab57f15", - "bf8/fp4/e4m3/e8m0/32": "58680391730f047e4022c05059738c716af65ee45c6df57864fbbda1d2df7958", - "bf8/fp4/e4m3/e8m0/16": "f5ec2e991e29dae40eee1b651849e79becc38dbc09698fbbb848ce0c1659cdb4", - "bf8/fp4/e4m3/e5m3/32": "e80bdf5cc96ce5e01650daf42fdd13ba8b6211d4256d27b1c60f23b0ff1e59e3", - "bf8/fp4/e4m3/e5m3/16": "62029fb2acf8849a0af6fad0e169cb9eb3d07be320adfde7ae3590b7815bec61", - "bf8/fp4/e4m3/e4m3/32": "ab2e08ab3902a34a6de565662ba042ebfaaa405c00572e9cbc51a940d8c2d4b3", - "bf8/fp4/e4m3/e4m3/16": "ffce5de9d33a21e610bac1e8bf37eaf296361659adc7dca47f39cbb1fca91ed1", - "fp6/fp8/e8m0/e5m3/32": "03816060815a4c0a19fd53fd0b6dba826b87dbc71f8631fdba41b0bdc880e5ae", - "fp6/fp8/e8m0/e5m3/16": "d1b6dfd6ef7e319c512fee90903aff7f7d74ebf2d0b75837d753c81835f52758", - "fp6/fp8/e8m0/e4m3/32": "f72e15c42946feac252785f4e7f81ce400cac894d8395f5588f3de81fe6b06c2", - "fp6/fp8/e8m0/e4m3/16": "a85fc9cee6c2e4773547f3ae741748e4fc9f4098d3a974a18664788bbcf7a8cd", - "fp6/fp8/e5m3/e8m0/32": "cc2ca6b1f5d8007a0df575c65ab0b7339f54273bbd59006d6ee0512ee8c90fe0", - "fp6/fp8/e5m3/e8m0/16": "bc9d7bbf2037e9f7ba06a41a7abc43b2203bad41a5062918c02a81a3b59106c8", - "fp6/fp8/e5m3/e5m3/32": "a8b8316a337f3a6d7680dff26ed010d16e1083aab2511952b68454f018b9df44", - "fp6/fp8/e5m3/e5m3/16": "0a3ebc42eb8b6c721b91aeb7ba599e1346f637b9ea5b5193171ee5359053d1ce", - "fp6/fp8/e5m3/e4m3/32": "8f434338e315e85826f759090bfdeec267e554f978e4917c0e046aab5f648b90", - "fp6/fp8/e5m3/e4m3/16": "4608cf7188164e9351d41cf91e0de261cc66aee33a4af4b511936ae985b297d2", - "fp6/fp8/e4m3/e8m0/32": "5693da683ebf0b76054d646c1148d21a682bff22c8a1d8e3afdb71cf73034059", - "fp6/fp8/e4m3/e8m0/16": "3701a1066a696ac4188a5381613510c6119659f36e443c37b27455521dd00278", - "fp6/fp8/e4m3/e5m3/32": "05a9ae5da9c61af8d3321f4e18aa43c1be7eed995ebb83c5c36f101d76d6b226", - "fp6/fp8/e4m3/e5m3/16": "f9ded0874dd3ae03bb05afde2e83c8f424de6864bfe23e0ebf778c0137231f48", - "fp6/fp8/e4m3/e4m3/32": "dd219a11f7c479e1af40caf6d5cba7eea55154e28523adcf80b02d19b2cb8c92", - "fp6/fp8/e4m3/e4m3/16": "cf77ed532b9a3e5317c7e1f1c2bee3f34d4ecc8fb8914397cd83ad87f6d6e228", - "fp6/bf8/e8m0/e5m3/32": "3722e0b7d696c0a30fb0ab329b17cf9dd930741d336a8d05ddca02d3bd34d3ff", - "fp6/bf8/e8m0/e5m3/16": "ed0112c6be381a62d406558af7d37f38306e08cfba1c2413cadb7bf3cea81ed8", - "fp6/bf8/e8m0/e4m3/32": "ee9786b6cf4538ece59ec5eb68fb149673bbf434dcc4f96bfbec7045850176a3", - "fp6/bf8/e8m0/e4m3/16": "eac8398d9e85320ba2c43b6f7074459a357361a3b09951d4595b8c6e37438947", - "fp6/bf8/e5m3/e8m0/32": "42d19f2a7d228e2bcb564e4c8768c5b53c3a7bf9ed25f90551258e1a8a74486e", - "fp6/bf8/e5m3/e8m0/16": "bf26178604eb4ac39d7ba906c9dcb2c8e1eb35649e05ce24460137010c2f8831", - "fp6/bf8/e5m3/e5m3/32": "f728cb07588d21391468afefa65f251e15c23de2e4646841cb9cd9d1e8cf12db", - "fp6/bf8/e5m3/e5m3/16": "3974c59cc8b3b7362f3d18f6c908d4132a587463eaf6df926aff44f430f662c8", - "fp6/bf8/e5m3/e4m3/32": "50a2e3db8bcd66dfa6d2d6c001a78536439b9ee763d7e97887dd954d489940a7", - "fp6/bf8/e5m3/e4m3/16": "c5a7cdb63207deab4f41b6cd0f081b239ac5f422b53ae75d2f1829d4fa0490f0", - "fp6/bf8/e4m3/e8m0/32": "819a883eccdf83d2fe4e182c2ca776f4c3769770b5b3d38779c35ba36ab79584", - "fp6/bf8/e4m3/e8m0/16": "6fbf7fc13e7b922e9a247c57b6b5c90011db06649137673e0dc6f22393d25b9a", - "fp6/bf8/e4m3/e5m3/32": "7e65d74ae1240e9c8067f00dcd53d0907b39f3f8fb1b5619ec19c3286440aa2c", - "fp6/bf8/e4m3/e5m3/16": "0e76804563085d38139e105c89e0b4a5df4c6b30ba2adf9dabd3e61511ea3d2f", - "fp6/bf8/e4m3/e4m3/32": "7935094b6d42f684510f9d24cbbbe50bae94a5e6a39412e688e792acc8f22129", - "fp6/bf8/e4m3/e4m3/16": "e1e69aa540f8a5169dc8fc773a6dc4917bcbf91c9fc23c2c36f32aeeb3493c99", - "fp6/fp6/e8m0/e5m3/32": "7f99d0c4df85319067e83094052424c5dcc20d7feeb7d0beca8de2ba653089c8", - "fp6/fp6/e8m0/e5m3/16": "a30202e2e3801f91c41aa7175ff8a63cea15e537ba2c918a0438abdaa4317d8b", - "fp6/fp6/e8m0/e4m3/32": "e4af5cda8447e5cc9e72b8a1cf2409718b5e1b03d6199200b0798c02c6c56137", - "fp6/fp6/e8m0/e4m3/16": "31136793649b7dfa54964f7df0dffc42c8e0dbb4c63a08685be786b40e797249", - "fp6/fp6/e5m3/e8m0/32": "7ffbf5ee4c7ad63c76cd44d4d0ee3a9416f689a068a46165f728059eb07c8dde", - "fp6/fp6/e5m3/e8m0/16": "3487c39e6547670e0dc1f0bab7f87564f54e278b37d7f2b030d9a63355cfdcc1", - "fp6/fp6/e5m3/e5m3/32": "f8f6c109db242f9521091144498199d99d792392dc8451fb4a7b1f542a959d35", - "fp6/fp6/e5m3/e5m3/16": "9169bba294e3c0df7845e0436d7ccd346a1f457e61beb67c24383324ee64a98c", - "fp6/fp6/e5m3/e4m3/32": "5898eae101249d8f3e1e2c749acaf04eaea7f5a81949224b64c843caeee16b47", - "fp6/fp6/e5m3/e4m3/16": "f5ae46a0076af03f4e8e31720cfed9d0396befcaa7c1c5e366610aace4317dec", - "fp6/fp6/e4m3/e8m0/32": "d9c4e47cfe4ce8d255f15b565f7ecbc3b42e18aee4dcf0714696bed4e824e859", - "fp6/fp6/e4m3/e8m0/16": "6144dbd1ad8f4640f579b31129754449001ea47eeec554ec67685dfba75c424a", - "fp6/fp6/e4m3/e5m3/32": "ee395430e11f584fc1dde4993afabd3640ae32ea3ee09f26dffe0e03a616692c", - "fp6/fp6/e4m3/e5m3/16": "90ca3f03545cec8d51312be343d13b22bf5067c421228290d4db769af472f767", - "fp6/fp6/e4m3/e4m3/32": "0753297fe26006452f308ae9245125cfba0de8b3465605327402097204fc7d93", - "fp6/fp6/e4m3/e4m3/16": "2a62670cd685844712153180cf379482aec7a964d9e6892d5354e01fbdc8f2b2", - "fp6/bf6/e8m0/e5m3/32": "105348c2e78b82a4527ef1ac90c71499258b317b74ace2352d390739ea005a0e", - "fp6/bf6/e8m0/e5m3/16": "31713bbd9b445cfe7f6bd346701ddac7f9dc925db55a6a40cf50c2cd7c5b8cda", - "fp6/bf6/e8m0/e4m3/32": "c8b73a6878378701abe14c87f130dff9ce05c950a54da4a9094322c490b0e95f", - "fp6/bf6/e8m0/e4m3/16": "cc5d92ba276315961bdf4bf7dce52429f621855cc6df48f37d88a40c4da01adf", - "fp6/bf6/e5m3/e8m0/32": "ddacc734ad139b1512ab3d6b0b71e41615375b4df23f14fd85d808d81ea7a5b4", - "fp6/bf6/e5m3/e8m0/16": "a7c46f2e6b333888d4cec2443b5c3fdb3346e9495a8bf561f6c3f932af2c8ab1", - "fp6/bf6/e5m3/e5m3/32": "9810a711a39b9e28567362157c0f930e7dd41840929a3b58b77315d7927e36b2", - "fp6/bf6/e5m3/e5m3/16": "f9104ac1d0149b41b419501856ba1843e6f1cca4c308594c45bec0c8ae363642", - "fp6/bf6/e5m3/e4m3/32": "bbf45b765be001271d1ab603bb7b11f575146e17749bf2bb133b0564b06f05fd", - "fp6/bf6/e5m3/e4m3/16": "05b7a59fa31a3fa1aeea427afb90442e3c78d6cb4616252d729e7ce04bb1647c", - "fp6/bf6/e4m3/e8m0/32": "5d543f34f98f02e7312e91ee07816570d2191e7a4646e9ca08a7f1a8a7123916", - "fp6/bf6/e4m3/e8m0/16": "cbf800ca968fc1eff36ead048c0dcd1484def3b5c95693ceb3780fb3b8fb7419", - "fp6/bf6/e4m3/e5m3/32": "1e25496d4d633099945a593599708e0d19ef73bbc7120b2bc8e9bda88602d656", - "fp6/bf6/e4m3/e5m3/16": "68ce52e9a257a3ef6f4bdcf63570935a2050af336e049e61a875465d5341b082", - "fp6/bf6/e4m3/e4m3/32": "d511b206187727267a09ef36965abe8d239a84c01e114d683728ba1154d20533", - "fp6/bf6/e4m3/e4m3/16": "9105abdf342ed0f0af7240b1eec27d84d2eec50647f3f9a4724c56a0003c8a02", - "fp6/fp4/e5m3/e8m0/32": "faab9d36134ffdb8f062c7f7c93952752505fecdbf7f9da88134fbcc7029e17c", - "fp6/fp4/e5m3/e8m0/16": "d5ab4c07a20bf2627af636cf9794388bc9addc6871a65c5282300934a5661f4f", - "fp6/fp4/e5m3/e5m3/32": "bc32296a57e0ec7cc62ef9061aca229919cb0e00cdfee3091bb24640871b2037", - "fp6/fp4/e5m3/e5m3/16": "fb54df7c84d95a8aec6723dae30346c4d9bfda81675d3684966b5507a751049a", - "fp6/fp4/e5m3/e4m3/32": "925a151731a02c2e774400ccb71dc4efa42cf785c65322e54a96d3f7c0596d78", - "fp6/fp4/e5m3/e4m3/16": "6c055810b436a0b0eabaaf67f7f3fc9c4bc0ffa7d6cfb6646b4da98fe8949a3c", - "fp6/fp4/e4m3/e8m0/32": "24cc1afd74062aa8eb7f0309e64c18831cb6fcf198c204274c0000bee25bd785", - "fp6/fp4/e4m3/e8m0/16": "bbacca550b876e1e3af0f18af6e45d76b69094fb7f8c135bdb3d39b6f1fa6913", - "fp6/fp4/e4m3/e5m3/32": "dd1c68c367424ffc073609aa7fab8104f4b1ccae8865a2f6c69a177c70f59e80", - "fp6/fp4/e4m3/e5m3/16": "026c2b41ff0a21c31781ed985d0059ad631eaa269f554053c574c77798dfe038", - "fp6/fp4/e4m3/e4m3/32": "273cba267eeb719cab6f11dbf9d4b66be170e8a81df8f2c8090183398ddb09a2", - "fp6/fp4/e4m3/e4m3/16": "e409615a9fcef589d3e749ea20f39603b2a51c3d0c41c5e2cdb74a8926d82ba0", - "bf6/fp8/e8m0/e5m3/32": "eeb24b60744ef3886b940d40b00362cacc6f0c956690d1af202861c496d8e4c6", - "bf6/fp8/e8m0/e5m3/16": "f516ad9a2e2c1f14445e887314e0312f4e561bf7433f0deaef472aa78cee4ebd", - "bf6/fp8/e8m0/e4m3/32": "3f36880529be07fa8c83ee6f3cb8ff74c65337415038b98e98a8882c18295c0c", - "bf6/fp8/e8m0/e4m3/16": "1caf5d5d51279b059f212ddccb79b68aada0d107076c73b30bd1ef54bc656abe", - "bf6/fp8/e5m3/e8m0/32": "0c831457ec4ff89c6540f2a420138531c46169d947dc6088bcc07c9b9763ac37", - "bf6/fp8/e5m3/e8m0/16": "58d38bd0b44697fef32d985eafbac4884e0baa560d508be66c529398ec638d1f", - "bf6/fp8/e5m3/e5m3/32": "155f5e0ef1ef634974dd8f3007cba8883c18a5d640179594e33027145c161465", - "bf6/fp8/e5m3/e5m3/16": "af529718d9672089cf24114236a39f41eb0881db22808e20cc818a0f0e6fe367", - "bf6/fp8/e5m3/e4m3/32": "aef3375f13c97957c05b82ee29d6ef3a9ec1d711496571caca89bd8cd3717413", - "bf6/fp8/e5m3/e4m3/16": "71a115567c55c940fab3c10000a88b099786627b9f001521c535913c79381498", - "bf6/fp8/e4m3/e8m0/32": "1d677d86029a35f708b48171adf09194008d781ed1aa75e81161f66ddfc5de5a", - "bf6/fp8/e4m3/e8m0/16": "903026f2636a14a756a91c1becc1c40ecdb82c63ec62015831d15f575e77c0b0", - "bf6/fp8/e4m3/e5m3/32": "8e914dbf52426e3cfe5dcf0205481662403ff296d2d3329a72988bb4d1ed8279", - "bf6/fp8/e4m3/e5m3/16": "ac12968ef9db085b93a8543e61a8f7494ca79dbf4c572e717d9b8f3792fa776c", - "bf6/fp8/e4m3/e4m3/32": "01e938f675cf4ce9a7cab3fb76a7e44274926b0f11dc5b04e331d2cd4fb046ee", - "bf6/fp8/e4m3/e4m3/16": "f1f7f8f3cb504786e3da327b0c80134af48a178a36b880360a864015ec0aecb1", - "bf6/bf8/e8m0/e5m3/32": "283b7f069a4b2339d928f16de5733ec0f339a2a24e443f4b5bf76982c5f88dba", - "bf6/bf8/e8m0/e5m3/16": "01fc8e783ec1e1f975a1bed6db5ff2bdc7586560c1d727425e1440077224b1c4", - "bf6/bf8/e8m0/e4m3/32": "2f91197fafa4bf501f5478bcbfda2b7b7e36d97e86b22eebdd70d82c866941c5", - "bf6/bf8/e8m0/e4m3/16": "471550447702ff2d41c4d156aa5e1142daa29e8a939be2989ac2953df892d825", - "bf6/bf8/e5m3/e8m0/32": "b0a49d3f7c9dd9690165d7147cdadbcdae873e8331ff569621fc440772f9544f", - "bf6/bf8/e5m3/e8m0/16": "54a7acd085554ebed111c34af46ef0a6272cc42a3a188494a8d5515c52ae205d", - "bf6/bf8/e5m3/e5m3/32": "e5be573db446851b799ae6f52ed2ad72e1731efd02edc25af9acbce383e42c10", - "bf6/bf8/e5m3/e5m3/16": "a78b606408072246d52a0d0a03b726c580ea245342f1e7cc2ccae81c6d8242aa", - "bf6/bf8/e5m3/e4m3/32": "4f383aab8ffe0c23ae5e9abebfae8170313890a827a6ec42e2314ab60db87d5a", - "bf6/bf8/e5m3/e4m3/16": "4217ee75bf0bb42616f613569767ebd677b330029b71e9a98dd7ad1e44acf397", - "bf6/bf8/e4m3/e8m0/32": "c2471b517cdf8d91eaea0e49fd9db260b661c6469f7fe9bebee74025fe056e9c", - "bf6/bf8/e4m3/e8m0/16": "2f7ff8d556bfdc149f10949f3d1c242ff531c78cac4448f7cb27a6d808024b69", - "bf6/bf8/e4m3/e5m3/32": "3f00e79a55a77d35ea8bea3bf597cd48070c9c70f966f4752126be6141ac5e22", - "bf6/bf8/e4m3/e5m3/16": "df990786fd04718612772f90ce6c2619129c0e23122e534da16374ad03af9d7d", - "bf6/bf8/e4m3/e4m3/32": "b99fbd5ceab2fa4af4d70e28032448be5616aac9c5baa7805723f0eae0518ed4", - "bf6/bf8/e4m3/e4m3/16": "d68fae154927b102eaed126ba5dffba8f09b6a4edafe49a401ca6fa282ec2261", - "bf6/fp6/e8m0/e5m3/32": "23b022b264b2c4ad2d6e1338eb3ac24d064d2d5dbea86c218433c6edb1e62fa9", - "bf6/fp6/e8m0/e5m3/16": "4a3a441cd8381977530837ecca2de7e3d55679cca7ef47acb7da04ab0db8d29c", - "bf6/fp6/e8m0/e4m3/32": "721d7f1343ca1301325760e76f04e552177844ae9e8b0e091325d125b4fd046b", - "bf6/fp6/e8m0/e4m3/16": "050bc4fc8dd184d0ebf6ea11c77b4167eb2007d0186b3596de1e2d3ae4b6a414", - "bf6/fp6/e5m3/e8m0/32": "45823a850e02ee3dc46ea4974fc0e7bf276740b9f6cb877ef19a6f3cfdc8b6ea", - "bf6/fp6/e5m3/e8m0/16": "be1a956a3b5f5bedcbc47589e09291e3680948e1e7aa73e63853695659f18f31", - "bf6/fp6/e5m3/e5m3/32": "47d76f4e7c61edf67d79d8d6091caff0fcd85e07806a87923fab99018aeed947", - "bf6/fp6/e5m3/e5m3/16": "3ce7c53b6dd7dc815fd54d6e0a1a2b536f77aacf41ff58058ebe941d2101288d", - "bf6/fp6/e5m3/e4m3/32": "7dd71953b0ebe38a7bdffa231e97dc59108044a58e72afefdcb40d30977b3d7f", - "bf6/fp6/e5m3/e4m3/16": "327ddaa1b6fa62a0d19176199e7e25dcb01db6a4f9102275104c22c44620475a", - "bf6/fp6/e4m3/e8m0/32": "6b641131ffdd005bcb19b6e76c138e0cc56ae70f8affcbfe3c5cf4aa9b515789", - "bf6/fp6/e4m3/e8m0/16": "36dfce46ed4e631e6ed3118aa590728fdf732919cdb0f1f233d4626cab605850", - "bf6/fp6/e4m3/e5m3/32": "f27b757fc913e6ebdb5821b70335fe403e938903ea4afb9b9e4acea9d052a5e8", - "bf6/fp6/e4m3/e5m3/16": "471fb451e4405b0ab7b38c9b6f77ec619378fdf2a85420671091a11f00593a13", - "bf6/fp6/e4m3/e4m3/32": "72a2394e356b94ccf75514fff67a7e7a3e94b994357d92e1620816108560fa23", - "bf6/fp6/e4m3/e4m3/16": "b0e9c029ad5957632ca73edde0118c7ccf393c91d56368c15554cd9f684750e2", - "bf6/bf6/e8m0/e5m3/32": "92f4cb18395a5d0cd11d7345f17578c6c419c9b77acb79914f6b841d67e317ed", - "bf6/bf6/e8m0/e5m3/16": "4957edf77119d0e4d336ffa1688675b3ccf7f5e7d131d19b6a0591afe5c33106", - "bf6/bf6/e8m0/e4m3/32": "76153849c501734d6b1f2b76f32dec78c84cd163fbba8e3a88907d3247044947", - "bf6/bf6/e8m0/e4m3/16": "91c14aa795da944d6f288d558e186bf39b4fd00131cab82c89569166f6b3580c", - "bf6/bf6/e5m3/e8m0/32": "7fee78a45c19db932da84cbb2dc1282394596f77cf05c6b93bb78a2349382134", - "bf6/bf6/e5m3/e8m0/16": "91fea0ddc932f845088c7f28b9fd598b067505fcc27f6f35d34251ff5f9132be", - "bf6/bf6/e5m3/e5m3/32": "7e08adee9a0a5b7c9ebbaf7a30936b5820d6ee25df621771a1ae037521b6c310", - "bf6/bf6/e5m3/e5m3/16": "1b8b1ad21310ffc755338dc635825a6e83ce02e034688e6145416494bd406d77", - "bf6/bf6/e5m3/e4m3/32": "9577aebc348be9f0a6c79444bc613f4a8b9276035fb2b008e5782be3430c5464", - "bf6/bf6/e5m3/e4m3/16": "d226a95b9b36edaa7117b981325d4bb082e53377203879dfd1c33aee624690db", - "bf6/bf6/e4m3/e8m0/32": "fb13ade64fb4b6c286f94cb63a53a5e771239bc6edeada159b736680acea1f85", - "bf6/bf6/e4m3/e8m0/16": "53399c7c4951ef9c6d1f083870ad59fb734b43f545bea3494d2f4b9bd6b5dadd", - "bf6/bf6/e4m3/e5m3/32": "99fbfae3492310e7f3e64de5d821a17a9ed14f65838e237467a019a46f9a6a6e", - "bf6/bf6/e4m3/e5m3/16": "1d1e08819200db18bb703c91b53c435b1345b5d3195b5b8ff01d04c5b3102116", - "bf6/bf6/e4m3/e4m3/32": "0844309de13ab3757efac82731e452dc55effc4f742136309e10a78d45a2c035", - "bf6/bf6/e4m3/e4m3/16": "253f0b81f65b560aa751cea6ed80e384ef8df5166f034544a173599c9c9b825b", - "bf6/fp4/e5m3/e8m0/32": "b0530d58dd9ddacfa3c497d892ad560ede969078ae3ccd2e2521f3219396999c", - "bf6/fp4/e5m3/e8m0/16": "b149fc0b32f1b1d9adf2552acd14c6603e9d8263ba33400e2e9a53652ba74e9f", - "bf6/fp4/e5m3/e5m3/32": "9b1ddf0c531112ab1d6f2380ce34dc77d72f8181e098ea09761c99bc3a0f1fd8", - "bf6/fp4/e5m3/e5m3/16": "12eaa4fe79c78ae238e677b0fdd9912a333ab0cb349af322719defc7b864e0a2", - "bf6/fp4/e5m3/e4m3/32": "fb25dc4354828a7e510798c99493df66d86b8061a8e4536485b6fc7a7ae52b3d", - "bf6/fp4/e5m3/e4m3/16": "16b0827b1fc5ada714071aa85d03a0ec2bae0700ce2652a15fc23339213f4b63", - "bf6/fp4/e4m3/e8m0/32": "09c3012ee25b9a9f3cc03dc9ead3e35db6eb38c4f00c01b3de73cc11bdc2dd34", - "bf6/fp4/e4m3/e8m0/16": "6bc5e5035b8260a0f1eb741c05783ec9c1f3830449f2387a887d54c54b5096ee", - "bf6/fp4/e4m3/e5m3/32": "10d280ce5fef41c5ace6ada641416e9f8633be556ce5eb36235417576165b4e4", - "bf6/fp4/e4m3/e5m3/16": "fb370fd72e5a1dfa337139b4ac25c6b18044e10694edc370d81b0e2fb3940c89", - "bf6/fp4/e4m3/e4m3/32": "8235b69fc70877ca1229fe5397d50f2569e58a7f6ccbf7f98fe6c2d93ffd55fc", - "bf6/fp4/e4m3/e4m3/16": "9729c52d5b9f148a50749e16cb26c8c875c29e883d603e25398dc0fc58c50c3a", - "fp4/fp8/e8m0/e5m3/32": "b06729d6f01be089b4d012df7e69618c16291a9123c9863763441139d3f7be5a", - "fp4/fp8/e8m0/e5m3/16": "a8f0db0c4c4d447bf01bf8ad2ed8ba2b49cbeb1b314bd17f7ca67fe73dfa4743", - "fp4/fp8/e8m0/e4m3/32": "5efb47f2ad6f196f195ee4791530ad086bd97b5c5b4da1b1972b1a60637d6ecb", - "fp4/fp8/e8m0/e4m3/16": "f2a4020e74d184e9c89a576ed090e92702feae04b52046d13d984ab61563762d", - "fp4/fp8/e5m3/e5m3/32": "4866c2929146dae0c3bc11fc698f99e5051cd9bbca786494773c59fb21e32b3e", - "fp4/fp8/e5m3/e5m3/16": "6bebc8bb6d9fb425651d94be96820ed9415a4e743344c082673a1f6e27751369", - "fp4/fp8/e5m3/e4m3/32": "3f51d1ff30fbcdf8fcefe5a4765998d3acbe5be698bb082695991ce43c0d7c6b", - "fp4/fp8/e5m3/e4m3/16": "642ad05d1ac3aff7fe30695d5bead353f481ebc8789407c84073b23c47a8860e", - "fp4/fp8/e4m3/e5m3/32": "41f6c622c0fd24b0ec082031899f40ee1b46aabee668a2686fef437727c5fe62", - "fp4/fp8/e4m3/e5m3/16": "e8ee2ea5860d92b20f2b801eaffb6bb7c3b88c1802baa9458bd92a131766f1e6", - "fp4/fp8/e4m3/e4m3/32": "458e78d06e293be53149b4e76e57bab55f4e24a5bfc8dbf87aa7a2073b22a07b", - "fp4/fp8/e4m3/e4m3/16": "195a82f9c0c72c2538e4b92493007a16bd1def8ae80ff4d93425c2bcf2f658ab", - "fp4/bf8/e8m0/e5m3/32": "fafc4dc0034112ddf6e1e34fc5c132b2444dc713b43fc815b7467c068e78487e", - "fp4/bf8/e8m0/e5m3/16": "a3fff2dedb6e65c99050423eb5c8d7c6bf1d5b1c359f09c2b1e3e35fe43f37f0", - "fp4/bf8/e8m0/e4m3/32": "cc0595ed0cde391623693504dd132ebebc6daf03e096b2305a4749a007c4e852", - "fp4/bf8/e8m0/e4m3/16": "84466c20fa688db1926d19c38380f904fdf15540ff74c3d393def973c7e939df", - "fp4/bf8/e5m3/e5m3/32": "f5b642cf633d10aeaae17ddcc0e07f9c6e5901eda5b1500cc709c6e7f4f0b2ab", - "fp4/bf8/e5m3/e5m3/16": "554b9470ff7e1fb79b7beca0ab8942026c446a15a5a1c21bffaed056149bd7cb", - "fp4/bf8/e5m3/e4m3/32": "d09b6e7613bdcf43595f5391624fae51ce3c352a539a681e5e327ffabfe28852", - "fp4/bf8/e5m3/e4m3/16": "81e8e798235d4279163e0a1d706aa91e67c7289abaeb9f2439a7414ce6d5c023", - "fp4/bf8/e4m3/e5m3/32": "6b0597a2ad2117630edc45142fc9478e42feaeb26880d97fe2e5d0b015af2125", - "fp4/bf8/e4m3/e5m3/16": "b575d04bafc6e43d8d61f797c32ab00f873c2dddb5e25133d43e1361af5a400f", - "fp4/bf8/e4m3/e4m3/32": "994637afd07c7c523bb1cf9c270f51a384a6e3ad702b154de64d7149c292d089", - "fp4/bf8/e4m3/e4m3/16": "94159f433473f2ea83bb675318495816c09ccc629e36cff62fb8a01000663c86", - "fp4/fp6/e8m0/e5m3/32": "670684d268b5df752aa6becfab6c67d6f049f8bdf11e42fbb8f5193855d3e532", - "fp4/fp6/e8m0/e5m3/16": "e5f5514079be9f92b9b44c98e417c792bf1813c2c0f2d15472da2890d7db73d6", - "fp4/fp6/e8m0/e4m3/32": "6e1130a41c5fd8f2b5018fdc733f33894b93813b06a0c653a80fdec4b0a48381", - "fp4/fp6/e8m0/e4m3/16": "cd87c515a19ac15e63920aebfdc6b867b063829fad30ec5daa83b248b7667d65", - "fp4/fp6/e5m3/e5m3/32": "8466f703260278bbe5fc7e814089f0cbe23ea0e54945192f0643b53871344746", - "fp4/fp6/e5m3/e5m3/16": "589a06ca2b5000a4d594991677833b6c6be78ca36b6a69aeeba3c6a2e7500e29", - "fp4/fp6/e5m3/e4m3/32": "a27f48d294dd42c876be5a3a9bc418409f29f739e48165e682059f2bc1812571", - "fp4/fp6/e5m3/e4m3/16": "929e418e6cfc39d6ccb3b2a069eef3787a14ca700f312d78c4a664c7dcede475", - "fp4/fp6/e4m3/e5m3/32": "af95a99193325f03dc5fd43e6bfa21ada00f3f3d26c9da02c65c4019f0c313f0", - "fp4/fp6/e4m3/e5m3/16": "e4a5165940b5df77951c93d38428fc41f561d8b16ad33548a6e9b4e7b591d4f5", - "fp4/fp6/e4m3/e4m3/32": "97da994159532f3245f81f571badb66eed85d19c4fd1253e17e5aca0c7becc36", - "fp4/fp6/e4m3/e4m3/16": "a8afaf72967253f57b53502b430fdad74157274530c8b88c8c86a44cbab0c575", - "fp4/bf6/e8m0/e5m3/32": "b83c11fb4d0e3c3303aaa0ce531c3f8a9de0012c44a1a4fdaa1b3147472ed86a", - "fp4/bf6/e8m0/e5m3/16": "123418e5c59dc24fed54a603c8888056a069ecec30bb48bdf4b21bf2991c0623", - "fp4/bf6/e8m0/e4m3/32": "348cc241c497e67305fd146d9947318ff5db28f6d316eb4c13d92da7d3328663", - "fp4/bf6/e8m0/e4m3/16": "bff00f28e8db5b7200e0785d0e88766e63bb12a90f83eb93213f1675f26b67a8", - "fp4/bf6/e5m3/e5m3/32": "042065ec2f4a53c30126bb0c12c6f677e6e00c2ce9e2b487ec0129619ba604f2", - "fp4/bf6/e5m3/e5m3/16": "6530823c8b17dee4b574f79dbcb53f804a5172435ca553b4cddbcc5163135ef7", - "fp4/bf6/e5m3/e4m3/32": "60c5b248ab017506e79747f8d93d67f2945931f56d1138e02c8d1e0a7b09dc9b", - "fp4/bf6/e5m3/e4m3/16": "f3f74b4617fd516353fee9c6bb85df32d98b6101ef45b5ccfd2b99a914087727", - "fp4/bf6/e4m3/e5m3/32": "50634c3edd88023a58a499b2ef5f6571fb809702d8ec271861ec8b7ee12271cd", - "fp4/bf6/e4m3/e5m3/16": "eddf0fd2b857cd1a5139487136f4f66b8dd0d401185f022586c8362b48d56c9d", - "fp4/bf6/e4m3/e4m3/32": "1c3e10354b0219b9631767fce1e946923c9d2df37aae8f68d802b4903b544b2a", - "fp4/bf6/e4m3/e4m3/16": "079ea7fc5761df17e487e9a9c6234b97bd10f28c5c0d61ee2403cd8f0e40c5b6", - "fp4/fp4/e8m0/e5m3/32": "873db2230629fda89e5504fa3607af3721c0790819b79d307ad2bc07c8104856", - "fp4/fp4/e8m0/e5m3/16": "614688537f04af76cec6ff46a8bcd8cd6dfac3c162e83b011ff495b5c8aff2bd", - "fp4/fp4/e8m0/e4m3/32": "337c9a1c8f85c34a8d9b31930daafe4479fe302a3353515625de0f73e93a81d6", - "fp4/fp4/e8m0/e4m3/16": "04425faa0c5d5f1fae0867c044ea6cef40765f052c43c2feef9e03945f6935e8", - "fp4/fp4/e5m3/e8m0/32": "f06925802cba1da41bab46e07402b49aefb7227c2c109c60b9fb92cf2a89f9ac", - "fp4/fp4/e5m3/e8m0/16": "191c5a67b93b673b43ff0c15159d707e9f1054ea82db40ee225ed76cf3906e86", - "fp4/fp4/e5m3/e4m3/32": "f4fce23e6412f9e1bdbebd273830b04060a4274d5bdf75e675546db623d13a68", - "fp4/fp4/e5m3/e4m3/16": "de5183c1cf53a9c802875ead562ea73b0b6d2c1fd323890597ac8310adde4617", - "fp4/fp4/e4m3/e8m0/32": "807be10ab14d7e8ab572f26e8d6eef6375fca00f2bf2ab8d95a3601a5839adcf", - "fp4/fp4/e4m3/e8m0/16": "bd6da3b9449059b0e59a72d319d2542ba81c24d78fdf629c0608e74dc05ff73f", - "fp4/fp4/e4m3/e5m3/32": "4aab2c3dc6614104d423b193ed22007b1880bc0247ecee5a1aeb8777b4d0a353", - "fp4/fp4/e4m3/e5m3/16": "a547dcdaa82408dc8f7bc3d5467e5e584b8f02b1df24502a446e4b2de582438d" + "fp4/fp4/e4m3/e4m3/16": "e68ccf62c4dbccbe3a20d32399aa3d54e48e40ccd5e72d4fa52a54aa888999f7" } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp index ee21291b62a8..a47b710106a8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp @@ -109,7 +109,7 @@ int main(void) check_atom(&b, op.op_id, 8, false); ++scaled; } - CHECK(scaled == 450, "all scaled atoms must have accumulator metadata"); + CHECK(scaled == 86, "all scaled atoms must have accumulator metadata"); } /* Unknown op_id must be rejected. The engine's error path either returns diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py index d6c44b90a5bf..59a8848bac3e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py @@ -33,14 +33,14 @@ ) -def test_expanded_catalog_matches_pre_refactor_order_and_fields(): - # Independent snapshot of all 457 hand-written rows at 0bba8265. A generator - # bug shared by both engines must not pass just because parity stays green. +def test_filtered_catalog_preserves_supported_order_and_fields(): + # Independent snapshot of the supported rows at 72c28a69, filtered with + # the LLVM legality table. Preserve all remaining fields and relative order. rows = _load_specs()["gfx1250"]["mma"] - assert len(rows) == 457 + assert len(rows) == 93 encoded = json.dumps(rows, sort_keys=True, separators=(",", ":")).encode() assert hashlib.sha256(encoded).hexdigest() == ( - "95ae6cc6ecb4d704eea729eafd291ba2d1d89183d59be28ba50af66a0377590a" + "d31fe60f6172c7c2172a420cc4558591fa1389d41cc2829eafc99bceb66e858a" ) @@ -291,8 +291,8 @@ def test_e5m2_is_not_a_scale_dtype_alias(dtype, field): def test_scaled_catalog_identity_and_backend_contract(): catalog = ArchTarget.from_gfx("gfx1250").mma rows = [row for row in catalog.ops if row.family == "wmma_scaled"] - assert len(rows) == 450 - assert len({row.op_id for row in rows}) == 450 + assert len(rows) == 86 + assert len({row.op_id for row in rows}) == 86 for row in rows: tokens = { "fp8e4m3": "fp8", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py index 5b82bfdefeec..b9835ee64de3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mixed_scaled_wmma.py @@ -108,7 +108,7 @@ def test_mixed_atoms_do_not_leak_to_other_targets(a, b): ) -@pytest.mark.parametrize("scales", [("e4m3", "e8m0", 32), ("e5m3", "e8m0", 16)]) +@pytest.mark.parametrize("scales", [("e8m0", "e4m3", 32), ("e8m0", "e5m3", 16)]) def test_mixed_atoms_admit_independent_scale_contracts(scales): assert ( ArchTarget.from_gfx("gfx1250").mma.op_for_shape( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py index 829fad2a2134..bde0b0213e75 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py @@ -30,10 +30,22 @@ FORMATS = ("fp8", "bf8", "fp6", "bf6", "fp4") SCALES = ("e8m0", "e5m3", "e4m3") + + +def legal_combination(a, b, sa, sb): + # Independent oracle for LLVM commit 7597c0c938b3's supported table. + allowed = {"fp4": SCALES} + return ( + sa in allowed.get(a, ("e8m0",)) + and sb in allowed.get(b, ("e8m0",)) + and (a != "fp4" or b != "fp4" or sa == sb) + ) + + PAIRS = [ (a, b, sa, sb) for a, b, sa, sb in product(FORMATS, FORMATS, SCALES, SCALES) - if (sa, sb) != ("e8m0", "e8m0") + if (sa, sb) != ("e8m0", "e8m0") and legal_combination(a, b, sa, sb) ] @@ -67,8 +79,13 @@ def test_catalog_and_builder_agree_on_legal_scale_combinations(a, b, sa, sb, blo k=128, scales=(sa, sb, block_k), ) - assert atom is not None - assert is_valid_spec(spec_for(a, b, sa, sb, block_k))[0] + legal = legal_combination(a, b, sa, sb) + spec = spec_for(a, b, sa, sb, block_k) + assert (atom is not None) == legal + assert is_valid_spec(spec)[0] == legal + if not legal: + with pytest.raises(ValueError, match="no gfx1250 scaled WMMA atom"): + build_block_scaled_gemm(spec) @pytest.mark.parametrize("a,b,sa,sb", PAIRS) @@ -218,15 +235,50 @@ def test_backend_rejects_invalid_contract_even_if_catalog_injected( atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( family="wmma_scaled", - a_dtype="fp6", + a_dtype="fp4", b_dtype="fp8", c_dtype="fp32", m=16, n=16, k=128, - scales=("e4m3", "e5m3", 32), + scales=("e4m3", "e8m0", 32), ) invalid = replace(atom, **{field: value}) monkeypatch.setattr(MmaCatalog, "by_op_id", lambda self, op_id: invalid) with pytest.raises(ValueError, match="unsupported scaled WMMA backend contract"): gfx1250_scaled_wmma(atom.op_id) + + +@pytest.mark.parametrize( + "a,b,sa,sb", + [ + (a, b, sa, sb) + for a, b, sa, sb in product(FORMATS, FORMATS, SCALES, SCALES) + if not legal_combination(a, b, sa, sb) + ], +) +def test_backend_rejects_invalid_scale_combination_even_if_catalog_injected( + monkeypatch, a, b, sa, sb +): + from rocke.core.arch.target import MmaCatalog, normalize_dtype + + atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( + family="wmma_scaled", + a_dtype="fp4", + b_dtype="fp4", + c_dtype="fp32", + m=16, + n=16, + k=128, + scales=("e4m3", "e4m3", 32), + ) + invalid = replace( + atom, + a_dtype=normalize_dtype(a), + b_dtype=normalize_dtype(b), + a_scale_dtype=sa, + b_scale_dtype=sb, + ) + monkeypatch.setattr(MmaCatalog, "by_op_id", lambda self, op_id: invalid) + with pytest.raises(ValueError, match="matrix/scale format combination"): + gfx1250_scaled_wmma(atom.op_id) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py index 342d7a09a8f3..2f5496aaa15f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py @@ -65,7 +65,10 @@ def test_backend_selects_scale_type_independently_on_either_input(self): catalog = ArchTarget.from_gfx("gfx1250").mma for operand, field in enumerate(("a_scale_dtype", "b_scale_dtype")): for selector, dtype in enumerate(("e8m0", "e5m3", "e4m3")): - atom = replace(_scaled_atom(), **{field: dtype}) + matrix_field = ("a_dtype", "b_dtype")[operand] + atom = replace( + _scaled_atom(), **{matrix_field: "fp4e2m1", field: dtype} + ) with ( self.subTest(field=field, dtype=dtype), mock.patch.object(catalog, "by_op_id", return_value=atom), diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp index a0244de9ed73..b543524a1fba 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp @@ -315,7 +315,7 @@ static int test_scale_contracts() ++scaled_rows; } } - // Ordered identity fingerprint from the hand-written 0bba8265 catalog. + // Ordered identity fingerprint from the independently filtered 72c28a69 catalog. // Also exercises all generated static ID pointers after repeated queries. uint64_t fingerprint = UINT64_C(14695981039346656037); for(int i = 0; i < arch->mma.num_ops; ++i) @@ -327,9 +327,9 @@ static int test_scale_contracts() fingerprint *= UINT64_C(1099511628211); } while(*text++); } - CHECK(arch->mma.num_ops == 457); - CHECK(fingerprint == UINT64_C(16714491246104151946)); - CHECK(packed_rows == 4 && scaled_rows == 450); + CHECK(arch->mma.num_ops == 93); + CHECK(fingerprint == UINT64_C(8996553530891217874)); + CHECK(packed_rows == 4 && scaled_rows == 86); return 0; } @@ -398,7 +398,59 @@ static int test_scale_layouts_and_families() } } } - CHECK(scaled == 450); + CHECK(scaled == 86); + return 0; +} + +static int test_scaled_wmma_supported_combinations() +{ + const auto* arch = rocke_arch_target_from_gfx("gfx1250"); + const char* inputs[] = {"fp8e4m3", "bf8e5m2", "fp6e2m3", "fp6e3m2", "fp4e2m1"}; + const char* scales[] = {"e8m0", "e5m3", "e4m3"}; + const rocke_mma_scale_filter_t e8 = {"e8m0", "e8m0", ROCKE_MMA_SCALE_K32}; + const auto* base = rocke_mma_catalog_op_for_shape( + &arch->mma, "wmma_scaled", "fp8", "fp8", "fp32", 16, 16, 128, &e8); + CHECK(base); + int accepted = 0, rejected = 0; + for(auto block : {ROCKE_MMA_SCALE_K32, ROCKE_MMA_SCALE_K16}) + for(int a = 0; a < 5; ++a) + for(int b = 0; b < 5; ++b) + for(int sa = 0; sa < 3; ++sa) + for(int sb = 0; sb < 3; ++sb) + { + // Independent oracle for LLVM commit 7597c0c938b3. + const bool legal = (sa == 0 || a == 4) && (sb == 0 || b == 4) + && (!(a == 4 && b == 4) || sa == sb); + const rocke_mma_scale_filter_t filter = {scales[sa], scales[sb], block}; + const auto* atom = rocke_mma_catalog_op_for_shape(&arch->mma, + "wmma_scaled", + inputs[a], + inputs[b], + "fp32", + 16, + 16, + 128, + &filter); + CHECK((atom != NULL) == legal); + auto injected = *base; + injected.a_dtype = inputs[a]; + injected.b_dtype = inputs[b]; + injected.a_scale_dtype = scales[sa]; + injected.b_scale_dtype = scales[sb]; + injected.scale_block_k = block; + injected.a_scale_frag_len = injected.b_scale_frag_len = 128 / block; + if(legal) + { + rocke_scaled_wmma_contract(&injected); + ++accepted; + } + else + { + CHECK(rejects_query([&] { rocke_scaled_wmma_contract(&injected); })); + ++rejected; + } + } + CHECK(accepted == 86 && rejected == 364); return 0; } @@ -406,7 +458,8 @@ int main() { // Keep this first: no previous query may prime the shared family index. if(test_family_index_first_use() || test_family_index_duplicates() || test_mma_result_names() - || test_scale_contracts() || test_scale_layouts_and_families()) + || test_scale_contracts() || test_scale_layouts_and_families() + || test_scaled_wmma_supported_combinations()) return 1; int checked = 0; for(const char* gfx : {"gfx950", "gfx1250"}) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py index 7811952e6fa1..318bc2f7ddcf 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py @@ -50,7 +50,7 @@ def test_every_scaled_catalog_atom_has_the_same_padded_abi(self): for row in _load_specs()["gfx1250"]["mma"] if row["family"] == "wmma_scaled" ] - self.assertEqual(len(rows), 450) + self.assertEqual(len(rows), 86) for row in rows: info = _MMA_FRAGMENT_INFO[row["op_id"]] self.assertEqual( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_scale_layouts.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_scale_layouts.py index daefbcdd011e..a6a9bf59cae7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_scale_layouts.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_scale_layouts.py @@ -96,7 +96,7 @@ def test_every_scaled_catalog_coordinates_match_independent_lane_oracle(): for op in ArchTarget.from_gfx("gfx1250").mma.ops if op.family == "wmma_scaled" ] - assert len(atoms) == 450 + assert len(atoms) == 86 for atom in atoms: for role in ("a_scale", "b_scale"): layout = getattr(atom, f"{role}_layout")() diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py index dbced3359433..168b4d775f31 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/test_gate_verdict.py @@ -671,19 +671,17 @@ def setUpClass(cls): cls.config_count = len(emitter.CONFIGS) def test_scale_pair_order_matches_explicit_tables(self): - # Fingerprint captured from the explicit tuples before loop generation. + # Fingerprint captured from the supported explicit tuples before loop generation. # The order is observable: both emitters select kernels by case index. - legacy = self.emitter._LEGACY_SCALE_PAIRS - remaining = self.emitter._OTHER_SCALE_PAIRS - self.assertEqual((len(legacy), len(remaining)), (18, 182)) - pairs = legacy + remaining - self.assertEqual(len(set(pairs)), 200) + pairs = self.emitter._SCALE_PAIRS + self.assertEqual(len(pairs), 18) + self.assertEqual(len(set(pairs)), 18) fingerprint = hashlib.sha256( json.dumps(pairs, separators=(",", ":")).encode() ).hexdigest() self.assertEqual( fingerprint, - "2e67cba9ddce17e2d13ff93f25a8492085b1eac54d9daa464d1931f48392f9a4", + "170e1ae8da6070187bb0877ee05a9ecc3999fd20e42dc4907e44478833e0b9aa", ) def drive_family(self, driver, mismatch=None): @@ -753,7 +751,7 @@ def lower(cli, family, ir, ref): return rows, ir_artifact_diff.family_status(rows)[0], indices def test_both_drivers_visit_all_configs_and_stop_at_end(self): - self.assertEqual(self.config_count, 477) + self.assertEqual(self.config_count, 113) for driver, status in (("builders", "GREEN"), ("artifacts", "IDENTICAL")): with self.subTest(driver=driver): rows, actual_status, indices = self.drive_family(driver) @@ -763,7 +761,7 @@ def test_both_drivers_visit_all_configs_and_stop_at_end(self): def test_new_feature_probes_follow_the_existing_scale_configs(self): self.assertEqual( - [build.__name__ for build, arch in self.emitter.CONFIGS[472:]], + [build.__name__ for build, arch in self.emitter.CONFIGS[108:]], [ "build_data_prefetch", "build_cluster", @@ -773,21 +771,24 @@ def test_new_feature_probes_follow_the_existing_scale_configs(self): ], ) self.assertTrue( - all(arch == "gfx1250" for _, arch in self.emitter.CONFIGS[472:]) + all(arch == "gfx1250" for _, arch in self.emitter.CONFIGS[108:]) ) def test_both_drivers_detect_drift_beyond_default_cap(self): late = run_diff.MAX_CFG - for driver in ("builders", "artifacts"): - with self.subTest(driver=driver): - rows, status, _ = self.drive_family(driver, mismatch=late) - self.assertEqual(status, "DRIFT") - self.assertIn(rows[late]["verdict"], ("MISMATCH", "DRIFT")) + # The supported catalog is now smaller; retain the future-growth guard. + with mock.patch.object(self, "config_count", late + 17): + for driver in ("builders", "artifacts"): + with self.subTest(driver=driver): + rows, status, _ = self.drive_family(driver, mismatch=late) + self.assertEqual(status, "DRIFT") + self.assertIn(rows[late]["verdict"], ("MISMATCH", "DRIFT")) def test_truncated_128_record_batch_is_not_complete_for_scaled_family(self): success = run_diff.Emission(0, True, "same", "") ended = run_diff.Emission(2, False, None, "unknown config") - full = batch(*([success] * self.config_count), ended) + synthetic_count = run_diff.MAX_CFG + 17 + full = batch(*([success] * synthetic_count), ended) truncated = batch(*([success] * run_diff.MAX_CFG)) for native, python in ((full, truncated), (truncated, full)): with ( @@ -806,6 +807,7 @@ def test_default_cap_would_miss_late_drift(self): for driver, status in (("builders", "GREEN"), ("artifacts", "IDENTICAL")): with ( self.subTest(driver=driver), + mock.patch.object(self, "config_count", run_diff.MAX_CFG + 17), mock.patch.object(run_diff, "FAMILY_MAX_CFG", {}), ): rows, actual_status, _ = self.drive_family( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c index 94bd64f26566..9bbe720e4b2d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c @@ -17,6 +17,7 @@ #include "rocke/ir_serialize.h" #include "rocke/lower_llvm.h" #include "rocke/verify.h" +#include "rocke/wmma_scale_internal.h" /* Every builder call below is its own statement, and the statement order * matches the Python emitter exactly. Nesting a builder call inside an argument @@ -660,13 +661,12 @@ static const char* const MATRIX_DTYPES[] = {"fp8", "bf8", "fp6", "bf6", "fp4"}; static const char* const SCALE_DTYPES[] = {"e8m0", "e5m3", "e4m3"}; static const int NUM_MATRIX_DTYPES = (int)(sizeof(MATRIX_DTYPES) / sizeof(MATRIX_DTYPES[0])); static const int NUM_SCALE_DTYPES = (int)(sizeof(SCALE_DTYPES) / sizeof(SCALE_DTYPES[0])); -static const int FP4_INDEX = NUM_MATRIX_DTYPES - 1; static const int NUM_CONFIGS = NUM_BASE_CONFIGS + NUM_FEATURE_CONFIGS - + 2 * NUM_MATRIX_DTYPES * NUM_MATRIX_DTYPES * (NUM_SCALE_DTYPES * NUM_SCALE_DTYPES - 1); + + 2 * (2 * (NUM_MATRIX_DTYPES - 1) * (NUM_SCALE_DTYPES - 1) + NUM_SCALE_DTYPES - 1); -/* Mirror the Python emitter's legacy FP4 cases, then its remaining scale pairs. - * Selecting by index keeps the table immutable and preserves every case index. +/* Mirror the Python emitter's supported scale pairs in their original order. + * Selecting by index keeps the table immutable. */ static bool select_config(int idx, config_t* out) { @@ -683,33 +683,28 @@ static bool select_config(int idx, config_t* out) return true; } idx -= NUM_BASE_CONFIGS; - for(int phase = 0; phase < 2; ++phase) - for(int mode = 0; mode < 2; ++mode) - for(int a = 0; a < NUM_MATRIX_DTYPES; ++a) - for(int b = 0; b < NUM_MATRIX_DTYPES; ++b) - for(int sa = 0; sa < NUM_SCALE_DTYPES; ++sa) - for(int sb = 0; sb < NUM_SCALE_DTYPES; ++sb) + for(int mode = 0; mode < 2; ++mode) + for(int a = 0; a < NUM_MATRIX_DTYPES; ++a) + for(int b = 0; b < NUM_MATRIX_DTYPES; ++b) + for(int sa = 0; sa < NUM_SCALE_DTYPES; ++sa) + for(int sb = 0; sb < NUM_SCALE_DTYPES; ++sb) + { + if(sa == 0 && sb == 0) + continue; + if(!rocke_scaled_wmma_formats_valid(a, b, sa, sb)) + continue; + if(idx-- == 0) { - if(sa == 0 && sb == 0) - continue; - bool legacy - = (a != FP4_INDEX && b == FP4_INDEX && sa == 0 && sb != 0) - || (a == FP4_INDEX && b != FP4_INDEX && sa != 0 && sb == 0) - || (a == FP4_INDEX && b == FP4_INDEX && sa == sb); - if(legacy != (phase == 0)) - continue; - if(idx-- == 0) - { - *out = (config_t){NULL, - "gfx1250", - MATRIX_DTYPES[a], - mode != 0, - MATRIX_DTYPES[b], - SCALE_DTYPES[sa], - SCALE_DTYPES[sb]}; - return true; - } + *out = (config_t){NULL, + "gfx1250", + MATRIX_DTYPES[a], + mode != 0, + MATRIX_DTYPES[b], + SCALE_DTYPES[sa], + SCALE_DTYPES[sb]}; + return true; } + } return false; } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py index a10bc9f033dc..878acaee7061 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py @@ -20,6 +20,7 @@ # # arch is per-config (see _spec), llvm_flavor = AUTO, matching the C side. from rocke.core.arch import ArchTarget +from rocke.core.arch.target import _gfx1250_scale_combination_valid from rocke.core.ir import BF16, F16, F32, I16, I32, I64, IRBuilder, KernelDef, PtrType from _emit_common import run_emit @@ -321,29 +322,22 @@ def build(b): ) -# Keep the original FP4 scale cases first so configuration indices remain stable. +# Keep the supported FP4 scale cases in their original order. _MATRIX_DTYPES = ("fp8", "bf8", "fp6", "bf6", "fp4") _SCALE_DTYPES = ("e8m0", "e5m3", "e4m3") -_LEGACY_SCALE_PAIRS = ( - [(a, "fp4", "e8m0", s) for a in _MATRIX_DTYPES[:-1] for s in _SCALE_DTYPES[1:]] - + [("fp4", b, s, "e8m0") for b in _MATRIX_DTYPES[:-1] for s in _SCALE_DTYPES[1:]] - + [("fp4", "fp4", s, s) for s in _SCALE_DTYPES[1:]] -) -_LEGACY_SCALE_PAIR_SET = set(_LEGACY_SCALE_PAIRS) -_OTHER_SCALE_PAIRS = [ +_SCALE_PAIRS = [ (a, b, sa, sb) for a in _MATRIX_DTYPES for b in _MATRIX_DTYPES for sa in _SCALE_DTYPES for sb in _SCALE_DTYPES - if (sa, sb) != ("e8m0", "e8m0") and (a, b, sa, sb) not in _LEGACY_SCALE_PAIR_SET + if (sa, sb) != ("e8m0", "e8m0") and _gfx1250_scale_combination_valid(a, b, sa, sb) ] CONFIGS.extend( (_wmma_scaled(a, b, mode, sa, sb), "gfx1250") - for pairs in (_LEGACY_SCALE_PAIRS, _OTHER_SCALE_PAIRS) for mode in ("scale", "scale16") - for a, b, sa, sb in pairs + for a, b, sa, sb in _SCALE_PAIRS ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py index 543be13acfa7..323386fe1d8a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_example.py @@ -14,7 +14,7 @@ @pytest.mark.parametrize( - "a,b,sa,sb", [("fp4", "fp4", "e4m3", "e5m3"), ("fp6", "fp8", "e4m3", "e5m3")] + "a,b,sa,sb", [("fp4", "fp4", "e4m3", "e4m3"), ("fp6", "fp4", "e8m0", "e5m3")] ) @pytest.mark.parametrize("matrix_path", ["wmma_scale", "wmma_scale16"]) @pytest.mark.parametrize("output_dtype", ["bf16", "fp16"]) @@ -138,7 +138,9 @@ def test_scale_formats_gemm_cli_rejects_invalid_spec_before_verify( @pytest.mark.parametrize("dtype", ["fp8", "bf8", "fp6", "bf6", "fp4"]) @pytest.mark.parametrize("operand", ["a", "b"]) -def test_scale_formats_gemm_cli_accepts_alternate_scales(monkeypatch, dtype, operand): +def test_scale_formats_gemm_cli_filters_alternate_scales( + monkeypatch, capsys, dtype, operand +): from rocke.examples.gfx1250.gemm import scale_formats_gemm as example calls = [] @@ -149,6 +151,26 @@ def verify(spec, args): return 0 monkeypatch.setattr(example, "verify", verify) - assert example.main([f"--dtype-{operand}", dtype]) == 0 - assert len(calls) == 1 - assert calls[0].resolved_scale_dtypes() == ("e4m3", "e4m3") + if dtype == "fp4": + assert example.main([f"--dtype-{operand}", dtype]) == 0 + assert len(calls) == 1 + assert calls[0].resolved_scale_dtypes() == ("e4m3", "e4m3") + else: + with pytest.raises(SystemExit) as exc: + example.main([f"--dtype-{operand}", dtype]) + assert exc.value.code == 2 + assert not calls + assert "no gfx1250 scaled WMMA atom" in capsys.readouterr().err + + +@pytest.mark.parametrize("operand", ["a", "b"]) +def test_scale_formats_gemm_cli_rejects_unequal_fp4_scales( + monkeypatch, capsys, operand +): + from rocke.examples.gfx1250.gemm import scale_formats_gemm as example + + monkeypatch.setattr(example, "verify", lambda *args: pytest.fail("must not verify")) + with pytest.raises(SystemExit) as exc: + example.main([f"--scale-dtype-{operand}", "e5m3"]) + assert exc.value.code == 2 + assert "no gfx1250 scaled WMMA atom" in capsys.readouterr().err diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py index 10e6688fb3de..81a300197c20 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scale_formats_numeric.py @@ -8,6 +8,8 @@ import pytest +from rocke.core.arch.target import _gfx1250_scale_combination_valid + _MODULE = "rocke.examples.gfx1250.gemm.scale_formats_gemm" # Independent scale encodings through both COMGR and HIP. @@ -17,7 +19,7 @@ for a, b, sa, sb in product( _FORMATS, _FORMATS, ("e8m0", "e5m3", "e4m3"), ("e8m0", "e5m3", "e4m3") ) - if (sa, sb) != ("e8m0", "e8m0") + if (sa, sb) != ("e8m0", "e8m0") and _gfx1250_scale_combination_valid(a, b, sa, sb) ] @@ -117,8 +119,13 @@ def matrix(kind, k): rt = Runtime() for scale in unity: - da, db = (dtype, "fp4") if operand == "a" else ("fp4", dtype) + # Sweep alternate selectors only on FP4, with a non-FP4 E8M0 partner. + # Non-FP4 operands still get the fixed-byte E8M0 control. + partner = "fp8e4m3" if dtype == "fp4" else "fp4" + da, db = (dtype, partner) if operand == "a" else (partner, dtype) sa, sb = (scale, "e8m0") if operand == "a" else ("e8m0", scale) + if not _gfx1250_scale_combination_valid(da, db, sa, sb): + continue spec = BlockScaledGemmSpec( name="fixed_scale_bytes", M=16, From e6c65de3aa527bcc440896ea55ffb2f2813b1b28 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:38:53 -0400 Subject: [PATCH 30/32] docs(rocke): align scale-format overview with supported combinations --- .../platform/dsl_docs/architecture/adding_an_arch.md | 6 ++++-- .../python/rocke/examples/gfx1250/gemm/README.md | 7 ++++--- .../platform/python/rocke/instances/SUPPORT_MATRIX.md | 9 +++++---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/adding_an_arch.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/adding_an_arch.md index 9410f00ef531..e907872e6579 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/adding_an_arch.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/adding_an_arch.md @@ -118,8 +118,10 @@ are compared by the byte-identity gate, not by review alone. The gfx1250 scaled family uses one JSON declaration, `gfx1250_wmma_scaled_v1`, instead of individual rows. It expands independent A/B matrix types (`fp8`, `bf8`, `fp6`, `bf6`, `fp4`), independent A/B scale types -(`e8m0`, `e5m3`, `e4m3`), and K-group sizes 32/16 into 450 ordinary -`wmma_scaled` rows with shape 16×16×128 and fp32 accumulators. The loader +(`e8m0`, `e5m3`, `e4m3`), and K-group sizes 32/16 into 86 ordinary +`wmma_scaled` rows with shape 16×16×128 and fp32 accumulators. The +[supported-combination filter](https://github.com/llvm/llvm-project/commit/7597c0c938b364306cbee21dd03835ff5b699c74) +requires E8M0 for non-FP4 operands and matching scales for FP4 × FP4. The loader validates the complete declaration before expansion and rejects duplicate IDs. Version 1 fixes both these axes and their historical enumeration order; changing support requires an explicit loader/native generator update and regression diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md index 34197e630dcc..8923330bbc9b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md @@ -15,7 +15,7 @@ LLVM 23 ROCm libraries, NumPy, and `ml_dtypes`. Torch is optional. | `mxfp6_gemm` | Packed FP6 E2M3 or E3M2 on both operands | E8M0 | | `mxfp4_gemm` | Packed FP4 E2M1 on both operands | E8M0 | | [`mixed_scaled_gemm`](mixed_scaled_gemm.py) | Different A/B formats from FP8, BF8, FP6, BF6, and FP4 | E8M0 | -| [`scale_formats_gemm`](scale_formats_gemm.py) | Any A/B pair drawn from FP8/BF8/FP6/BF6/FP4 | Independent E8M0, E4M3, or E5M3 scales for each operand | +| [`scale_formats_gemm`](scale_formats_gemm.py) | Any A/B pair drawn from FP8/BF8/FP6/BF6/FP4 | E8M0 for non-FP4; E8M0/E4M3/E5M3 for FP4; matching scales for FP4 × FP4 | ```sh ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.mxfp8_gemm @@ -92,8 +92,9 @@ alone does not establish support for other operand contracts. The [scale-format example](SCALE_FORMATS.md) defaults to FP4 x FP4 with E4M3 scales. `--scale-dtype-a` and `--scale-dtype-b` select the input encodings; -each operand independently accepts E8M0, E4M3, or E5M3 scales across -FP8/BF8, FP6/BF6, and FP4 matrix formats. The shared verifier decodes each scale buffer independently. +FP8/BF8 and FP6/BF6 operands require E8M0, while FP4 accepts E8M0, E4M3, +or E5M3. FP4 × FP4 requires matching formats. The shared verifier decodes +each scale buffer independently. ```sh ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.scale_formats_gemm --case all diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md index 62b0fee6882c..2b265b69838d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/SUPPORT_MATRIX.md @@ -45,10 +45,11 @@ These emit generic AMDGPU IR; arch only sets the comgr target triple. ## gfx1250 native scaled GEMM `block_scaled_gemm` supports homogeneous and mixed FP8 E4M3, BF8 E5M2, -FP6 E2M3/E3M2, and FP4 E2M1 through SCALE and SCALE16 with LLVM 23. -E8M0, E4M3, and E5M3 scales are accepted independently for each operand -across all matrix pairs. The expanded alternate-scale combinations have host -admission/lowering coverage; fresh gfx1250 numerical validation is pending. +FP6 E2M3/E3M2, and FP4 E2M1 through SCALE and SCALE16 with the `llvm23` +emission flavor. FP8/BF8/FP6/BF6 operands require E8M0 scales; FP4 accepts +E8M0, E4M3, or E5M3. FP4 × FP4 requires matching scale formats. Host tests +cover admission and rejection; numeric fixtures cover bounded finite inputs, +varying non-unit scales, and fixed-byte selector discrimination through COMGR/HIP. See the [scale-format contract](../examples/gfx1250/gemm/SCALE_FORMATS.md) and [packed FP6 input contract](../examples/gfx1250/gemm/FP6.md). From 5027f0de3dd899de6d39879071f0b0701c1ff33e Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:53:10 -0400 Subject: [PATCH 31/32] fix(rocke): preserve positional block-scaled GEMM spec arguments --- .../instances/gfx1250/block_scaled_gemm.py | 4 +- .../tests/core/test_gfx1250_prefetch_users.py | 39 ++++++++++++++++--- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py index 037d7b808240..5c4cf49f6035 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py @@ -103,13 +103,13 @@ class BlockScaledGemmSpec: tile_m: int = 16 tile_n: int = 16 tile_k: int = 128 - scale_dtype_a: str | None = None - scale_dtype_b: str | None = None # Data-prefetch hints (gfx1250, llvm23 only): one scalar prefetch of the # workgroup's A-scale rows and a per-lane ``global_prefetch`` of the next # K step's A/B rows. Results do not change; off keeps the default IR. prefetch: bool = False + scale_dtype_a: str | None = None + scale_dtype_b: str | None = None def resolved_scale_dtypes(self) -> tuple[str, str]: aliases = {"i8": "e8m0", "fp8e4m3": "e4m3"} diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_prefetch_users.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_prefetch_users.py index 20d99667e19e..742dcd0fd1be 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_prefetch_users.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_prefetch_users.py @@ -69,22 +69,51 @@ def _traffic(llvm): class TestBlockScaledGemmPrefetch(unittest.TestCase): + def test_legacy_positional_prefetch_preserves_keyword_contract(self): + for path in ("wmma", "wmma_scale", "wmma_scale16"): + for prefetch in (False, True): + with self.subTest(path=path, prefetch=prefetch): + expected = _spec(path, prefetch=prefetch) + positional = BlockScaledGemmSpec( + expected.name, + expected.M, + expected.N, + expected.K, + expected.dtype_a, + expected.dtype_b, + expected.dtype_c, + expected.dtype_acc, + expected.scale_dtype, + expected.block_k, + expected.layout, + expected.matrix_path, + expected.tile_m, + expected.tile_n, + expected.tile_k, + prefetch, + ) + self.assertEqual(positional, expected) + self.assertIs(positional.prefetch, prefetch) + self.assertIsNone(positional.scale_dtype_a) + self.assertIsNone(positional.scale_dtype_b) + self.assertEqual(_lower(positional), _lower(expected)) + def test_independent_scale_formats_compose_with_prefetch(self): for path in ("wmma_scale", "wmma_scale16"): with self.subTest(path=path): off = _spec( path, prefetch=False, - dtype_a="fp6", + dtype_a="fp4", dtype_b="bf8", scale_dtype_a="e4m3", - scale_dtype_b="e5m3", + scale_dtype_b="e8m0", ) on = replace(off, prefetch=True) - self.assertEqual(on.resolved_scale_dtypes(), ("e4m3", "e5m3")) + self.assertEqual(on.resolved_scale_dtypes(), ("e4m3", "e8m0")) self.assertEqual(on.kernel_name(), off.kernel_name() + "_pf") self.assertIn("sa_e4m3", on.kernel_name()) - self.assertIn("sb_e5m3", on.kernel_name()) + self.assertNotIn("sb_", on.kernel_name()) llvm = _lower(on) self.assertEqual(_traffic(llvm), _traffic(_lower(off))) self.assertEqual( @@ -101,7 +130,7 @@ def test_independent_scale_formats_compose_with_prefetch(self): self.assertEqual(len(calls), 2) for call in calls: self.assertEqual( - re.findall(r"i32 0, i32 (\d+), i(?:32|64) %", call), ["2", "1"] + re.findall(r"i32 0, i32 (\d+), i(?:32|64) %", call), ["2", "0"] ) def test_default_is_off(self): From f0ece5e36d4e06aad4ae98c53d94823ac27f85fb Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:34:38 -0400 Subject: [PATCH 32/32] fix(rocke): reject invalid scale fixture formats --- .../rocke/examples/common/scaled_gemm_reference.py | 2 ++ .../platform/tests/core/test_gfx1250_scale_formats.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/common/scaled_gemm_reference.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/common/scaled_gemm_reference.py index b0c66cb655d4..23aa4f566005 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/common/scaled_gemm_reference.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/common/scaled_gemm_reference.py @@ -225,6 +225,8 @@ def scales(dtype, shape): ), 127, ) + if dtype not in ("e4m3", "e5m3"): + raise ValueError("scale dtype must be e8m0, e4m3, or e5m3") bias = 7 if dtype == "e4m3" else 15 # Include non-power-of-two scales to distinguish these from E8M0. codes = np.array( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py index bde0b0213e75..4807246917b1 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scale_formats.py @@ -177,12 +177,18 @@ def test_invalid_scale_spellings(dtype, field): assert not is_valid_spec(spec)[0] with pytest.raises(ValueError): build_block_scaled_gemm(spec) + with pytest.raises(ValueError, match="scale dtype must be e8m0, e4m3, or e5m3"): + make_case_inputs(spec, "mixed") def test_per_operand_overrides_preserve_defaults_and_aliases(): implicit = spec_for(sa=None, sb=None) explicit = replace(implicit, scale_dtype_a="e8m0", scale_dtype_b="i8") assert implicit.kernel_name() == explicit.kernel_name() + for expected, actual in zip( + make_case_inputs(implicit, "mixed"), make_case_inputs(explicit, "mixed") + ): + np.testing.assert_array_equal(actual, expected) lower = lambda spec: lower_kernel_to_llvm( build_block_scaled_gemm(spec), arch="gfx1250", llvm_flavor="llvm23" ) @@ -192,6 +198,10 @@ def test_per_operand_overrides_preserve_defaults_and_aliases(): ) assert alias.resolved_scale_dtypes() == ("e4m3", "e4m3") assert alias.kernel_name() == spec_for().kernel_name() + for expected, actual in zip( + make_case_inputs(spec_for(), "mixed"), make_case_inputs(alias, "mixed") + ): + np.testing.assert_array_equal(actual, expected) assert lower(alias) == lower(spec_for()) assert lower(implicit) != lower(spec_for())