From bff1cc03e4e33aab76d52410a2d2ab10d480344a Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 18 Sep 2026 00:55:58 -0400 Subject: [PATCH 01/15] fix(rocke): share MXFP contracts and canonical catalog lookups --- .../rocke/platform/cpp/core/arch/data.cpp | 83 +++++++++++----- .../rocke/platform/cpp/core/ir/ir_tile.cpp | 2 + .../cpp/core/lower_hip/lower_hip_mma.cpp | 26 ++--- .../platform/cpp/core/lower_llvm/mma.cpp | 35 ++++--- .../cpp/include/rocke/wmma_scale_internal.h | 68 +++++++++++++ .../rocke/core/arch/data/arch_specs.json | 6 +- .../platform/python/rocke/core/arch/target.py | 26 ++++- .../python/rocke/core/arch/wmma_scale.py | 63 ++++++++++++ .../rocke/platform/python/rocke/core/ir.py | 2 + .../platform/python/rocke/core/isa/backend.py | 47 +++++---- .../platform/python/rocke/core/lower_hip.py | 27 ++++-- .../rocke/examples/gfx1250/gemm/README.md | 28 ++++++ .../gfx1250/gemm/_scaled_gemm_example.py | 46 +++++++++ .../gfx1250/gemm/block_scaled_gemm_verify.py | 24 +++-- .../rocke/examples/gfx1250/gemm/mxfp8_gemm.py | 44 +++++++++ .../instances/gfx1250/block_scaled_gemm.py | 76 ++++++++++----- .../rocke/platform/tests/CMakeLists.txt | 61 ++++++++---- .../tests/core/gfx1250_mxfp8_llvm23.json | 6 ++ .../tests/core/test_gfx1250_mxfp8_example.py | 90 +++++++++++++++++ .../tests/core/test_gfx1250_scaled_wmma.py | 15 ++- .../tests/core/test_lowbit_dtype_aliases.py | 97 +++++++++++++++++++ .../tests/core/test_mma_catalog_lookup.cpp | 72 ++++++++++++++ ...250_scaled_wmma_numeric.py => conftest.py} | 24 +++-- .../instances/parity/gfx1250_lowering_emit.c | 23 +++-- .../instances/parity/gfx1250_lowering_emit.py | 6 ++ .../instances/test_gfx1250_mxfp8_numeric.py | 26 +++++ .../instances/test_gfx1250_mxfp_examples.py | 30 ++++++ 27 files changed, 900 insertions(+), 153 deletions(-) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/_scaled_gemm_example.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/mxfp8_gemm.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_mxfp8_llvm23.json create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mxfp8_example.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp rename dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/{test_gfx1250_scaled_wmma_numeric.py => conftest.py} (80%) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp8_numeric.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp_examples.py diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp index 3644634e3bc6..1aad45b7ecb3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp @@ -55,10 +55,6 @@ static const rocke_ati_dtype_alias_t k_dtype_aliases[] = { {"f32", "fp32"}, {"float", "fp32"}, {"fp32", "fp32"}, - {"fp8", "fp8e4m3"}, - {"fp8e4m3", "fp8e4m3"}, - {"bf8", "bf8e5m2"}, - {"bf8e5m2", "bf8e5m2"}, {"iu8", "iu8"}, {"iu4", "iu4"}, {"i8", "i8"}, @@ -67,6 +63,16 @@ static const rocke_ati_dtype_alias_t k_dtype_aliases[] = { {"int4", "i4"}, {"i32", ROCKE_DTYPE_I32}, {"int32", ROCKE_DTYPE_I32}, + {"fp8", "fp8e4m3"}, + {"fp8e4m3", "fp8e4m3"}, + {"bf8", "bf8e5m2"}, + {"bf8e5m2", "bf8e5m2"}, + {"fp6", "fp6e2m3"}, + {"fp6e2m3", "fp6e2m3"}, + {"bf6", "fp6e3m2"}, + {"fp6e3m2", "fp6e3m2"}, + {"fp4", "fp4e2m1"}, + {"fp4e2m1", "fp4e2m1"}, }; #define K_NUM_DTYPE_ALIASES ((int)(sizeof(k_dtype_aliases) / sizeof(k_dtype_aliases[0]))) @@ -1018,7 +1024,9 @@ static const rocke_ati_mma_frag_row_t rocke_ati_mma_frag[] = { {"wmma_gfx1250_f32_16x16x64_bf8_fp8", 8}, {"wmma_gfx1250_f32_16x16x64_bf8_bf8", 8}, {"wmma_scale_f32_16x16x128_fp8_fp8", 8}, + {"wmma_scale_f32_16x16x128_bf8_bf8", 8}, {"wmma_scale16_f32_16x16x128_fp8_fp8", 8}, + {"wmma_scale16_f32_16x16x128_bf8_bf8", 8}, }; int rocke_arch_mma_c_frag_len(const char* op_id) @@ -1052,8 +1060,7 @@ int rocke_arch_mma_c_frag_len(const char* op_id) * family, a_dtype, b_dtype, c_dtype, m, n, k, op_id, * a_frag_len, b_frag_len, c_frag_len, wave_size, a_layout, b_layout, c_layout * - * NOTE: fp4/fp6 dtypes are not in _DTYPE_ALIASES, so normalize_dtype passes them - * through as the lowercased spelling "fp4"/"fp6" (Python identity fallthrough). + * Low-bit spellings normalize to explicit exponent/mantissa catalog keys. */ /* ----------------------------- gfx90a (CDNA2) ---------------------------- */ @@ -1489,11 +1496,11 @@ static const rocke_mma_op_t k_mma_gfx950[] = { &lm_mfma_32x32x16_a, &lm_mfma_32x32x16_b, &lm_mfma_32x32x16_c}, - /* fp4/fp6 pass through normalize_dtype unchanged; op_ids carry frag lengths + /* Low-bit catalog keys use exponent/mantissa names; op_ids carry frag lengths * only (no verified maps), per _MMA_FRAGMENT_INFO. */ {"mma", - "fp4", - "fp4", + "fp4e2m1", + "fp4e2m1", "fp32", 16, 16, @@ -1507,8 +1514,8 @@ static const rocke_mma_op_t k_mma_gfx950[] = { NULL, NULL}, {"mma", - "fp6", - "fp6", + "fp6e2m3", + "fp6e2m3", "fp32", 16, 16, @@ -1742,8 +1749,8 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { &lm_wmma_gfx1250_32_b, &lm_wmma_gfx12_c}, {"wmma", - "fp8", - "fp8", + "fp8e4m3", + "fp8e4m3", "fp32", 16, 16, @@ -1757,8 +1764,8 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { NULL, &lm_wmma_gfx12_c}, {"wmma", - "fp8", - "bf8", + "fp8e4m3", + "bf8e5m2", "fp32", 16, 16, @@ -1772,8 +1779,8 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { NULL, &lm_wmma_gfx12_c}, {"wmma", - "bf8", - "fp8", + "bf8e5m2", + "fp8e4m3", "fp32", 16, 16, @@ -1787,8 +1794,8 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { NULL, &lm_wmma_gfx12_c}, {"wmma", - "bf8", - "bf8", + "bf8e5m2", + "bf8e5m2", "fp32", 16, 16, @@ -1802,8 +1809,8 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { NULL, &lm_wmma_gfx12_c}, {"wmma_scale", - "fp8", - "fp8", + "fp8e4m3", + "fp8e4m3", "fp32", 16, 16, @@ -1816,9 +1823,24 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { NULL, NULL, &lm_wmma_gfx12_c}, + {"wmma_scale", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_scale_f32_16x16x128_bf8_bf8", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c}, {"wmma_scale16", - "fp8", - "fp8", + "fp8e4m3", + "fp8e4m3", "fp32", 16, 16, @@ -1831,6 +1853,21 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { NULL, NULL, &lm_wmma_gfx12_c}, + {"wmma_scale16", + "bf8e5m2", + "bf8e5m2", + "fp32", + 16, + 16, + 128, + "wmma_scale16_f32_16x16x128_bf8_bf8", + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c}, }; /* ========================================================================= diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp index 358e870d3ef7..b335e4b334a1 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp @@ -50,7 +50,9 @@ static const rocke_mma_hint_row_t ROCKE_MMA_RESULT_HINT[] = { {"mfma_f32_16x16x128_fp8", "acc128"}, {"mfma_scale_f32_16x16x128_f8f6f4", "mxacc"}, {"wmma_scale_f32_16x16x128_fp8_fp8", "mxacc"}, + {"wmma_scale_f32_16x16x128_bf8_bf8", "mxacc"}, {"wmma_scale16_f32_16x16x128_fp8_fp8", "mxacc"}, + {"wmma_scale16_f32_16x16x128_bf8_bf8", "mxacc"}, }; /* Accumulator fragment length for op_id, from the arch SSOT diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mma.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mma.cpp index 959d67dd1bf9..e9e3dd508221 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mma.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mma.cpp @@ -30,6 +30,7 @@ #include "rocke/ir.h" #include "rocke/lower_hip.h" #include "rocke/lower_hip_internal.h" +#include "rocke/wmma_scale_internal.h" #include /* snprintf */ #include /* atoi */ @@ -75,11 +76,16 @@ static const char* h_elem_scalar(const rocke_type_t* t) * op.result (and the WMMA gate keys off the op_id *string* it is passed, not * op.attrs), so a synthetic op aliasing the same operands/results/regions and * reusing the original attrs map reproduces the Python emission exactly. */ -static rocke_status_t - h_emit_gfx1250_scaled_wmma(rocke_h_lowerer_t* lw, const rocke_op_t* op, bool scale16) +static rocke_status_t h_emit_gfx1250_scaled_wmma(rocke_h_lowerer_t* lw, const rocke_op_t* op) { - const char* op_id - = scale16 ? "wmma_scale16_f32_16x16x128_fp8_fp8" : "wmma_scale_f32_16x16x128_fp8_fp8"; + const rocke_scaled_wmma_op_t* spec = rocke_gfx1250_scaled_wmma_from_op(op); + if(!spec) + { + return rocke_h_fail(lw, ROCKE_ERR_NOTIMPL, "unsupported scaled WMMA op '%s'", op->name); + } + const char* op_id = spec->op_id; + const bool scale16 = spec->scales.block_k == 16; + const int fmt = spec->matrix_format; const char* builtin = scale16 ? "__builtin_amdgcn_wmma_scale16_f32_16x16x128_f8f6f4" : "__builtin_amdgcn_wmma_scale_f32_16x16x128_f8f6f4"; if(!lw->arch.gfx || __builtin_strcmp(lw->arch.gfx, "gfx1250") != 0) @@ -95,11 +101,13 @@ static rocke_status_t return rocke_h_fail(lw, ROCKE_ERR_VALUE, "%s expects 5 operands and 1 result", op_id); } rocke_h_emitf(lw, - "f32x8 %s = %s(0, %s, 0, %s, (int16_t)0, %s, " + "f32x8 %s = %s(%d, %s, %d, %s, (int16_t)0, %s, " "0, 0, %s, 0, 0, %s, false, false);", rocke_h_name(lw, op->results[0]), builtin, + fmt, rocke_h_name(lw, op->operands[0]), + fmt, rocke_h_name(lw, op->operands[1]), rocke_h_name(lw, op->operands[2]), rocke_h_name(lw, op->operands[3]), @@ -123,13 +131,9 @@ static rocke_status_t rocke_h_op_tile_mma(rocke_h_lowerer_t* lw, const rocke_op_ { return rocke_h_fail(lw, ROCKE_ERR_KEY, "tile.mma: missing 'op_id' attr"); } - if(__builtin_strcmp(op_id, "wmma_scale_f32_16x16x128_fp8_fp8") == 0) - { - return h_emit_gfx1250_scaled_wmma(lw, op, false); - } - if(__builtin_strcmp(op_id, "wmma_scale16_f32_16x16x128_fp8_fp8") == 0) + if(rocke_gfx1250_scaled_wmma(op_id)) { - return h_emit_gfx1250_scaled_wmma(lw, op, true); + return h_emit_gfx1250_scaled_wmma(lw, op); } snprintf(dotted, sizeof(dotted), "tile.%s", op_id); legacy_opcode = rocke_opcode_from_name(dotted); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp index 0c9c79e1c3d5..93554dac6bc7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp @@ -25,6 +25,7 @@ #include "rocke/ir.h" #include "rocke/lower_llvm.h" #include "rocke/lower_llvm_internal.h" +#include "rocke/wmma_scale_internal.h" namespace ckc { @@ -35,7 +36,7 @@ namespace ckc static void _op_tile_wmma_f32_16x16x16_f16(rocke_lower_t* L, const rocke_op_t* op); static void _op_tile_wmma_f32_16x16x16_bf16(rocke_lower_t* L, const rocke_op_t* op); static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id); -static void _emit_wmma_scale(rocke_lower_t* L, const rocke_op_t* op, bool scale16); +static void _emit_wmma_scale(rocke_lower_t* L, const rocke_op_t* op); static void _op_tile_mma(rocke_lower_t* L, const rocke_op_t* op); static void _op_tile_mfma_f32_16x16x32_fp8(rocke_lower_t* L, const rocke_op_t* op); static void _op_tile_mfma_f32_16x16x32_bf8(rocke_lower_t* L, const rocke_op_t* op); @@ -134,13 +135,9 @@ static void _op_tile_mma(rocke_lower_t* L, const rocke_op_t* op) * reject (Python ISABackend.emit_wmma raises NotImplementedError). The * gfx12-specific op_ids ("wmma_gfx12_*") can only occur on RDNA4. */ } - else if(strcmp(op_id, "wmma_scale_f32_16x16x128_fp8_fp8") == 0) + else if(rocke_gfx1250_scaled_wmma(op_id)) { - _emit_wmma_scale(L, op, false); - } - else if(strcmp(op_id, "wmma_scale16_f32_16x16x128_fp8_fp8") == 0) - { - _emit_wmma_scale(L, op, true); + _emit_wmma_scale(L, op); } else if(strncmp(op_id, "wmma_", 5) == 0) { @@ -173,10 +170,10 @@ static void _op_tile_mma(rocke_lower_t* L, const rocke_op_t* op) } } -/* gfx1250 native MX FP8 WMMA. ROCm 7.13 introduced the LLVM 23 ABI: +/* gfx1250 native scaled FP8/FP4 WMMA. ROCm 7.13 introduced the LLVM 23 ABI: * matrix operands are <16 x i32>; SCALE carries four packed E8M0 bytes in * each i32 scale operand and SCALE16 carries eight in i64. */ -static void _emit_wmma_scale(rocke_lower_t* L, const rocke_op_t* op, bool scale16) +static void _emit_wmma_scale(rocke_lower_t* L, const rocke_op_t* op) { const char* intrinsic; const char* decl_key; @@ -187,8 +184,16 @@ static void _emit_wmma_scale(rocke_lower_t* L, const rocke_op_t* op, bool scale1 { return; } - op_name = scale16 ? "tile.wmma_scale16_f32_16x16x128_fp8_fp8" - : "tile.wmma_scale_f32_16x16x128_fp8_fp8"; + const rocke_scaled_wmma_op_t* spec = rocke_gfx1250_scaled_wmma_from_op(op); + if(!spec) + { + rocke_ll_fail(L, ROCKE_ERR_NOTIMPL, "unsupported scaled WMMA op '%s'", op->name); + } + const bool scale16 = spec->scales.block_k == 16; + const int fmt = spec->matrix_format; + char concrete_name[160]; + snprintf(concrete_name, sizeof(concrete_name), "tile.%s", spec->op_id); + op_name = concrete_name; if(!L->backend || strcmp(L->backend->gfx, "gfx1250") != 0) { rocke_ll_fail(L, @@ -211,7 +216,9 @@ static void _emit_wmma_scale(rocke_lower_t* L, const rocke_op_t* op, bool scale1 L, ROCKE_ERR_VALUE, "%s expects 5 operands, got %d", op_name, op->num_operands); } - scale_ty = scale16 ? "i64" : "i32"; + char packed_type[8]; + snprintf(packed_type, sizeof(packed_type), "i%d", rocke_e8m0_scale_word_bits(&spec->scales)); + scale_ty = packed_type; if(strcmp(op->operands[3]->type->name, scale_ty) != 0 || strcmp(op->operands[4]->type->name, scale_ty) != 0) { @@ -237,12 +244,14 @@ static void _emit_wmma_scale(rocke_lower_t* L, const rocke_op_t* op, bool scale1 rocke_ll_need(L, decl_key); rocke_ll_emitf(L, " %s = call <8 x float> @%s(" - "i32 0, <16 x i32> %s, i32 0, <16 x i32> %s, " + "i32 %d, <16 x i32> %s, i32 %d, <16 x i32> %s, " "i16 0, <8 x float> %s, i32 0, i32 0, %s %s, " "i32 0, i32 0, %s %s, i1 false, i1 false)", mma_result_name(L, op), intrinsic, + fmt, rocke_ll_operand(L, op->operands[0]), + fmt, rocke_ll_operand(L, op->operands[1]), rocke_ll_operand(L, op->operands[2]), scale_ty, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h new file mode 100644 index 000000000000..4b18e8331e9f --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h @@ -0,0 +1,68 @@ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +/* Shared gfx1250 operand contracts. Mirrors core/arch/wmma_scale.py: + * E8M0ScalePacking -> rocke_e8m0_scale_packing_t + * ScaledWmmaOp -> rocke_scaled_wmma_op_t + * gfx1250_scaled_wmma -> rocke_gfx1250_scaled_wmma + */ +#ifndef ROCKE_WMMA_SCALE_INTERNAL_H +#define ROCKE_WMMA_SCALE_INTERNAL_H + +#include + +#include "rocke/ir.h" + +typedef struct rocke_e8m0_scale_packing +{ + /* Consecutive K groups, first group in the low byte, independently for A/B. */ + int count; + int block_k; +} rocke_e8m0_scale_packing_t; + +typedef struct rocke_scaled_wmma_op +{ + const char* op_id; + int matrix_format; + rocke_e8m0_scale_packing_t scales; +} rocke_scaled_wmma_op_t; + +static inline const rocke_scaled_wmma_op_t* rocke_gfx1250_scaled_wmma(const char* op_id) +{ + static const rocke_e8m0_scale_packing_t scale = {/*count=*/4, /*block_k=*/32}; + static const rocke_e8m0_scale_packing_t scale16 = {/*count=*/8, /*block_k=*/16}; + static const rocke_scaled_wmma_op_t ops[] = { + {"wmma_scale_f32_16x16x128_fp8_fp8", 0, scale}, + {"wmma_scale_f32_16x16x128_bf8_bf8", 1, scale}, + {"wmma_scale16_f32_16x16x128_fp8_fp8", 0, scale16}, + {"wmma_scale16_f32_16x16x128_bf8_bf8", 1, scale16}, + }; + if(!op_id) + { + return NULL; + } + if(strncmp(op_id, "tile.", 5) == 0) + { + op_id += 5; + } + for(size_t i = 0; i < sizeof(ops) / sizeof(ops[0]); ++i) + { + if(strcmp(op_id, ops[i].op_id) == 0) + { + return &ops[i]; + } + } + return NULL; +} + +static inline const rocke_scaled_wmma_op_t* rocke_gfx1250_scaled_wmma_from_op(const rocke_op_t* op) +{ + const char* op_id = rocke_attr_get_str(&op->attrs, "op_id"); + return rocke_gfx1250_scaled_wmma(op_id ? op_id : op->name); +} + +static inline int rocke_e8m0_scale_word_bits(const rocke_e8m0_scale_packing_t* packing) +{ + return packing->count * 8; +} + +#endif /* ROCKE_WMMA_SCALE_INTERNAL_H */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json index 0607dc02932c..4269fb380b17 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json @@ -142,7 +142,7 @@ ] }, "gfx1250": { - "_comment": "gfx1250-class CDNA multi-chip device on the GFX12 programming model (wave32, WMMA, no MFMA). Primary fp16/bf16 WMMA atom is 16x16x32 (NOT gfx1201's 16x16x16). ABI empirically confirmed on ROCm 7.13 clang 23 (gfx1250 device target): __builtin_amdgcn_wmma_f32_16x16x32_f16 -> llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1,<16 x half>,i1,<16 x half>,i16,<8 x float>,i1,i1); bf16 uses v16bf16 (<16 x bfloat>) directly, not the gfx11 <16 x i16> bitcast. Datalayout/triple byte-identical to gfx950/gfx1201. This catalog ships validated fp16/bf16 and fp8/bf8 WMMA atoms plus the FP8xFP8 SCALE/SCALE16 prerequisite slice; FP4/FP6/BF6 block-scaled forms remain intentionally omitted. Lane maps are a HYPOTHESIS verified by examples/gfx1250/wmma_probe.py. MEMORY: gfx1250 HAS async global->LDS DMA (its own GFX12 family: global_load_async_to_lds_b128 + s_wait_asynccnt -- NOT the gfx9 buffer_load_lds/global_load_lds, which do not select), so has_async_global_lds=true (GPU-validated via the DTLA path). It also HAS a hardware transpose-LDS read, ds_load_tr16_b128 (GPU-validated via compute_pv_dstr). The has_async_lds and has_ds_read_tr flags stay false on purpose: in this catalog they denote the gfx9/gfx950 ds_read_*_tr_* / async-LDS *ABI* used to admit the gfx950 MFMA tiled builder (attention_arch.py), which gfx1250 does NOT have -- its async/transpose use distinct GFX12 opcodes gated directly in the ISA backend, not via these MFMA-pipeline bits. has_tdm=false: TDM not verified on this part.", + "_comment": "gfx1250-class CDNA multi-chip device on the GFX12 programming model (wave32, WMMA, no MFMA). Primary fp16/bf16 WMMA atom is 16x16x32 (NOT gfx1201's 16x16x16). ABI empirically confirmed on ROCm 7.13 clang 23 (gfx1250 device target): __builtin_amdgcn_wmma_f32_16x16x32_f16 -> llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1,<16 x half>,i1,<16 x half>,i16,<8 x float>,i1,i1); bf16 uses v16bf16 (<16 x bfloat>) directly, not the gfx11 <16 x i16> bitcast. Datalayout/triple byte-identical to gfx950/gfx1201. This catalog ships validated fp16/bf16 and fp8/bf8 WMMA atoms plus the FP8xFP8 SCALE/SCALE16 prerequisite slice; FP4xFP4 SCALE/SCALE16 accepts prepacked E2M1 bytes; FP6/BF6 forms remain omitted. Lane maps are a HYPOTHESIS verified by examples/gfx1250/wmma_probe.py. MEMORY: gfx1250 HAS async global->LDS DMA (its own GFX12 family: global_load_async_to_lds_b128 + s_wait_asynccnt -- NOT the gfx9 buffer_load_lds/global_load_lds, which do not select), so has_async_global_lds=true (GPU-validated via the DTLA path). It also HAS a hardware transpose-LDS read, ds_load_tr16_b128 (GPU-validated via compute_pv_dstr). The has_async_lds and has_ds_read_tr flags stay false on purpose: in this catalog they denote the gfx9/gfx950 ds_read_*_tr_* / async-LDS *ABI* used to admit the gfx950 MFMA tiled builder (attention_arch.py), which gfx1250 does NOT have -- its async/transpose use distinct GFX12 opcodes gated directly in the ISA backend, not via these MFMA-pipeline bits. has_tdm=false: TDM not verified on this part.", "family": "cdna", "target_family": "gfx12_cdna", "matrix_path": "wmma", @@ -178,7 +178,9 @@ {"family": "wmma", "a": "bf8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_fp8"}, {"family": "wmma", "a": "bf8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_bf8"}, {"family": "wmma_scale", "a": "fp8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 128, "op_id": "wmma_scale_f32_16x16x128_fp8_fp8"}, - {"family": "wmma_scale16", "a": "fp8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 128, "op_id": "wmma_scale16_f32_16x16x128_fp8_fp8"} + {"family": "wmma_scale", "a": "bf8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 128, "op_id": "wmma_scale_f32_16x16x128_bf8_bf8"}, + {"family": "wmma_scale16", "a": "fp8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 128, "op_id": "wmma_scale16_f32_16x16x128_fp8_fp8"}, + {"family": "wmma_scale16", "a": "bf8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 128, "op_id": "wmma_scale16_f32_16x16x128_bf8_bf8"} ] }, "gfx11-generic": { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py index fd8f09e839f1..4b4e95e42d61 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py @@ -40,6 +40,12 @@ "fp8e4m3": "fp8e4m3", "bf8": "bf8e5m2", "bf8e5m2": "bf8e5m2", + "fp6": "fp6e2m3", + "fp6e2m3": "fp6e2m3", + "bf6": "fp6e3m2", + "fp6e3m2": "fp6e3m2", + "fp4": "fp4e2m1", + "fp4e2m1": "fp4e2m1", # Integer WMMA: "iu8"/"iu4" are the RDNA WMMA integer operand families # (signedness is an instruction operand, not the dtype); "i32" is the # integer accumulator. Scalar int spellings pass through for completeness. @@ -749,7 +755,7 @@ class _FragInfo: None, _wmma_gfx12_acc_16x16, ), - # Native gfx1250 MX FP8 WMMA. A/B each carry 64 bytes as <16 x i32>; + # Native gfx1250 scaled WMMA. FP8/BF8 use 64 bytes per lane as <16 x i32>. # SCALE packs four K=32 E8M0 bytes in i32 and SCALE16 packs eight K=16 # bytes in i64. Both share the gfx12 column-distributed accumulator. "wmma_scale_f32_16x16x128_fp8_fp8": _FragInfo( @@ -761,6 +767,15 @@ class _FragInfo: None, _wmma_gfx12_acc_16x16, ), + "wmma_scale_f32_16x16x128_bf8_bf8": _FragInfo( + 16, + 16, + 8, + 32, + None, + None, + _wmma_gfx12_acc_16x16, + ), "wmma_scale16_f32_16x16x128_fp8_fp8": _FragInfo( 16, 16, @@ -770,6 +785,15 @@ class _FragInfo: None, _wmma_gfx12_acc_16x16, ), + "wmma_scale16_f32_16x16x128_bf8_bf8": _FragInfo( + 16, + 16, + 8, + 32, + None, + None, + _wmma_gfx12_acc_16x16, + ), "wmma_gfx1250_f32_16x16x32_bf16": _FragInfo( 16, 16, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py new file mode 100644 index 000000000000..3a80ab24c451 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py @@ -0,0 +1,63 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""gfx1250 scaled-WMMA operand contracts shared by packing and lowering.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class E8M0ScalePacking: + """Pack consecutive K groups into a word, first group in its low byte. + + Each scale is one E8M0 byte. Both A and B use this ordering independently; + the integer word is only a carrier for the encoded floating-point scales. + """ + + count: int + block_k: int + + @property + def element_bits(self) -> int: + return 8 + + @property + def word_bits(self) -> int: + return self.count * self.element_bits + + @property + def llvm_type(self) -> str: + return f"i{self.word_bits}" + + +@dataclass(frozen=True) +class ScaledWmmaOp: + """One supported matrix format and its packed E8M0 scale contract.""" + + op_id: str + matrix_dtype: str + scales: E8M0ScalePacking + + @property + def scale16(self) -> bool: + return self.scales.block_k == 16 + + @property + def matrix_format(self) -> int: + return {"fp8": 0, "bf8": 1}[self.matrix_dtype] + + +_SCALE = E8M0ScalePacking(count=4, block_k=32) +_SCALE16 = E8M0ScalePacking(count=8, block_k=16) +_GFX1250_WMMA_SCALE = { + op_id: ScaledWmmaOp(op_id=op_id, matrix_dtype=dtype, scales=packing) + for family, packing in (("wmma_scale", _SCALE), ("wmma_scale16", _SCALE16)) + for dtype in ("fp8", "bf8") + for op_id in (f"{family}_f32_16x16x128_{dtype}_{dtype}",) +} + + +def gfx1250_scaled_wmma(op_id: str) -> ScaledWmmaOp | None: + """Resolve a catalog ID or concrete tile op name; reject unknown variants.""" + return _GFX1250_WMMA_SCALE.get(op_id.removeprefix("tile.")) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py index f77f6463f7f2..14a0348c9b06 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py @@ -86,7 +86,9 @@ def __repr__(self) -> str: "mfma_f32_16x16x128_fp8": "acc128", "mfma_scale_f32_16x16x128_f8f6f4": "mxacc", "wmma_scale_f32_16x16x128_fp8_fp8": "mxacc", + "wmma_scale_f32_16x16x128_bf8_bf8": "mxacc", "wmma_scale16_f32_16x16x128_fp8_fp8": "mxacc", + "wmma_scale16_f32_16x16x128_bf8_bf8": "mxacc", } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/isa/backend.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/isa/backend.py index 8970d949570e..ae92acb1de08 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/isa/backend.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/isa/backend.py @@ -31,6 +31,7 @@ from typing import Callable, Dict, Tuple, Union from ..arch import ArchTarget +from ..arch.wmma_scale import gfx1250_scaled_wmma class ISABackend: @@ -332,23 +333,6 @@ class Gfx9MfmaBackend(ISABackend): } -# gfx1250 native MX FP8 WMMA, available with the LLVM 23 toolchain used by -# ROCm 7.13+. Both operations consume <16 x i32> matrix fragments; SCALE packs -# four E8M0 scale bytes in i32 while SCALE16 packs eight in i64. -_GFX1250_WMMA_FP8_SCALE = { - "tile.wmma_scale_f32_16x16x128_fp8_fp8": ( - "wmma.scale.gfx1250.f32.16x16x128.fp8.fp8", - "llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32", - "i32", - ), - "tile.wmma_scale16_f32_16x16x128_fp8_fp8": ( - "wmma.scale16.gfx1250.f32.16x16x128.fp8.fp8", - "llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32", - "i64", - ), -} - - class Gfx11RdnaBackend(ISABackend): """RDNA3 / RDNA3.5 (gfx11, e.g. gfx1151 Strix Halo). **wave32**, **WMMA** (no MFMA), and a distinct ``s_waitcnt`` layout from gfx9/10. Datalayout + @@ -575,9 +559,9 @@ def emit_lds_barrier_drain(self, lowerer, *, drain_vmem: bool) -> None: lowerer._current().emit(" call void @llvm.amdgcn.s.wait.dscnt(i16 0)") def emit_wmma(self, lowerer, op) -> None: - scale_spec = _GFX1250_WMMA_FP8_SCALE.get(op.name) + scale_spec = gfx1250_scaled_wmma(op.name) if scale_spec is not None: - self._emit_wmma_scale(lowerer, op, scale_spec) + self._emit_wmma_scale(lowerer, op) return fp8_spec = _GFX1250_WMMA_FP8.get(op.name) if fp8_spec is not None: @@ -585,9 +569,14 @@ def emit_wmma(self, lowerer, op) -> None: return spec = _GFX1250_WMMA.get(op.name) if spec is None: + scaled_ops = [ + f"tile.{atom.op_id}" + for atom in self.arch.mma.ops + if gfx1250_scaled_wmma(atom.op_id) is not None + ] raise NotImplementedError( f"WMMA op {op.name!r} not yet wired for {self.arch.gfx}; " - f"known: {sorted(_GFX1250_WMMA) + sorted(_GFX1250_WMMA_FP8) + sorted(_GFX1250_WMMA_FP8_SCALE)}" + f"known: {sorted(_GFX1250_WMMA) + sorted(_GFX1250_WMMA_FP8) + sorted(scaled_ops)}" ) decl_key, intrinsic, elt = spec a, b, c = op.operands @@ -622,15 +611,23 @@ def _emit_wmma_fp8(self, lowerer, op, spec) -> None: f"i1 false, i1 false)" ) - def _emit_wmma_scale(self, lowerer, op, spec) -> None: - """Emit the ROCm 7.13+ gfx1250 SCALE/SCALE16 FP8 call.""" + def _emit_wmma_scale(self, lowerer, op) -> None: + """Emit the ROCm 7.13+ gfx1250 SCALE/SCALE16 FP8/FP4 call.""" if lowerer._flavor != "llvm23": raise NotImplementedError( f"{op.name} requires llvm23 (ROCm 7.13+), got {lowerer._flavor}" ) if len(op.operands) != 5: raise ValueError(f"{op.name} expects 5 operands, got {len(op.operands)}") - decl_key, intrinsic, scale_ty = spec + spec = gfx1250_scaled_wmma(op.name) + if spec is None: + raise NotImplementedError(f"unsupported scaled WMMA op {op.name!r}") + mode = "scale16" if spec.scale16 else "scale" + # Declaration keys are shared across matrix dtypes: the ABI is identical. + decl_key = f"wmma.{mode}.gfx1250.f32.16x16x128.fp8.fp8" + intrinsic = f"llvm.amdgcn.wmma.{mode}.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32" + scale_ty = spec.scales.llvm_type + fmt = spec.matrix_format a, b, c, a_scale, b_scale = op.operands if a_scale.type.name != scale_ty or b_scale.type.name != scale_ty: raise ValueError( @@ -640,8 +637,8 @@ def _emit_wmma_scale(self, lowerer, op, spec) -> None: lowerer._need(decl_key) lowerer._current().emit( f" {op.result.name} = call <8 x float> @{intrinsic}(" - f"i32 0, <16 x i32> {lowerer._operand(a)}, " - f"i32 0, <16 x i32> {lowerer._operand(b)}, " + f"i32 {fmt}, <16 x i32> {lowerer._operand(a)}, " + f"i32 {fmt}, <16 x i32> {lowerer._operand(b)}, " f"i16 0, <8 x float> {lowerer._operand(c)}, " f"i32 0, i32 0, {scale_ty} {lowerer._operand(a_scale)}, " f"i32 0, i32 0, {scale_ty} {lowerer._operand(b_scale)}, " diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py index bbd4e482c8de..b980755a8624 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py @@ -17,6 +17,7 @@ from typing import List, Optional +from .arch.wmma_scale import gfx1250_scaled_wmma from .ir import ( KernelDef, Op, @@ -653,10 +654,16 @@ def _op_tile_wmma_gfx1250_f32_16x16x64_bf8_bf8(self, op: Op) -> None: self._emit_wmma_gfx1250_fp8(op, "bf8_bf8") def _op_tile_wmma_scale_f32_16x16x128_fp8_fp8(self, op: Op) -> None: - self._emit_wmma_gfx1250_scaled(op, scale16=False) + self._emit_wmma_gfx1250_scaled(op) + + def _op_tile_wmma_scale_f32_16x16x128_bf8_bf8(self, op: Op) -> None: + self._emit_wmma_gfx1250_scaled(op) def _op_tile_wmma_scale16_f32_16x16x128_fp8_fp8(self, op: Op) -> None: - self._emit_wmma_gfx1250_scaled(op, scale16=True) + self._emit_wmma_gfx1250_scaled(op) + + def _op_tile_wmma_scale16_f32_16x16x128_bf8_bf8(self, op: Op) -> None: + self._emit_wmma_gfx1250_scaled(op) def _emit_wmma_gfx1250_fp8(self, op: Op, ab: str) -> None: # gfx1250 K=64 FP8/BF8 builtin: A/B are <8 x i32> (32 low-bit @@ -669,22 +676,22 @@ def _emit_wmma_gfx1250_fp8(self, op: Op, ab: str) -> None: f"{_name(a)}, {_name(b)}, (int16_t)0, {_name(c)}, false, false);" ) - def _emit_wmma_gfx1250_scaled(self, op: Op, *, scale16: bool) -> None: - op_id = ( - "wmma_scale16_f32_16x16x128_fp8_fp8" - if scale16 - else "wmma_scale_f32_16x16x128_fp8_fp8" - ) + def _emit_wmma_gfx1250_scaled(self, op: Op) -> None: + spec = gfx1250_scaled_wmma(op.name) + if spec is None: + raise NotImplementedError(f"unsupported scaled WMMA op {op.name!r}") + op_id = spec.op_id + fmt = spec.matrix_format self._require_wmma_arch(op_id) a, b, c, a_scale, b_scale = op.operands builtin = ( "__builtin_amdgcn_wmma_scale16_f32_16x16x128_f8f6f4" - if scale16 + if spec.scale16 else "__builtin_amdgcn_wmma_scale_f32_16x16x128_f8f6f4" ) self._emit( f"f32x8 {_name(op.result)} = {builtin}(" - f"0, {_name(a)}, 0, {_name(b)}, (int16_t)0, {_name(c)}, " + f"{fmt}, {_name(a)}, {fmt}, {_name(b)}, (int16_t)0, {_name(c)}, " f"0, 0, {_name(a_scale)}, 0, 0, {_name(b_scale)}, false, false);" ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md new file mode 100644 index 000000000000..ae28f7a242d8 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md @@ -0,0 +1,28 @@ +# gfx1250 block-scaled GEMM examples + +`mxfp8_gemm` runs homogeneous FP8 E4M3 and BF8 E5M2 separately by default. +Use `--dtype fp8` or `--dtype bf8` to run one encoding; `--dtype both` is +the explicit default. Canonical names are `fp8e4m3` and `bf8e5m2`. + +```bash +ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.mxfp8_gemm +ROCKE_LLVM_FLAVOR=llvm23 python -m rocke.examples.gfx1250.gemm.mxfp8_gemm --dtype bf8 --matrix-path wmma_scale16 +``` + +SCALE uses E8M0 blocks of 32 K elements; SCALE16 uses blocks of 16. +Use `--compile-route hip` for HIP compilation and `--case all` for neutral, +one-sided, combined, and isolated scale-group fixtures. The `mixed` fixture +means scale variation, not different A/B matrix formats. + +The examples share argument parsing, the spec-driven builder, compiler, +launcher, and numerical verifier. Numerical tests use opt-in fixtures in +`tests/instances/conftest.py` and are independently runnable by family. + +The global architecture aliases also recognize `fp4e2m1`, `fp6e2m3`, and +`fp6e3m2` (`fp4`, `fp6`, `bf6`). Recognition does not imply that a target +supports an atom; use its catalog to query support. Packed gfx1250 FP4, +FP6, mixed formats, and alternative scale formats are separate extensions. + +`e8m0` identifies a scale format carried in bytes and packed integer operands. +It is not a general scalar IR type or conversion API. Operation IDs retain +their instruction-format tokens. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/_scaled_gemm_example.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/_scaled_gemm_example.py new file mode 100644 index 000000000000..c19179a4b992 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/_scaled_gemm_example.py @@ -0,0 +1,46 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Shared arguments and execution for focused scaled GEMM examples.""" + +from __future__ import annotations + +import argparse + +from .block_scaled_gemm_verify import run_cases + + +def argument_parser(description: str) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=description) + parser.add_argument("--m", type=int, default=32) + parser.add_argument("--n", type=int, default=48) + parser.add_argument("--k", type=int, choices=(128, 256), default=256) + parser.add_argument( + "--matrix-path", + choices=("wmma_scale", "wmma_scale16"), + default="wmma_scale", + help="SCALE uses K=32 scale blocks; SCALE16 uses K=16 blocks", + ) + parser.add_argument("--compile-route", choices=("comgr", "hip"), default="comgr") + parser.add_argument( + "--case", + default="mixed", + help="neutral, a-only, b-only, mixed, group-N, or all", + ) + return parser + + +def verify(spec, args: argparse.Namespace) -> int: + """Prepare bounded inputs, compile, launch, and compare with the reference. + + The shared verifier owns packing, independent decoding, HIP allocations, + and result checks. Family examples select the public input contract. + """ + cases = ( + ("neutral", "a-only", "b-only", "mixed") + + tuple(f"group-{g}" for g in range(spec.K // spec.block_k)) + if args.case == "all" + else (args.case,) + ) + count = run_cases(spec, cases, compile_route=args.compile_route) + print(f"PASS: verified {count} cases", flush=True) + return 0 diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py index f703106d894e..bb29fee041ef 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py @@ -3,7 +3,7 @@ """Launch gfx1250 block-scaled GEMM and compare with an independent reference. The default invocation keeps the K=64 FP8/BF8 WMMA + FP32-scale verifier. -Native ``--matrix-path wmma_scale`` / ``wmma_scale16`` use FP8 and E8M0 +Native ``--matrix-path wmma_scale`` / ``wmma_scale16`` use FP8/BF8 and E8M0 scales with K=32 / K=16 groups. Native fixtures cover K=128 or 256 and use bounded dyadic values, permitting exact comparison after BF16 rounding. @@ -29,7 +29,9 @@ from ....helpers import compile_kernel from ....helpers.compile import compile_kernel_via_hipcc +from ....core.arch.target import normalize_dtype from ....instances.gfx1250.block_scaled_gemm import ( + _LOWBIT_DTYPES, BlockScaledGemmSpec, block_scaled_gemm_grid, build_block_scaled_gemm, @@ -65,8 +67,10 @@ def reference_result( sa = decode_e8m0(a_scale) if native else a_scale.astype(np.float64) sb = decode_e8m0(b_scale) if native else b_scale.astype(np.float64) - scaled_a = a.astype(np.float64) * np.repeat(sa, block_k, axis=1) - scaled_b = b.astype(np.float64) * np.repeat(sb.T, block_k, axis=1) + a_values = a.astype(np.float64) + b_values = b.astype(np.float64) + scaled_a = a_values * np.repeat(sa, block_k, axis=1) + scaled_b = b_values * np.repeat(sb.T, block_k, axis=1) ref = scaled_a @ scaled_b.T if native: ref = ref.astype(ml_dtypes.bfloat16) @@ -98,8 +102,9 @@ def make_case_inputs( lowbit_types[spec.dtype_b] ) if native: - sa = rng.integers(125, 131, size=(spec.M, groups), dtype=np.uint8) - sb = rng.integers(125, 131, size=(groups, spec.N), dtype=np.uint8) + max_exponent = 131 + sa = rng.integers(125, max_exponent, size=(spec.M, groups), dtype=np.uint8) + sb = rng.integers(125, max_exponent, size=(groups, spec.N), dtype=np.uint8) neutral = 127 else: sa = rng.uniform(0.5, 1.5, size=(spec.M, groups)).astype(np.float32) @@ -211,7 +216,7 @@ def run_cases( inputs = make_case_inputs(spec, case) expected = reference_result(*inputs, spec.block_k, native=native) label = ( - f"{spec.resolved_matrix_path()}/{compile_route}/{case} " + f"{spec.resolved_matrix_path()}/{spec.dtype_a}/{compile_route}/{case} " f"{spec.M}x{spec.N}x{spec.K} bk{spec.block_k}" ) got = _launch(rt, fn, spec, inputs) @@ -233,7 +238,12 @@ def main(argv: list[str] | None = None) -> int: p.add_argument("--n", type=int, default=16) p.add_argument("--k", type=int, default=128) p.add_argument("--block-k", type=int, default=None) - p.add_argument("--dtype", default="fp8e4m3", choices=("fp8e4m3", "bf8e5m2")) + p.add_argument( + "--dtype", + default="fp8e4m3", + type=normalize_dtype, + choices=sorted(_LOWBIT_DTYPES), + ) p.add_argument("--tol", type=float, default=2e-2, help="legacy WMMA tolerance only") p.add_argument( "--matrix-path", default="wmma", choices=("wmma", "wmma_scale", "wmma_scale16") diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/mxfp8_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/mxfp8_gemm.py new file mode 100644 index 000000000000..f1ff04090c6c --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/mxfp8_gemm.py @@ -0,0 +1,44 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Homogeneous FP8 E4M3 or BF8 E5M2 GEMM with E8M0 block scales on gfx1250.""" + +from __future__ import annotations + +from ....core.arch.target import normalize_dtype +from ....instances.gfx1250.block_scaled_gemm import BlockScaledGemmSpec +from ._scaled_gemm_example import argument_parser, verify + + +def make_spec(args) -> BlockScaledGemmSpec: + return BlockScaledGemmSpec( + name="mxfp8_gemm", + M=args.m, + N=args.n, + K=args.k, + dtype_a=args.dtype, + dtype_b=args.dtype, + dtype_c="bf16", + scale_dtype="e8m0", + matrix_path=args.matrix_path, + block_k=16 if args.matrix_path == "wmma_scale16" else 32, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argument_parser(__doc__) + parser.add_argument( + "--dtype", + type=normalize_dtype, + choices=("fp8e4m3", "bf8e5m2", "both"), + default="both", + ) + args = parser.parse_args(argv) + dtypes = ("fp8e4m3", "bf8e5m2") if args.dtype == "both" else (args.dtype,) + for dtype in dtypes: + args.dtype = dtype + verify(make_spec(args), args) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py index 4a8b128d4169..8a4bcc96502a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py @@ -13,6 +13,8 @@ from dataclasses import dataclass from typing import List, Tuple +from ...core.arch.target import normalize_dtype +from ...core.arch.wmma_scale import gfx1250_scaled_wmma from ...core.ir import ( BF16, F16, @@ -29,7 +31,8 @@ from ...helpers.quant import quant_ir_type from ...helpers.spec import SignatureBuilder, ceil_div_grid, kernel_name_join -_LOWBIT_DTYPES = {"fp8", "fp8e4m3", "bf8", "bf8e5m2"} +_LOWBIT_FORMATS = {"fp8e4m3": "fp8", "bf8e5m2": "bf8"} +_LOWBIT_DTYPES = frozenset(_LOWBIT_FORMATS) _OUTPUT_DTYPES = {"fp16", "f16", "bf16"} _SCALE_DTYPES = {"fp16", "f16", "fp32", "f32"} _SUPPORTED_MATRIX_PATHS = { @@ -56,11 +59,11 @@ def _wmma_op_id(dtype_a: str, dtype_b: str) -> str: def _canon_lowbit(dtype: str) -> str: - if dtype in ("fp8", "fp8e4m3"): - return "fp8" - if dtype in ("bf8", "bf8e5m2"): - return "bf8" - raise ValueError(f"expected fp8/bf8 low-bit dtype, got {dtype!r}") + """Map a normalized catalog dtype to its stable instruction-format token.""" + try: + return _LOWBIT_FORMATS[normalize_dtype(dtype)] + except KeyError: + raise ValueError(f"unsupported low-bit matrix dtype: {dtype!r}") from None def _wire_scale_dtype(dtype: str) -> str: @@ -79,8 +82,8 @@ class BlockScaledGemmSpec: M: int N: int K: int - dtype_a: str = "fp8" - dtype_b: str = "fp8" + dtype_a: str = "fp8e4m3" + dtype_b: str = "fp8e4m3" dtype_c: str = "bf16" dtype_acc: str = "fp32" scale_dtype: str = "fp32" @@ -91,6 +94,10 @@ class BlockScaledGemmSpec: tile_n: int = 16 tile_k: int = 128 + def __post_init__(self) -> None: + object.__setattr__(self, "dtype_a", normalize_dtype(self.dtype_a)) + object.__setattr__(self, "dtype_b", normalize_dtype(self.dtype_b)) + @property def block_size(self) -> int: return _WAVE @@ -142,7 +149,10 @@ def is_valid_spec(spec: BlockScaledGemmSpec, arch: str = "gfx1250") -> Tuple[boo if spec.M <= 0 or spec.N <= 0 or spec.K <= 0: return False, f"M/N/K must be positive (got M={spec.M}, N={spec.N}, K={spec.K})" - if spec.dtype_a not in _LOWBIT_DTYPES or spec.dtype_b not in _LOWBIT_DTYPES: + if ( + normalize_dtype(spec.dtype_a) not in _LOWBIT_DTYPES + or normalize_dtype(spec.dtype_b) not in _LOWBIT_DTYPES + ): return False, ( f"A/B must be fp8 or bf8 (got A={spec.dtype_a!r}, B={spec.dtype_b!r})" ) @@ -151,9 +161,13 @@ def is_valid_spec(spec: BlockScaledGemmSpec, arch: str = "gfx1250") -> Tuple[boo family = matrix_path if native_scale else "wmma" atom_k = _WMMA_SCALE_K if native_scale else _WMMA_K if native_scale and ( - _canon_lowbit(spec.dtype_a) != "fp8" or _canon_lowbit(spec.dtype_b) != "fp8" - ): - return False, "native gfx1250 SCALE/SCALE16 currently supports fp8 x fp8 only" + _canon_lowbit(spec.dtype_a), + _canon_lowbit(spec.dtype_b), + ) not in (("fp8", "fp8"), ("bf8", "bf8")): + return ( + False, + "native gfx1250 SCALE/SCALE16 supports matching fp8 or bf8 operands", + ) if not target.mma.has_shape( family=family, a_dtype=_canon_lowbit(spec.dtype_a), @@ -179,7 +193,11 @@ def is_valid_spec(spec: BlockScaledGemmSpec, arch: str = "gfx1250") -> Tuple[boo "native gfx1250 SCALE/SCALE16 requires packed E8M0 scale bytes " f"(got {spec.scale_dtype!r})" ) - required_block_k = 16 if matrix_path == "wmma_scale16" else 32 + scale_op = gfx1250_scaled_wmma( + f"{matrix_path}_f32_16x16x128_{_canon_lowbit(spec.dtype_a)}_{_canon_lowbit(spec.dtype_b)}" + ) + assert scale_op is not None + required_block_k = scale_op.scales.block_k if spec.block_k != required_block_k: return False, ( f"{matrix_path} requires block_k={required_block_k} E8M0 groups " @@ -202,7 +220,10 @@ def is_valid_spec(spec: BlockScaledGemmSpec, arch: str = "gfx1250") -> Tuple[boo return False, "M and N must be multiples of 16" if native_scale: - return True, f"ok: gfx1250 K=128 native {matrix_path} FP8 GEMM" + return ( + True, + f"ok: gfx1250 K=128 native {matrix_path} {_canon_lowbit(spec.dtype_a)} GEMM", + ) return True, "ok: gfx1250 K=64 FP8/BF8 WMMA block-scaled GEMM" @@ -232,6 +253,7 @@ def block_scaled_gemm_grid(spec: BlockScaledGemmSpec) -> Tuple[int, int, int]: def _storage_type(dtype: str) -> Type: + dtype = normalize_dtype(dtype) if dtype in ("fp16", "f16"): return F16 if dtype == "bf16": @@ -250,20 +272,21 @@ def _as_f32(b: IRBuilder, v): def build_block_scaled_gemm( spec: BlockScaledGemmSpec, arch: str = "gfx1250" ) -> KernelDef: - """Build a gfx1250 FP8/BF8 block-scaled GEMM (RCR, ``C = A @ B^T``). + """Build a gfx1250 FP8/BF8/FP4 block-scaled GEMM (RCR, ``C = A @ B^T``). One wave (32 lanes) computes one 16x16 output tile without LDS. The legacy ``wmma`` path uses K=64 FP8/BF8 atoms, accumulates each ``block_k`` group, and applies FP16/FP32 A/B scales in software. The native ``wmma_scale`` and - ``wmma_scale16`` paths use K=128 FP8 atoms and pass packed E8M0 scale bytes + ``wmma_scale16`` paths use K=128 FP8/FP4 atoms and pass packed E8M0 scale bytes directly to the instruction, with K=32 and K=16 scale groups respectively. Lane ``l`` owns output column ``l % 16`` and rows ``(l // 16) * 8 : (l // 16 + 1) * 8``. Legacy matrix fragments carry 32 - low-bit bytes per lane as ``<8 x i32>``. Native fragments carry 64 bytes as + low-bit bytes per lane as ``<8 x i32>``. Native FP8 fragments carry 64 bytes as ``<16 x i32>`` as four 16-byte K chunks, alternating chunks between lane halves. Both paths use the gfx12 column-distributed ``<8 x f32>`` accumulator layout. + Scale arrays remain A_scale[M, K/block_k] and B_scale[K/block_k, N]. """ ok, reason = is_valid_spec(spec, arch=arch) if not ok: @@ -276,10 +299,11 @@ def build_block_scaled_gemm( native_scale = matrix_path in ("wmma_scale", "wmma_scale16") scale_ty = I8 if native_scale else _scale_type(spec.scale_dtype) op_id = ( - f"{matrix_path}_f32_16x16x128_fp8_fp8" + f"{matrix_path}_f32_16x16x128_{_canon_lowbit(spec.dtype_a)}_{_canon_lowbit(spec.dtype_b)}" if native_scale else _wmma_op_id(spec.dtype_a, spec.dtype_b) ) + scale_op = gfx1250_scaled_wmma(op_id) atom_k = _WMMA_SCALE_K if native_scale else _WMMA_K frag_words = 16 if native_scale else _ACC a_frag_ty = VectorType(I32, frag_words) @@ -318,8 +342,9 @@ def build_block_scaled_gemm( n0 = ir.mul(ir.block_id_x(), c16) a_row = ir.add(m0, frag) # this lane's A row b_row = ir.add(n0, frag) # this lane's B row (= output col n) - a_base = ir.mul(a_row, cK) - b_base = ir.mul(b_row, cK) + storage_k = cK + a_base = ir.mul(a_row, storage_k) + b_base = ir.mul(b_row, storage_k) def _load_frag(ptr, base, storage_ty, k0): if not native_scale: @@ -354,9 +379,12 @@ def _load_frag(ptr, base, storage_ty, k0): return ir.bitcast(packed, a_frag_ty) def _pack_strided_scales(ptr, row_or_col, call_idx, *, for_b): - count = 8 if matrix_path == "wmma_scale16" else 4 - word_ty = I64 if count == 8 else I32 - packed = ir.const_i64(0) if count == 8 else ir.const_i32(0) + assert scale_op is not None + packing = scale_op.scales + count = packing.count + word_ty = I64 if packing.word_bits == 64 else I32 + word_const = ir.const_i64 if packing.word_bits == 64 else ir.const_i32 + packed = word_const(0) scale_groups = spec.K // spec.block_k for j in range(count): scale_group = call_idx * count + j @@ -369,7 +397,7 @@ def _pack_strided_scales(ptr, row_or_col, call_idx, *, for_b): ) byte = ir.global_load(ptr, idx, I8, align=1) widened = ir.zext(byte, word_ty) - shift = ir.const_i64(j * 8) if count == 8 else ir.const_i32(j * 8) + shift = word_const(j * packing.element_bits) shifted = ir.shl(widened, shift) packed = ir.lor(packed, shifted) return packed diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt b/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt index 9062cb78e9d5..7396b0535f8b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt @@ -2,8 +2,10 @@ # SPDX-License-Identifier: MIT # # By-layer C++ test registration for the rocKE engine. Included via -# add_subdirectory(tests) from the top-level rocke/platform/CMakeLists.txt. All targets -# link the engine archive rocke_core (built by the parent). Paths are relative to +# add_subdirectory(tests) from the top-level rocke/platform/CMakeLists.txt. All +# targets +# link the engine archive rocke_core (built by the parent). Paths are relative +# to # this file (tests/), keeping the tree copy-able. # --- core layer --------------------------------------------------------------- @@ -14,6 +16,19 @@ if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/core/smoke.cpp") endif() if(BUILD_TESTING) + add_executable(rocke_mma_catalog_lookup core/test_mma_catalog_lookup.cpp) + target_link_libraries(rocke_mma_catalog_lookup PRIVATE rocke_core) + add_test(NAME rocke_mma_catalog_lookup COMMAND rocke_mma_catalog_lookup) + set_tests_properties(rocke_mma_catalog_lookup PROPERTIES LABELS + "ckc;unit;host") + rocke_cxx_test_installed_name(_rocke_installed_name rocke_mma_catalog_lookup) + set_target_properties(rocke_mma_catalog_lookup + PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") + install(TARGETS rocke_mma_catalog_lookup + RUNTIME DESTINATION ${ROCKE_CXX_TEST_INSTALL_DIR}) + set_property(GLOBAL APPEND PROPERTY ROCKE_INSTALL_TEST_TARGETS + rocke_mma_catalog_lookup) + # ck.dsl.ir/v1 serializer round-trip + verifier test. Proves # lower(parse(serialize(k))) == lower(k) byte-for-byte, serialization # idempotence, and a clean verifier pass for the buildable GEMM configs. @@ -27,7 +42,8 @@ if(BUILD_TESTING) COMMAND rocke_ir_serialize_roundtrip) set_tests_properties(rocke_ir_serialize_roundtrip PROPERTIES LABELS "ckc;unit;host") - rocke_cxx_test_installed_name(_rocke_installed_name rocke_ir_serialize_roundtrip) + rocke_cxx_test_installed_name(_rocke_installed_name + rocke_ir_serialize_roundtrip) set_target_properties(rocke_ir_serialize_roundtrip PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") install(TARGETS rocke_ir_serialize_roundtrip @@ -38,13 +54,15 @@ if(BUILD_TESTING) # C++ parity for Python TestNewTargetIntrinsics. if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/core/future_intrinsic_lowering.cpp") - add_executable(rocke_future_intrinsic_lowering core/future_intrinsic_lowering.cpp) + add_executable(rocke_future_intrinsic_lowering + core/future_intrinsic_lowering.cpp) target_link_libraries(rocke_future_intrinsic_lowering PRIVATE rocke_core) add_test(NAME rocke_future_intrinsic_lowering COMMAND rocke_future_intrinsic_lowering) set_tests_properties(rocke_future_intrinsic_lowering PROPERTIES LABELS "ckc;unit;host") - rocke_cxx_test_installed_name(_rocke_installed_name rocke_future_intrinsic_lowering) + rocke_cxx_test_installed_name(_rocke_installed_name + rocke_future_intrinsic_lowering) set_target_properties(rocke_future_intrinsic_lowering PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") install(TARGETS rocke_future_intrinsic_lowering @@ -61,7 +79,8 @@ if(BUILD_TESTING) COMMAND rocke_flavor_datalayout_ssot) set_tests_properties(rocke_flavor_datalayout_ssot PROPERTIES LABELS "ckc;unit;host") - rocke_cxx_test_installed_name(_rocke_installed_name rocke_flavor_datalayout_ssot) + rocke_cxx_test_installed_name(_rocke_installed_name + rocke_flavor_datalayout_ssot) set_target_properties(rocke_flavor_datalayout_ssot PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") install(TARGETS rocke_flavor_datalayout_ssot @@ -98,10 +117,10 @@ if(BUILD_TESTING) # archs, byte-compared against the Python lowerer) lives in the Python driver # python/rocke/portable_ir/drivers/parity_matrix.py, which needs a Python # front end and so cannot be a ctest. - foreach(_pir_test dom_decoders recipe_vm_replay) - if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/portable_ir/${_pir_test}.cpp") - set(_pir_target "rocke_portable_ir_${_pir_test}") - add_executable(${_pir_target} portable_ir/${_pir_test}.cpp) + foreach(pir_test dom_decoders recipe_vm_replay) + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/portable_ir/${pir_test}.cpp") + set(_pir_target "rocke_portable_ir_${pir_test}") + add_executable(${_pir_target} portable_ir/${pir_test}.cpp) target_link_libraries(${_pir_target} PRIVATE rocke_core) if(UNIX) target_link_libraries(${_pir_target} PRIVATE m) @@ -113,7 +132,8 @@ if(BUILD_TESTING) PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") install(TARGETS ${_pir_target} RUNTIME DESTINATION ${ROCKE_CXX_TEST_INSTALL_DIR}) - set_property(GLOBAL APPEND PROPERTY ROCKE_INSTALL_TEST_TARGETS ${_pir_target}) + set_property(GLOBAL APPEND PROPERTY ROCKE_INSTALL_TEST_TARGETS + ${_pir_target}) endif() endforeach() @@ -129,10 +149,12 @@ if(BUILD_TESTING) endif() # --- instances layer -------------------------------------------------------- - if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/instances/tiled_attention_2d_reentrancy.cpp") + if(EXISTS + "${CMAKE_CURRENT_SOURCE_DIR}/instances/tiled_attention_2d_reentrancy.cpp") add_executable(rocke_tiled_attention_2d_reentrancy instances/tiled_attention_2d_reentrancy.cpp) - target_link_libraries(rocke_tiled_attention_2d_reentrancy PRIVATE rocke_core) + target_link_libraries(rocke_tiled_attention_2d_reentrancy PRIVATE + rocke_core) if(UNIX) target_link_libraries(rocke_tiled_attention_2d_reentrancy PRIVATE m) endif() @@ -140,7 +162,8 @@ if(BUILD_TESTING) COMMAND rocke_tiled_attention_2d_reentrancy) set_tests_properties(rocke_tiled_attention_2d_reentrancy PROPERTIES LABELS "ckc;unit;host") - rocke_cxx_test_installed_name(_rocke_installed_name rocke_tiled_attention_2d_reentrancy) + rocke_cxx_test_installed_name(_rocke_installed_name + rocke_tiled_attention_2d_reentrancy) set_target_properties(rocke_tiled_attention_2d_reentrancy PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") install(TARGETS rocke_tiled_attention_2d_reentrancy @@ -153,7 +176,8 @@ if(BUILD_TESTING) # whole-kernel LDS budget, the scaled-f8f6f4 arch gate and the fp8 wave-size # gate. The byte-identity gate only compares specs both engines accept, so an # acceptance divergence from the Python validators is invisible to it. - if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/instances/moe_fused_mega_spec_guards.cpp") + if(EXISTS + "${CMAKE_CURRENT_SOURCE_DIR}/instances/moe_fused_mega_spec_guards.cpp") add_executable(rocke_moe_fused_mega_spec_guards instances/moe_fused_mega_spec_guards.cpp) target_link_libraries(rocke_moe_fused_mega_spec_guards PRIVATE rocke_core) @@ -164,7 +188,8 @@ if(BUILD_TESTING) COMMAND rocke_moe_fused_mega_spec_guards) set_tests_properties(rocke_moe_fused_mega_spec_guards PROPERTIES LABELS "ckc;unit;host") - rocke_cxx_test_installed_name(_rocke_installed_name rocke_moe_fused_mega_spec_guards) + rocke_cxx_test_installed_name(_rocke_installed_name + rocke_moe_fused_mega_spec_guards) set_target_properties(rocke_moe_fused_mega_spec_guards PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") install(TARGETS rocke_moe_fused_mega_spec_guards @@ -182,7 +207,9 @@ if(BUILD_TESTING) # nothing for them. Guarded because rocke/platform also builds # standalone (outside hip-kernel-provider), where neither the shared # parser (apply_ctest_category_labels) nor - # ROCKE_ENGINE_CTEST_CATEGORIES_YAML/HIPKERNELPROVIDER_YAML_CATEGORIZATION_ENABLED + # + # ROCKE_ENGINE_CTEST_CATEGORIES_YAML/HIPKERNELPROVIDER_YAML_CATEGORIZATION_ENA + # BLED # exist. Without this, these tests carry only the hardcoded # "ckc;unit;host" label and silently drop out of `ctest -L ` # selection -- the mechanism TheRock CI's per-tier diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_mxfp8_llvm23.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_mxfp8_llvm23.json new file mode 100644 index 000000000000..ac6d786887b6 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_mxfp8_llvm23.json @@ -0,0 +1,6 @@ +{ + "wmma_scale/bf8/bf8/e8m0/e8m0": "58930ce80326df951f1c13fc57efd5dd895251dedc7dcd72a29b15e1ae5860fa", + "wmma_scale/fp8/fp8/e8m0/e8m0": "7a37dd799b3babf3e7357edeaaa7f0d466a677f461087cef2b6cbd455bdada6f", + "wmma_scale16/bf8/bf8/e8m0/e8m0": "20ee6884c4f6ea6d6c3454822377eb88aecd4081f4b7d296920fbd3417e4c0ed", + "wmma_scale16/fp8/fp8/e8m0/e8m0": "18f36023f8cae46fc6f099cdae42d9b489f13ce9f43444238a3c419d5696ac62" +} diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mxfp8_example.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mxfp8_example.py new file mode 100644 index 000000000000..2dee830650dc --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mxfp8_example.py @@ -0,0 +1,90 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Both homogeneous eight-bit formats are usable from the focused example.""" + +import argparse +from dataclasses import replace +import hashlib +import json +from pathlib import Path + +import numpy as np +import pytest + +from rocke.core.lower_hip import lower_kernel_to_hip +from rocke.core.arch.target import normalize_dtype +from rocke.core.lower_llvm import lower_kernel_to_llvm +from rocke.examples.gfx1250.gemm import _scaled_gemm_example, mxfp8_gemm +from rocke.examples.gfx1250.gemm.block_scaled_gemm_verify import ( + make_case_inputs, + reference_result, +) +from rocke.instances.gfx1250.block_scaled_gemm import build_block_scaled_gemm + +_GOLDENS = json.loads(Path(__file__).with_name("gfx1250_mxfp8_llvm23.json").read_text()) + + +@pytest.mark.parametrize( + "dtype,selector,ml_name", [("fp8", 0, "float8_e4m3fn"), ("bf8", 1, "float8_e5m2")] +) +@pytest.mark.parametrize("path", ["wmma_scale", "wmma_scale16"]) +def test_mxfp8_formats_and_existing_golden(dtype, selector, ml_name, path): + ml = pytest.importorskip("ml_dtypes") + spec = mxfp8_gemm.make_spec( + argparse.Namespace(m=32, n=48, k=256, dtype=dtype, matrix_path=path) + ) + assert spec.dtype_a == spec.dtype_b == normalize_dtype(dtype) + kernel = build_block_scaled_gemm(replace(spec, name="fp6_test")) + llvm = lower_kernel_to_llvm(kernel, arch="gfx1250", llvm_flavor="llvm23") + assert ( + hashlib.sha256(llvm.encode()).hexdigest() + == _GOLDENS[f"{path}/{dtype}/{dtype}/e8m0/e8m0"] + ) + hip = lower_kernel_to_hip(kernel, arch="gfx1250") + assert f"__builtin_amdgcn_{path}_f32_16x16x128_f8f6f4({selector}," in hip + inputs = make_case_inputs(spec, "mixed") + assert inputs[0].dtype == inputs[1].dtype == np.dtype(getattr(ml, ml_name)) + ref = reference_result(*inputs, spec.block_k, native=True) + assert ref.shape == (32, 48) and np.isfinite(ref).all() + + +@pytest.mark.parametrize( + "argv,dtypes", + [ + (["--dtype", "fp8e4m3"], ("fp8e4m3",)), + (["--dtype", " FP8 "], ("fp8e4m3",)), + (["--dtype", "bf8e5m2"], ("bf8e5m2",)), + (["--dtype", " BF8 "], ("bf8e5m2",)), + ([], ("fp8e4m3", "bf8e5m2")), + (["--dtype", "both"], ("fp8e4m3", "bf8e5m2")), + (["--dtype", "fp8"], ("fp8e4m3",)), + (["--dtype", "bf8"], ("bf8e5m2",)), + ], +) +def test_mxfp8_cli_selects_formats(monkeypatch, argv, dtypes): + calls = [] + + def run(spec, cases, **kwargs): + calls.append((spec, cases, kwargs)) + return len(cases) + + monkeypatch.setattr(_scaled_gemm_example, "run_cases", run) + assert ( + mxfp8_gemm.main( + argv + + [ + "--matrix-path", + "wmma_scale16", + "--compile-route", + "hip", + "--case", + "all", + ] + ) + == 0 + ) + assert tuple(spec.dtype_a for spec, _, _ in calls) == dtypes + for spec, cases, kwargs in calls: + assert spec.dtype_a == spec.dtype_b + assert spec.block_k == 16 and len(cases) == 20 + assert kwargs == {"compile_route": "hip"} diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py index 71028eed346a..2d43a966553e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py @@ -181,7 +181,7 @@ def test_native_slice_rejects_unsupported_contracts(self): ) ok, why = is_valid_spec(bad_dtype) self.assertFalse(ok) - self.assertIn("fp8 x fp8 only", why) + self.assertIn("matching fp8 or bf8 operands", why) bad_block = BlockScaledGemmSpec( name="bad_block", @@ -196,6 +196,19 @@ def test_native_slice_rejects_unsupported_contracts(self): self.assertFalse(ok) self.assertIn("requires block_k=16", why) + def test_unknown_scaled_atom_reports_supported_operations(self): + for scale16 in (False, True): + with self.subTest(scale16=scale16): + kernel = _build_scaled_atom(scale16=scale16) + call = next(op for op in kernel.body.ops if op.name == "tile.mma") + mode = "wmma_scale16" if scale16 else "wmma_scale" + call.attrs["op_id"] = f"{mode}_f32_16x16x128_fp16_fp16" + with self.assertRaisesRegex( + NotImplementedError, "not yet wired for gfx1250" + ) as error: + lower_kernel_to_llvm(kernel, arch="gfx1250", llvm_flavor="llvm23") + self.assertIn(f"{mode}_f32_16x16x128_bf8_bf8", str(error.exception)) + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py new file mode 100644 index 000000000000..e9fa5a3aebb1 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py @@ -0,0 +1,97 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Public dtype aliases resolve to explicit catalog keys without changing ops.""" + +import pytest +from rocke.core.arch import ArchTarget +from rocke.core.arch.target import normalize_dtype + + +@pytest.mark.parametrize( + "alias,canonical", + [ + ("fp8", "fp8e4m3"), + ("bf8", "bf8e5m2"), + ("fp6", "fp6e2m3"), + ("bf6", "fp6e3m2"), + ("fp4", "fp4e2m1"), + ], +) +def test_alias_catalog_lookup(alias, canonical): + assert normalize_dtype(alias) == canonical + assert normalize_dtype(" " + alias.upper() + " ") == canonical + assert normalize_dtype(canonical) == canonical + found = False + for gfx in ("gfx950", "gfx1250"): + catalog = ArchTarget.from_gfx(gfx).mma + for op in catalog.ops: + assert op.a_dtype not in ("fp8", "bf8", "fp6", "bf6", "fp4") + assert op.b_dtype not in ("fp8", "bf8", "fp6", "bf6", "fp4") + if op.a_dtype != canonical: + continue + found = True + assert catalog.has_shape( + family=op.family, + a_dtype=alias, + b_dtype=op.b_dtype, + c_dtype=op.c_dtype, + m=op.m, + n=op.n, + k=op.k, + ) + # BF6 is only present on the branch that supplies its packed atom. + if alias != "bf6": + assert found + + +@pytest.mark.parametrize("dtype", ["fp4", "fp4e2m1", "fp6", "fp6e2m3", "fp16"]) +def test_fp8_example_rejects_other_families(dtype): + from rocke.examples.gfx1250.gemm import mxfp8_gemm + + with pytest.raises(SystemExit) as exc: + mxfp8_gemm.main(["--dtype", dtype]) + assert exc.value.code == 2 + + +def test_supported_aliases_preserve_lowered_kernel(): + from dataclasses import replace + from rocke.core.lower_llvm import lower_kernel_to_llvm + from rocke.core.lower_hip import lower_kernel_to_hip + from rocke.instances.gfx1250.block_scaled_gemm import ( + _LOWBIT_DTYPES, + BlockScaledGemmSpec, + build_block_scaled_gemm, + block_scaled_gemm_signature, + ) + + for short in ("fp8", "bf8", "fp6", "bf6", "fp4"): + canonical = normalize_dtype(short) + if canonical not in _LOWBIT_DTYPES: + continue + for mode, block_k in (("wmma_scale", 32), ("wmma_scale16", 16)): + spec = BlockScaledGemmSpec( + name="alias_identity", + M=32, + N=48, + K=256, + dtype_a=short, + dtype_b=short, + scale_dtype="e8m0", + matrix_path=mode, + block_k=block_k, + ) + explicit = replace(spec, dtype_a=canonical, dtype_b=canonical) + mixed_case = replace(spec, dtype_a=" " + short.upper() + " ", dtype_b=short) + assert spec == explicit == mixed_case + assert spec.dtype_a == spec.dtype_b == canonical + assert hash(spec) == hash(explicit) == hash(mixed_case) + a, b = build_block_scaled_gemm(spec), build_block_scaled_gemm(explicit) + assert block_scaled_gemm_signature(spec) == block_scaled_gemm_signature( + explicit + ) + assert lower_kernel_to_llvm( + a, arch="gfx1250", llvm_flavor="llvm23" + ) == lower_kernel_to_llvm(b, arch="gfx1250", llvm_flavor="llvm23") + assert lower_kernel_to_hip(a, arch="gfx1250") == lower_kernel_to_hip( + b, arch="gfx1250" + ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp new file mode 100644 index 000000000000..927b4b2a912d --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp @@ -0,0 +1,72 @@ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +/* Exercise public catalog queries against stored keys, without normalizing + * away malformed table entries. Alias queries must find the same atom. */ +#include "rocke/arch_target.h" + +#include +#include +#include + +static const char* short_dtype(const char* dtype) +{ + const char* aliases[][2] = {{"fp8e4m3", "fp8"}, + {"bf8e5m2", "bf8"}, + {"fp6e2m3", "fp6"}, + {"fp6e3m2", "bf6"}, + {"fp4e2m1", "fp4"}}; + for(const auto& pair : aliases) + { + if(strcmp(dtype, pair[0]) == 0) + return pair[1]; + } + return dtype; +} + +int main() +{ + int checked = 0; + for(const char* gfx : {"gfx950", "gfx1250"}) + { + const rocke_arch_target_t* arch = rocke_arch_target_from_gfx(gfx); + if(!arch) + return 1; + for(int i = 0; i < arch->mma.num_ops; ++i) + { + const rocke_mma_op_t* op = &arch->mma.ops[i]; + for(const char* dtype : {op->a_dtype, op->b_dtype, op->c_dtype}) + { + char scratch[64]; + if(strcmp(dtype, rocke_normalize_dtype(dtype, scratch, sizeof(scratch))) != 0) + { + fprintf(stderr, "%s: noncanonical stored dtype %s\n", op->op_id, dtype); + return 1; + } + } + for(const char* a : {op->a_dtype, short_dtype(op->a_dtype)}) + { + for(const char* b : {op->b_dtype, short_dtype(op->b_dtype)}) + { + if(!rocke_mma_catalog_has_shape( + &arch->mma, op->family, a, b, op->c_dtype, op->m, op->n, op->k) + || rocke_mma_catalog_op_for_shape( + &arch->mma, op->family, a, b, op->c_dtype, op->m, op->n, op->k) + != op + || rocke_mma_catalog_select_largest_k( + &arch->mma, op->family, a, b, op->c_dtype, op->m, op->n, op->k) + != op + || rocke_mma_catalog_enumerate( + &arch->mma, op->family, a, b, op->c_dtype, op->m, op->n, NULL, 0) + < 1) + { + fprintf(stderr, "%s: catalog query missed %s/%s\n", op->op_id, a, b); + return 1; + } + } + } + ++checked; + } + } + printf("catalog lookup: %d rows passed canonical and alias queries\n", checked); + return 0; +} diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scaled_wmma_numeric.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/conftest.py similarity index 80% rename from dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scaled_wmma_numeric.py rename to dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/conftest.py index 9ceb92484625..c6cd4d321b8f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_scaled_wmma_numeric.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/conftest.py @@ -53,24 +53,14 @@ def gpu_env(): return env -@pytest.mark.parametrize( - "matrix_path,route,m,n,k,case,count", - [ - ("wmma_scale", "comgr", 16, 16, 128, "all", 8), - ("wmma_scale16", "comgr", 16, 16, 128, "all", 12), - ("wmma_scale", "comgr", 32, 48, 256, "mixed", 1), - ("wmma_scale16", "comgr", 32, 48, 256, "mixed", 1), - ("wmma_scale", "hip", 32, 48, 256, "mixed", 1), - ("wmma_scale16", "hip", 32, 48, 256, "mixed", 1), - ("wmma", "comgr", 16, 16, 128, "mixed", 1), - ], -) -def test_scaled_wmma_numeric(gpu_env, matrix_path, route, m, n, k, case, count): +def _run_numeric(gpu_env, dtype, matrix_path, route, m, n, k, case, count): result = subprocess.run( [ sys.executable, "-m", _MODULE, + "--dtype", + dtype, "--matrix-path", matrix_path, "--compile-route", @@ -94,3 +84,11 @@ def test_scaled_wmma_numeric(gpu_env, matrix_path, route, m, n, k, case, count): assert f"PASS: verified {count} cases" in output, output assert output.count("bad=0") == count, output print(output, end="") + + +@pytest.fixture +def numeric_case(gpu_env): + """Run one bounded scaled-GEMM case with the validated GPU environment.""" + from functools import partial + + return partial(_run_numeric, gpu_env) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c index bc5f0b71ea66..1fb8d10ac466 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c @@ -119,7 +119,7 @@ static void build_wmma_k64_bf8_bf8(rocke_ir_builder_t* b) /* K=128 FP8 SCALE/SCALE16 WMMA. Matrix fragments are <16 x i32>; packed * E8M0 scale operands are i32 for SCALE and i64 for SCALE16. */ -static void wmma_scaled(rocke_ir_builder_t* b, bool scale16) +static void wmma_scaled(rocke_ir_builder_t* b, bool scale16, const char* op_id) { const rocke_type_t* scale_ty = scale16 ? rocke_i64() : rocke_i32(); rocke_value_t* a_ptr = frag_param(b, "A", rocke_i32(), true); @@ -137,21 +137,30 @@ static void wmma_scaled(rocke_ir_builder_t* b, bool scale16) rocke_value_t* bb = rocke_b_vec_concat(b, b_lo, b_hi); rocke_value_t* c = rocke_b_global_load_vN(b, c_ptr, tid, rocke_f32(), 8, /*align=*/0); rocke_value_t* scale = rocke_b_global_load(b, scale_ptr, tid, scale_ty, /*align=*/1); - rocke_value_t* d = scale16 - ? rocke_b_wmma_scale16_f32_16x16x128_fp8_fp8(b, a, bb, c, scale, scale) - : rocke_b_wmma_scale_f32_16x16x128_fp8_fp8(b, a, bb, c, scale, scale); + rocke_value_t* scales[] = {scale, scale}; + rocke_value_t* d = rocke_b_mma(b, op_id, a, bb, c, scales, 2); rocke_b_global_store(b, c_ptr, tid, d, /*align=*/1); rocke_b_ret(b); } static void build_wmma_scale(rocke_ir_builder_t* b) { - wmma_scaled(b, false); + wmma_scaled(b, false, "wmma_scale_f32_16x16x128_fp8_fp8"); +} + +static void build_wmma_scale_bf8(rocke_ir_builder_t* b) +{ + wmma_scaled(b, false, "wmma_scale_f32_16x16x128_bf8_bf8"); } static void build_wmma_scale16(rocke_ir_builder_t* b) { - wmma_scaled(b, true); + wmma_scaled(b, true, "wmma_scale16_f32_16x16x128_fp8_fp8"); +} + +static void build_wmma_scale16_bf8(rocke_ir_builder_t* b) +{ + wmma_scaled(b, true, "wmma_scale16_f32_16x16x128_bf8_bf8"); } /* ds_read_b128_tr_b16. gfx950 has one type-agnostic opcode returning @@ -362,6 +371,8 @@ static const config_t CONFIGS[] = { {build_global_tr16_bf16, "gfx1250"}, {build_global_tr16_i16, "gfx1250"}, {build_tensor_transfers, "gfx1250"}, + {build_wmma_scale_bf8, "gfx1250"}, + {build_wmma_scale16_bf8, "gfx1250"}, }; static const int NUM_CONFIGS = (int)(sizeof(CONFIGS) / sizeof(CONFIGS[0])); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py index 80ca5f0f3075..bf23e17a7b1b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py @@ -254,6 +254,12 @@ def build_tensor_transfers(b: IRBuilder) -> None: ] +# Homogeneous BF8 belongs to the eight-bit example contract. +CONFIGS.extend( + (_wmma_scaled("bf8", "bf8", mode), "gfx1250") for mode in ("scale", "scale16") +) + + def _spec(idx: int): """Config selector: the (builder, arch) pair the shared driver expects.""" if not 0 <= idx < len(CONFIGS): diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp8_numeric.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp8_numeric.py new file mode 100644 index 000000000000..12e4b5ebced0 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp8_numeric.py @@ -0,0 +1,26 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""gfx1250 MXFP8 numerical cases.""" + +import pytest + + +@pytest.mark.parametrize( + "dtype,matrix_path,route,m,n,k,case,count", + [ + (dtype, *case) + for dtype in ("fp8", "bf8") + for case in [ + ("wmma_scale", "comgr", 16, 16, 128, "all", 8), + ("wmma_scale16", "comgr", 16, 16, 128, "all", 12), + ("wmma_scale", "comgr", 32, 48, 256, "mixed", 1), + ("wmma_scale16", "comgr", 32, 48, 256, "mixed", 1), + ("wmma_scale", "hip", 32, 48, 256, "mixed", 1), + ("wmma_scale16", "hip", 32, 48, 256, "mixed", 1), + ("wmma", "comgr", 16, 16, 128, "mixed", 1), + ] + if dtype == "fp8" or case[0] != "wmma" + ], +) +def test_numeric(numeric_case, dtype, matrix_path, route, m, n, k, case, count): + numeric_case(dtype, matrix_path, route, m, n, k, case, count) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp_examples.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp_examples.py new file mode 100644 index 000000000000..91e6513daf8c --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/test_gfx1250_mxfp_examples.py @@ -0,0 +1,30 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Focused examples build real kernels and constrain their input contracts.""" + +import argparse +import importlib + +import pytest + +from rocke.core.arch.target import normalize_dtype +from rocke.core.lower_llvm import lower_kernel_to_llvm +from rocke.instances.gfx1250.block_scaled_gemm import build_block_scaled_gemm + + +@pytest.mark.parametrize("family,dtype", [("mxfp8", "fp8")]) +@pytest.mark.parametrize("path,block_k", [("wmma_scale", 32), ("wmma_scale16", 16)]) +def test_example_contract_and_lowering(family, dtype, path, block_k): + example = importlib.import_module(f"rocke.examples.gfx1250.gemm.{family}_gemm") + args = argparse.Namespace(m=32, n=48, k=256, matrix_path=path, dtype=dtype) + spec = example.make_spec(args) + assert (spec.dtype_a, spec.dtype_b, spec.scale_dtype) == ( + normalize_dtype(dtype), + normalize_dtype(dtype), + "e8m0", + ) + assert spec.block_k == block_k + llvm = lower_kernel_to_llvm( + build_block_scaled_gemm(spec), arch="gfx1250", llvm_flavor="llvm23" + ) + assert "@llvm.amdgcn.wmma.scale" in llvm From 8d51b91b7bd6b14808a8edfabd2b7aaba0902dde Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 18 Sep 2026 01:03:58 -0400 Subject: [PATCH 02/15] docs(rocke): describe common FP8 and BF8 matrix support --- .../python/rocke/instances/gfx1250/block_scaled_gemm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py index 8a4bcc96502a..c24ef96b1fb8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py @@ -272,12 +272,12 @@ def _as_f32(b: IRBuilder, v): def build_block_scaled_gemm( spec: BlockScaledGemmSpec, arch: str = "gfx1250" ) -> KernelDef: - """Build a gfx1250 FP8/BF8/FP4 block-scaled GEMM (RCR, ``C = A @ B^T``). + """Build a gfx1250 FP8/BF8 block-scaled GEMM (RCR, ``C = A @ B^T``). One wave (32 lanes) computes one 16x16 output tile without LDS. The legacy ``wmma`` path uses K=64 FP8/BF8 atoms, accumulates each ``block_k`` group, and applies FP16/FP32 A/B scales in software. The native ``wmma_scale`` and - ``wmma_scale16`` paths use K=128 FP8/FP4 atoms and pass packed E8M0 scale bytes + ``wmma_scale16`` paths use K=128 FP8/BF8 atoms and pass packed E8M0 scale bytes directly to the instruction, with K=32 and K=16 scale groups respectively. Lane ``l`` owns output column ``l % 16`` and rows From e386ef810ffa3a8ee50083d9138d739b43761dc1 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 18 Sep 2026 02:53:59 -0400 Subject: [PATCH 03/15] test(rocke): consolidate MXFP8 IR cases in shared golden --- .../tests/core/gfx1250_mxfp8_llvm23.json | 6 -- .../tests/core/test_gfx1250_mxfp8_example.py | 18 +---- .../rocke_representative_ir_sha256.json | 80 +++++++++++++++++-- .../instances/rocke_ir_parity_harness.py | 35 ++++++++ 4 files changed, 111 insertions(+), 28 deletions(-) delete mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_mxfp8_llvm23.json diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_mxfp8_llvm23.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_mxfp8_llvm23.json deleted file mode 100644 index ac6d786887b6..000000000000 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/gfx1250_mxfp8_llvm23.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "wmma_scale/bf8/bf8/e8m0/e8m0": "58930ce80326df951f1c13fc57efd5dd895251dedc7dcd72a29b15e1ae5860fa", - "wmma_scale/fp8/fp8/e8m0/e8m0": "7a37dd799b3babf3e7357edeaaa7f0d466a677f461087cef2b6cbd455bdada6f", - "wmma_scale16/bf8/bf8/e8m0/e8m0": "20ee6884c4f6ea6d6c3454822377eb88aecd4081f4b7d296920fbd3417e4c0ed", - "wmma_scale16/fp8/fp8/e8m0/e8m0": "18f36023f8cae46fc6f099cdae42d9b489f13ce9f43444238a3c419d5696ac62" -} diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mxfp8_example.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mxfp8_example.py index 2dee830650dc..763e2b1020ed 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mxfp8_example.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_mxfp8_example.py @@ -3,17 +3,12 @@ """Both homogeneous eight-bit formats are usable from the focused example.""" import argparse -from dataclasses import replace -import hashlib -import json -from pathlib import Path import numpy as np import pytest from rocke.core.lower_hip import lower_kernel_to_hip -from rocke.core.arch.target import normalize_dtype -from rocke.core.lower_llvm import lower_kernel_to_llvm +from rocke.core.dtypes import normalize_dtype from rocke.examples.gfx1250.gemm import _scaled_gemm_example, mxfp8_gemm from rocke.examples.gfx1250.gemm.block_scaled_gemm_verify import ( make_case_inputs, @@ -21,25 +16,18 @@ ) from rocke.instances.gfx1250.block_scaled_gemm import build_block_scaled_gemm -_GOLDENS = json.loads(Path(__file__).with_name("gfx1250_mxfp8_llvm23.json").read_text()) - @pytest.mark.parametrize( "dtype,selector,ml_name", [("fp8", 0, "float8_e4m3fn"), ("bf8", 1, "float8_e5m2")] ) @pytest.mark.parametrize("path", ["wmma_scale", "wmma_scale16"]) -def test_mxfp8_formats_and_existing_golden(dtype, selector, ml_name, path): +def test_mxfp8_formats_and_hip_selectors(dtype, selector, ml_name, path): ml = pytest.importorskip("ml_dtypes") spec = mxfp8_gemm.make_spec( argparse.Namespace(m=32, n=48, k=256, dtype=dtype, matrix_path=path) ) assert spec.dtype_a == spec.dtype_b == normalize_dtype(dtype) - kernel = build_block_scaled_gemm(replace(spec, name="fp6_test")) - llvm = lower_kernel_to_llvm(kernel, arch="gfx1250", llvm_flavor="llvm23") - assert ( - hashlib.sha256(llvm.encode()).hexdigest() - == _GOLDENS[f"{path}/{dtype}/{dtype}/e8m0/e8m0"] - ) + kernel = build_block_scaled_gemm(spec) hip = lower_kernel_to_hip(kernel, arch="gfx1250") assert f"__builtin_amdgcn_{path}_f32_16x16x128_f8f6f4({selector}," in hip inputs = make_case_inputs(spec, "mixed") diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json index 9655445ec0fa..d9936c915f2f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json @@ -1115,7 +1115,24 @@ "status": "ok" } }, - "expected_failures": {}, + "expected_failures": { + "gemm/gfx1250/mxfp8/wmma_scale/bf8e5m2": { + "message": "tile.wmma_scale_f32_16x16x128_bf8_bf8 requires llvm23 (ROCm 7.13+), got llvm20", + "type": "NotImplementedError" + }, + "gemm/gfx1250/mxfp8/wmma_scale/fp8e4m3": { + "message": "tile.wmma_scale_f32_16x16x128_fp8_fp8 requires llvm23 (ROCm 7.13+), got llvm20", + "type": "NotImplementedError" + }, + "gemm/gfx1250/mxfp8/wmma_scale16/bf8e5m2": { + "message": "tile.wmma_scale16_f32_16x16x128_bf8_bf8 requires llvm23 (ROCm 7.13+), got llvm20", + "type": "NotImplementedError" + }, + "gemm/gfx1250/mxfp8/wmma_scale16/fp8e4m3": { + "message": "tile.wmma_scale16_f32_16x16x128_fp8_fp8 requires llvm23 (ROCm 7.13+), got llvm20", + "type": "NotImplementedError" + } + }, "summary": { "arches": { "gfx11-generic": 1, @@ -1126,7 +1143,7 @@ "gfx942": 31, "gfx950": 81 }, - "expected_failures": 0, + "expected_failures": 4, "families": { "attention": 14, "attention_d128_swa": 1, @@ -2262,7 +2279,24 @@ "status": "ok" } }, - "expected_failures": {}, + "expected_failures": { + "gemm/gfx1250/mxfp8/wmma_scale/bf8e5m2": { + "message": "tile.wmma_scale_f32_16x16x128_bf8_bf8 requires llvm23 (ROCm 7.13+), got llvm22", + "type": "NotImplementedError" + }, + "gemm/gfx1250/mxfp8/wmma_scale/fp8e4m3": { + "message": "tile.wmma_scale_f32_16x16x128_fp8_fp8 requires llvm23 (ROCm 7.13+), got llvm22", + "type": "NotImplementedError" + }, + "gemm/gfx1250/mxfp8/wmma_scale16/bf8e5m2": { + "message": "tile.wmma_scale16_f32_16x16x128_bf8_bf8 requires llvm23 (ROCm 7.13+), got llvm22", + "type": "NotImplementedError" + }, + "gemm/gfx1250/mxfp8/wmma_scale16/fp8e4m3": { + "message": "tile.wmma_scale16_f32_16x16x128_fp8_fp8 requires llvm23 (ROCm 7.13+), got llvm22", + "type": "NotImplementedError" + } + }, "summary": { "arches": { "gfx11-generic": 1, @@ -2273,7 +2307,7 @@ "gfx942": 31, "gfx950": 81 }, - "expected_failures": 0, + "expected_failures": 4, "families": { "attention": 14, "attention_d128_swa": 1, @@ -3176,6 +3210,38 @@ "sha256": "4602cb57a8b57916659d91de9dd76756616541649600d7c8596d4073c39f48c4", "status": "ok" }, + "gemm/gfx1250/mxfp8/wmma_scale/bf8e5m2": { + "arch": "gfx1250", + "bytes": 87339, + "family": "gemm", + "kernel_name": "irhash_mxfp8_bf8e5m2_wmma_scale_block_scaled_bf8_bf8_M32N48K256_bk32_t16x16x128_wmma_scale", + "sha256": "11d69f432274ae05c41449645050b93034a0d411811130cc5b057abc5c8834bc", + "status": "ok" + }, + "gemm/gfx1250/mxfp8/wmma_scale/fp8e4m3": { + "arch": "gfx1250", + "bytes": 87339, + "family": "gemm", + "kernel_name": "irhash_mxfp8_fp8e4m3_wmma_scale_block_scaled_fp8_fp8_M32N48K256_bk32_t16x16x128_wmma_scale", + "sha256": "094da13dc1d3249da64c13185df2cfc44eb913fa940a38274d3bb7e1cc569948", + "status": "ok" + }, + "gemm/gfx1250/mxfp8/wmma_scale16/bf8e5m2": { + "arch": "gfx1250", + "bytes": 92233, + "family": "gemm", + "kernel_name": "irhash_mxfp8_bf8e5m2_wmma_scale16_block_scaled_bf8_bf8_M32N48K256_bk16_t16x16x128_wmma_scale16", + "sha256": "2ab30988dfa2bd120babd756d3d2d17bd444a1bbcabfe426ad99722bc8130209", + "status": "ok" + }, + "gemm/gfx1250/mxfp8/wmma_scale16/fp8e4m3": { + "arch": "gfx1250", + "bytes": 92233, + "family": "gemm", + "kernel_name": "irhash_mxfp8_fp8e4m3_wmma_scale16_block_scaled_fp8_fp8_M32N48K256_bk16_t16x16x128_wmma_scale16", + "sha256": "2e4e68d15ffa755e78bca8dd468eef8e345418cae2b5e5ace7b4761e2faf0f05", + "status": "ok" + }, "gemm/gfx1250/t32x32x32": { "arch": "gfx1250", "bytes": 26610, @@ -3415,7 +3481,7 @@ "gfx11-generic": 1, "gfx1151": 10, "gfx1201": 7, - "gfx1250": 4, + "gfx1250": 8, "gfx90a": 5, "gfx942": 31, "gfx950": 81 @@ -3433,12 +3499,12 @@ "conv_wgrad_reduce": 3, "conv_wgrad_two_stage": 2, "deep_fused_conv": 9, - "gemm": 13, + "gemm": 17, "grouped_gemm": 1, "kda_chunkwise": 13, "moe": 9 }, - "ok_cases": 139 + "ok_cases": 143 } } }, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/rocke_ir_parity_harness.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/rocke_ir_parity_harness.py index acd6e66de3ee..0cd922c8d688 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/rocke_ir_parity_harness.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/rocke_ir_parity_harness.py @@ -999,6 +999,31 @@ def _build(): return _build +def build_mxfp8_gemm_case(dtype, matrix_path): + def _build(): + from rocke.instances.gfx1250.block_scaled_gemm import ( + BlockScaledGemmSpec, + build_block_scaled_gemm, + ) + + return build_block_scaled_gemm( + BlockScaledGemmSpec( + name=f"irhash_mxfp8_{dtype}_{matrix_path}", + M=32, + N=48, + K=256, + dtype_a=dtype, + dtype_b=dtype, + dtype_c="bf16", + scale_dtype="e8m0", + matrix_path=matrix_path, + block_k=16 if matrix_path == "wmma_scale16" else 32, + ) + ) + + return _build + + def cases(): out = [] @@ -1274,6 +1299,16 @@ def add(family, case_id, arch, build): ), ) + # Homogeneous FP8/BF8 with E8M0 scales, shared by source and installed gates. + for dtype in ("fp8e4m3", "bf8e5m2"): + for matrix_path in ("wmma_scale", "wmma_scale16"): + add( + "gemm", + f"gemm/gfx1250/mxfp8/{matrix_path}/{dtype}", + "gfx1250", + build_mxfp8_gemm_case(dtype, matrix_path), + ) + # Conv: problem-shape and arch variants. conv1 = (1, 8, 8, 16, 32, 3, 3, 1, 1, 1, 1, 1, 1) conv2 = (2, 16, 16, 32, 32, 1, 1, 1, 1, 0, 0, 1, 1) From 11fc0973bd6e524a08d4761cccc694b1c07c9a04 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 18 Sep 2026 03:08:56 -0400 Subject: [PATCH 04/15] test(rocke): remove unrelated CMake formatting changes --- .../rocke/platform/tests/CMakeLists.txt | 48 +++++++------------ 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt b/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt index 28a7fdcac27b..da2fff71df97 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt @@ -2,10 +2,8 @@ # SPDX-License-Identifier: MIT # # By-layer C++ test registration for the rocKE engine. Included via -# add_subdirectory(tests) from the top-level rocke/platform/CMakeLists.txt. All -# targets -# link the engine archive rocke_core (built by the parent). Paths are relative -# to +# add_subdirectory(tests) from the top-level rocke/platform/CMakeLists.txt. All targets +# link the engine archive rocke_core (built by the parent). Paths are relative to # this file (tests/), keeping the tree copy-able. # --- core layer --------------------------------------------------------------- @@ -65,8 +63,7 @@ if(BUILD_TESTING) COMMAND rocke_ir_serialize_roundtrip) set_tests_properties(rocke_ir_serialize_roundtrip PROPERTIES LABELS "ckc;unit;host") - rocke_cxx_test_installed_name(_rocke_installed_name - rocke_ir_serialize_roundtrip) + rocke_cxx_test_installed_name(_rocke_installed_name rocke_ir_serialize_roundtrip) set_target_properties(rocke_ir_serialize_roundtrip PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") install(TARGETS rocke_ir_serialize_roundtrip @@ -77,15 +74,13 @@ if(BUILD_TESTING) # C++ parity for Python TestNewTargetIntrinsics. if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/core/future_intrinsic_lowering.cpp") - add_executable(rocke_future_intrinsic_lowering - core/future_intrinsic_lowering.cpp) + add_executable(rocke_future_intrinsic_lowering core/future_intrinsic_lowering.cpp) target_link_libraries(rocke_future_intrinsic_lowering PRIVATE rocke_core) add_test(NAME rocke_future_intrinsic_lowering COMMAND rocke_future_intrinsic_lowering) set_tests_properties(rocke_future_intrinsic_lowering PROPERTIES LABELS "ckc;unit;host") - rocke_cxx_test_installed_name(_rocke_installed_name - rocke_future_intrinsic_lowering) + rocke_cxx_test_installed_name(_rocke_installed_name rocke_future_intrinsic_lowering) set_target_properties(rocke_future_intrinsic_lowering PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") install(TARGETS rocke_future_intrinsic_lowering @@ -102,8 +97,7 @@ if(BUILD_TESTING) COMMAND rocke_flavor_datalayout_ssot) set_tests_properties(rocke_flavor_datalayout_ssot PROPERTIES LABELS "ckc;unit;host") - rocke_cxx_test_installed_name(_rocke_installed_name - rocke_flavor_datalayout_ssot) + rocke_cxx_test_installed_name(_rocke_installed_name rocke_flavor_datalayout_ssot) set_target_properties(rocke_flavor_datalayout_ssot PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") install(TARGETS rocke_flavor_datalayout_ssot @@ -140,10 +134,10 @@ if(BUILD_TESTING) # archs, byte-compared against the Python lowerer) lives in the Python driver # python/rocke/portable_ir/drivers/parity_matrix.py, which needs a Python # front end and so cannot be a ctest. - foreach(pir_test dom_decoders recipe_vm_replay) - if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/portable_ir/${pir_test}.cpp") - set(_pir_target "rocke_portable_ir_${pir_test}") - add_executable(${_pir_target} portable_ir/${pir_test}.cpp) + foreach(_pir_test dom_decoders recipe_vm_replay) + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/portable_ir/${_pir_test}.cpp") + set(_pir_target "rocke_portable_ir_${_pir_test}") + add_executable(${_pir_target} portable_ir/${_pir_test}.cpp) target_link_libraries(${_pir_target} PRIVATE rocke_core) if(UNIX) target_link_libraries(${_pir_target} PRIVATE m) @@ -155,8 +149,7 @@ if(BUILD_TESTING) PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") install(TARGETS ${_pir_target} RUNTIME DESTINATION ${ROCKE_CXX_TEST_INSTALL_DIR}) - set_property(GLOBAL APPEND PROPERTY ROCKE_INSTALL_TEST_TARGETS - ${_pir_target}) + set_property(GLOBAL APPEND PROPERTY ROCKE_INSTALL_TEST_TARGETS ${_pir_target}) endif() endforeach() @@ -172,12 +165,10 @@ if(BUILD_TESTING) endif() # --- instances layer -------------------------------------------------------- - if(EXISTS - "${CMAKE_CURRENT_SOURCE_DIR}/instances/tiled_attention_2d_reentrancy.cpp") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/instances/tiled_attention_2d_reentrancy.cpp") add_executable(rocke_tiled_attention_2d_reentrancy instances/tiled_attention_2d_reentrancy.cpp) - target_link_libraries(rocke_tiled_attention_2d_reentrancy PRIVATE - rocke_core) + target_link_libraries(rocke_tiled_attention_2d_reentrancy PRIVATE rocke_core) if(UNIX) target_link_libraries(rocke_tiled_attention_2d_reentrancy PRIVATE m) endif() @@ -185,8 +176,7 @@ if(BUILD_TESTING) COMMAND rocke_tiled_attention_2d_reentrancy) set_tests_properties(rocke_tiled_attention_2d_reentrancy PROPERTIES LABELS "ckc;unit;host") - rocke_cxx_test_installed_name(_rocke_installed_name - rocke_tiled_attention_2d_reentrancy) + rocke_cxx_test_installed_name(_rocke_installed_name rocke_tiled_attention_2d_reentrancy) set_target_properties(rocke_tiled_attention_2d_reentrancy PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") install(TARGETS rocke_tiled_attention_2d_reentrancy @@ -199,8 +189,7 @@ if(BUILD_TESTING) # whole-kernel LDS budget, the scaled-f8f6f4 arch gate and the fp8 wave-size # gate. The byte-identity gate only compares specs both engines accept, so an # acceptance divergence from the Python validators is invisible to it. - if(EXISTS - "${CMAKE_CURRENT_SOURCE_DIR}/instances/moe_fused_mega_spec_guards.cpp") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/instances/moe_fused_mega_spec_guards.cpp") add_executable(rocke_moe_fused_mega_spec_guards instances/moe_fused_mega_spec_guards.cpp) target_link_libraries(rocke_moe_fused_mega_spec_guards PRIVATE rocke_core) @@ -211,8 +200,7 @@ if(BUILD_TESTING) COMMAND rocke_moe_fused_mega_spec_guards) set_tests_properties(rocke_moe_fused_mega_spec_guards PROPERTIES LABELS "ckc;unit;host") - rocke_cxx_test_installed_name(_rocke_installed_name - rocke_moe_fused_mega_spec_guards) + rocke_cxx_test_installed_name(_rocke_installed_name rocke_moe_fused_mega_spec_guards) set_target_properties(rocke_moe_fused_mega_spec_guards PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") install(TARGETS rocke_moe_fused_mega_spec_guards @@ -230,9 +218,7 @@ if(BUILD_TESTING) # nothing for them. Guarded because rocke/platform also builds # standalone (outside hip-kernel-provider), where neither the shared # parser (apply_ctest_category_labels) nor - # - # ROCKE_ENGINE_CTEST_CATEGORIES_YAML/HIPKERNELPROVIDER_YAML_CATEGORIZATION_ENA - # BLED + # ROCKE_ENGINE_CTEST_CATEGORIES_YAML/HIPKERNELPROVIDER_YAML_CATEGORIZATION_ENABLED # exist. Without this, these tests carry only the hardcoded # "ckc;unit;host" label and silently drop out of `ctest -L ` # selection -- the mechanism TheRock CI's per-tier From 9695f05a4d8138ae1bf8d73b1b9557316a760440 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 18 Sep 2026 14:55:20 -0400 Subject: [PATCH 05/15] refactor(rocke): select scaled WMMA by semantic operand contracts --- .../rocke/platform/cpp/core/arch/data.cpp | 1050 +++++++---------- .../rocke/platform/cpp/core/arch/helpers.cpp | 9 +- .../rocke/platform/cpp/core/arch/query.cpp | 339 ++++-- .../rocke/platform/cpp/core/ir/ir_tile.cpp | 77 +- .../cpp/core/lower_hip/lower_hip_mma.cpp | 15 +- .../platform/cpp/core/lower_llvm/data.cpp | 4 +- .../platform/cpp/core/lower_llvm/mma.cpp | 104 +- .../rocke/platform/cpp/helpers/epilogues.cpp | 6 +- .../platform/cpp/helpers/mfma_attention.cpp | 68 +- .../platform/cpp/include/rocke/arch_target.h | 158 ++- .../include/rocke/helper_rocke.core.arch.h | 13 +- .../rocke/helper_rocke.helpers.epilogues.h | 2 +- .../include/rocke/instance_gemm_internal.h | 2 +- .../rocke/platform/cpp/include/rocke/ir.h | 14 +- .../cpp/include/rocke/wmma_scale_internal.h | 80 +- .../conv_implicit_gemm_conv_build_glue.cpp | 6 +- .../conv_implicit_gemm_conv_epilogue.cpp | 2 +- .../common/conv_implicit_gemm_dgrad.cpp | 10 +- .../common/conv_implicit_gemm_wgrad.cpp | 8 +- ...d_conv_pool_conv1_gemm_and_lds_staging.cpp | 12 +- .../instances/common/gemm_build_compute.cpp | 2 +- .../cpp/instances/common/gemm_mma.cpp | 8 +- ...ol_scatter_maxpool_finalization_phases.cpp | 8 +- ...gemm_fragment_loaders_register_handoff.cpp | 32 +- .../dsl_docs/architecture/authoring_model.md | 40 + .../python/rocke/core/arch/__init__.py | 6 + .../rocke/core/arch/data/arch_specs.json | 108 +- .../platform/python/rocke/core/arch/target.py | 590 ++++++--- .../python/rocke/core/arch/wmma_scale.py | 70 +- .../rocke/platform/python/rocke/core/ir.py | 91 +- .../platform/python/rocke/core/isa/backend.py | 19 +- .../platform/python/rocke/core/lower_hip.py | 27 +- .../platform/python/rocke/core/lower_llvm.py | 4 +- .../rocke/examples/gfx1250/gemm/README.md | 14 +- .../instances/gfx1250/block_scaled_gemm.py | 62 +- .../tests/core/future_intrinsic_lowering.cpp | 14 +- .../platform/tests/core/mma_frag_ssot.cpp | 216 +++- .../platform/tests/core/test_arch_mma_ssot.py | 244 +++- .../tests/core/test_gfx1250_scaled_wmma.py | 110 +- .../tests/core/test_lowbit_dtype_aliases.py | 3 +- .../tests/core/test_mma_catalog_lookup.cpp | 59 +- .../platform/tests/core/test_mma_contracts.py | 121 ++ .../tests/core/test_mma_frag_tables.py | 40 +- .../tests/core/test_wmma_gfx12_acc_layout.py | 81 +- .../rocke_representative_ir_sha256.json | 16 +- .../instances/parity/gfx1250_lowering_emit.c | 20 +- .../instances/parity/gfx1250_lowering_emit.py | 14 +- 47 files changed, 2496 insertions(+), 1502 deletions(-) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_contracts.py diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp index 7d478a2c7401..f93637c95353 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp @@ -49,12 +49,12 @@ return; \ } while(0) -/* MFMA 16x16 accumulator: slot i -> (m_blk*4 + i, lane % 16). */ -static void _mfma_acc_16x16(rocke_ir_builder_t* b, - rocke_value_t* lane, - int slot, - rocke_value_t** out0, - rocke_value_t** out1) +/* MFMA 16x16 src2/dst: slot i -> (m_blk * 4 + i, lane % 16). */ +static void _mfma_row_col_16x16(rocke_ir_builder_t* b, + rocke_value_t* lane, + int slot, + rocke_value_t** out0, + rocke_value_t** out1) { rocke_value_t *c16, *n_in_atom, *m_blk, *row; ROCKE_ATI_COORD_GUARD(b, out0, out1); @@ -74,13 +74,13 @@ static void _mfma_acc_16x16(rocke_ir_builder_t* b, *out1 = n_in_atom; } -/* MFMA 32x32 accumulator: +/* MFMA 32x32 src2/dst row-column map: * row = (slot//4)*8 + (lane//32)*4 + (slot%4), col = lane % 32. */ -static void _mfma_acc_32x32(rocke_ir_builder_t* b, - rocke_value_t* lane, - int slot, - rocke_value_t** out0, - rocke_value_t** out1) +static void _mfma_row_col_32x32(rocke_ir_builder_t* b, + rocke_value_t* lane, + int slot, + rocke_value_t** out0, + rocke_value_t** out1) { rocke_value_t *c32, *n_in_atom, *m_blk, *row; int rb, ri; @@ -384,12 +384,12 @@ static void _mfma_b_32x32x2_f32(rocke_ir_builder_t* b, *out1 = n_in_atom; } -/* WMMA 16x16x16 accumulator (wave32): (2*slot + lane//16, lane % 16). */ -static void _wmma_acc_16x16(rocke_ir_builder_t* b, - rocke_value_t* lane, - int slot, - rocke_value_t** out0, - rocke_value_t** out1) +/* WMMA 16x16x16 src2/dst row-column map (wave32). */ +static void _wmma_row_col_16x16(rocke_ir_builder_t* b, + rocke_value_t* lane, + int slot, + rocke_value_t** out0, + rocke_value_t** out1) { rocke_value_t *c16, *col, *half, *row; ROCKE_ATI_COORD_GUARD(b, out0, out1); @@ -505,12 +505,12 @@ static void _wmma_b_16x16_iu4(rocke_ir_builder_t* b, *out1 = col; } -/* RDNA4 WMMA 16x16x16 accumulator (wave32): ((lane//16)*8 + slot, lane % 16). */ -static void _wmma_gfx12_acc_16x16(rocke_ir_builder_t* b, - rocke_value_t* lane, - int slot, - rocke_value_t** out0, - rocke_value_t** out1) +/* RDNA4 WMMA 16x16x16 src2/dst row-column map (wave32). */ +static void _wmma_gfx12_row_col_16x16(rocke_ir_builder_t* b, + rocke_value_t* lane, + int slot, + rocke_value_t** out0, + rocke_value_t** out1) { rocke_value_t *c16, *col, *half, *row; ROCKE_ATI_COORD_GUARD(b, out0, out1); @@ -606,9 +606,10 @@ bool rocke_layout_map_coord(const rocke_layout_map_t* m, { if(b != NULL) { - const char* role_txt = (m->role == ROCKE_MMA_ROLE_ACC) ? "acc" - : (m->role == ROCKE_MMA_ROLE_A) ? "a" - : "b"; + const char* role_txt = (m->role == ROCKE_MMA_ROLE_SRC0) ? "src0" + : (m->role == ROCKE_MMA_ROLE_SRC1) ? "src1" + : (m->role == ROCKE_MMA_ROLE_SRC2) ? "src2" + : "dst"; b->status = ROCKE_ERR_VALUE; snprintf(b->err, ROCKE_ERR_MSG_CAP, @@ -631,75 +632,112 @@ bool rocke_layout_map_coord(const rocke_layout_map_t* m, * Per-op_id LayoutMap statics (the precomputed _build_mma_op output) * ========================================================================= * - * In Python _build_mma_op constructs, for each catalog row, up to three + * In Python _build_mma_op constructs, for each catalog row, up to four * LayoutMap instances from the op_id's _FragInfo (role + frag_len + wave_size + * fn). Because the maps depend ONLY on op_id (not on the per-arch row), we define - * one static LayoutMap triplet per op_id here and point every arch's catalog rows + * one static set of LayoutMaps per op_id here and point every arch's catalog rows * at them. A map is omitted (left NULL) when the _FragInfo has no fn for that * role or a zero frag_len (the `_mk(...) is None` case). * - * Naming: lm__{a,b,c}. Only op_ids with at least one verified map need a - * struct; the rest carry NULL layout pointers in the catalog directly. + * Private names, declaration order, stored roles, and catalog positions all use + * the canonical src0/src1/src2/dst machine operand order. Only op_ids with at + * least one verified map need descriptors; the rest carry NULL layout pointers + * in the catalog directly. */ -/* --- mfma_f32_16x16x16_f16 / _bf16: a/b/c all present (frag 4/4/4, wave64) --- */ -static const rocke_layout_map_t lm_mfma_16x16x16_a = {ROCKE_MMA_ROLE_A, 4, 64, _mfma_a_16x16}; -static const rocke_layout_map_t lm_mfma_16x16x16_b = {ROCKE_MMA_ROLE_B, 4, 64, _mfma_b_16x16}; -static const rocke_layout_map_t lm_mfma_16x16x16_c = {ROCKE_MMA_ROLE_ACC, 4, 64, _mfma_acc_16x16}; +/* --- mfma_f32_16x16x16_f16 / _bf16: src0/src1/src2/dst (frag 4/4/4/4, wave64) --- */ +static const rocke_layout_map_t lm_mfma_16x16x16_src0 = {ROCKE_MMA_ROLE_SRC0, 4, 64, _mfma_a_16x16}; +static const rocke_layout_map_t lm_mfma_16x16x16_src1 = {ROCKE_MMA_ROLE_SRC1, 4, 64, _mfma_b_16x16}; +static const rocke_layout_map_t lm_mfma_16x16x16_src2 + = {ROCKE_MMA_ROLE_SRC2, 4, 64, _mfma_row_col_16x16}; +static const rocke_layout_map_t lm_mfma_16x16x16_dst + = {ROCKE_MMA_ROLE_DST, 4, 64, _mfma_row_col_16x16}; -/* --- mfma_f32_16x16x32_{f16,bf16,fp8,bf8}: a/b/c present (frag 8/8/4, wave64). +/* --- mfma_f32_16x16x32_{f16,bf16,fp8,bf8}: src0/src1/src2/dst (frag 8/8/4/4, wave64). * fp8/bf8 share the f16 operand lane layout (same a_per_lane=8, K-packing). --- */ -static const rocke_layout_map_t lm_mfma_16x16x32_a = {ROCKE_MMA_ROLE_A, 8, 64, _mfma_a_16x16x32}; -static const rocke_layout_map_t lm_mfma_16x16x32_b = {ROCKE_MMA_ROLE_B, 8, 64, _mfma_b_16x16x32}; -static const rocke_layout_map_t lm_mfma_16x16x32_c = {ROCKE_MMA_ROLE_ACC, 4, 64, _mfma_acc_16x16}; - -/* --- mfma_f32_32x32x8_f16: a/b/c present (frag 4/4/16, wave64) --- */ -static const rocke_layout_map_t lm_mfma_32x32x8_a = {ROCKE_MMA_ROLE_A, 4, 64, _mfma_a_32x32x8}; -static const rocke_layout_map_t lm_mfma_32x32x8_b = {ROCKE_MMA_ROLE_B, 4, 64, _mfma_b_32x32x8}; -static const rocke_layout_map_t lm_mfma_32x32x8_c = {ROCKE_MMA_ROLE_ACC, 16, 64, _mfma_acc_32x32}; - -/* --- mfma_f32_32x32x16_{f16,bf16,fp8,bf8}: a/b/c present (frag 8/8/16, wave64). +static const rocke_layout_map_t lm_mfma_16x16x32_src0 + = {ROCKE_MMA_ROLE_SRC0, 8, 64, _mfma_a_16x16x32}; +static const rocke_layout_map_t lm_mfma_16x16x32_src1 + = {ROCKE_MMA_ROLE_SRC1, 8, 64, _mfma_b_16x16x32}; +static const rocke_layout_map_t lm_mfma_16x16x32_src2 + = {ROCKE_MMA_ROLE_SRC2, 4, 64, _mfma_row_col_16x16}; +static const rocke_layout_map_t lm_mfma_16x16x32_dst + = {ROCKE_MMA_ROLE_DST, 4, 64, _mfma_row_col_16x16}; + +/* --- mfma_f32_32x32x8_f16: src0/src1/src2/dst (frag 4/4/16/16, wave64) --- */ +static const rocke_layout_map_t lm_mfma_32x32x8_src0 + = {ROCKE_MMA_ROLE_SRC0, 4, 64, _mfma_a_32x32x8}; +static const rocke_layout_map_t lm_mfma_32x32x8_src1 + = {ROCKE_MMA_ROLE_SRC1, 4, 64, _mfma_b_32x32x8}; +static const rocke_layout_map_t lm_mfma_32x32x8_src2 + = {ROCKE_MMA_ROLE_SRC2, 16, 64, _mfma_row_col_32x32}; +static const rocke_layout_map_t lm_mfma_32x32x8_dst + = {ROCKE_MMA_ROLE_DST, 16, 64, _mfma_row_col_32x32}; + +/* --- mfma_f32_32x32x16_{f16,bf16,fp8,bf8}: src0/src1/src2/dst (frag 8/8/16/16, wave64). * fp8/bf8 share the f16 operand lane layout (same a_per_lane=8, K-packing). --- */ -static const rocke_layout_map_t lm_mfma_32x32x16_a = {ROCKE_MMA_ROLE_A, 8, 64, _mfma_a_32x32x16}; -static const rocke_layout_map_t lm_mfma_32x32x16_b = {ROCKE_MMA_ROLE_B, 8, 64, _mfma_b_32x32x16}; -static const rocke_layout_map_t lm_mfma_32x32x16_c = {ROCKE_MMA_ROLE_ACC, 16, 64, _mfma_acc_32x32}; - -/* --- mfma_f32_16x16x4_f32: a/b/c present (frag 1/1/4, wave64) (#8348) --- */ -static const rocke_layout_map_t lm_mfma_16x16x4_f32_a - = {ROCKE_MMA_ROLE_A, 1, 64, _mfma_a_16x16x4_f32}; -static const rocke_layout_map_t lm_mfma_16x16x4_f32_b - = {ROCKE_MMA_ROLE_B, 1, 64, _mfma_b_16x16x4_f32}; -static const rocke_layout_map_t lm_mfma_16x16x4_f32_c - = {ROCKE_MMA_ROLE_ACC, 4, 64, _mfma_acc_16x16}; - -/* --- mfma_f32_32x32x2_f32: a/b/c present (frag 1/1/16, wave64) (#8348) --- */ -static const rocke_layout_map_t lm_mfma_32x32x2_f32_a - = {ROCKE_MMA_ROLE_A, 1, 64, _mfma_a_32x32x2_f32}; -static const rocke_layout_map_t lm_mfma_32x32x2_f32_b - = {ROCKE_MMA_ROLE_B, 1, 64, _mfma_b_32x32x2_f32}; -static const rocke_layout_map_t lm_mfma_32x32x2_f32_c - = {ROCKE_MMA_ROLE_ACC, 16, 64, _mfma_acc_32x32}; - -/* --- wmma_f32_16x16x16_{f16,bf16}: a/b/c present (frag 16/16/8, wave32) --- */ -static const rocke_layout_map_t lm_wmma_16x16x16_a = {ROCKE_MMA_ROLE_A, 16, 32, _wmma_a_16x16}; -static const rocke_layout_map_t lm_wmma_16x16x16_b = {ROCKE_MMA_ROLE_B, 16, 32, _wmma_b_16x16}; -static const rocke_layout_map_t lm_wmma_16x16x16_c = {ROCKE_MMA_ROLE_ACC, 8, 32, _wmma_acc_16x16}; - -/* --- wmma_i32_16x16x16_iu8: a/b/c present (frag 4/4/8, wave32) --- */ -static const rocke_layout_map_t lm_wmma_iu8_a = {ROCKE_MMA_ROLE_A, 4, 32, _wmma_a_16x16_iu8}; -static const rocke_layout_map_t lm_wmma_iu8_b = {ROCKE_MMA_ROLE_B, 4, 32, _wmma_b_16x16_iu8}; -static const rocke_layout_map_t lm_wmma_iu8_c = {ROCKE_MMA_ROLE_ACC, 8, 32, _wmma_acc_16x16}; - -/* --- wmma_i32_16x16x16_iu4: a/b/c present (frag 2/2/8, wave32) --- */ -static const rocke_layout_map_t lm_wmma_iu4_a = {ROCKE_MMA_ROLE_A, 2, 32, _wmma_a_16x16_iu4}; -static const rocke_layout_map_t lm_wmma_iu4_b = {ROCKE_MMA_ROLE_B, 2, 32, _wmma_b_16x16_iu4}; -static const rocke_layout_map_t lm_wmma_iu4_c = {ROCKE_MMA_ROLE_ACC, 8, 32, _wmma_acc_16x16}; - -/* --- wmma_gfx12_f32_16x16x16_{f16,bf16}: a/b/c present (frag 8/8/8, wave32) --- */ -static const rocke_layout_map_t lm_wmma_gfx12_a = {ROCKE_MMA_ROLE_A, 8, 32, _wmma_gfx12_a_16x16}; -static const rocke_layout_map_t lm_wmma_gfx12_b = {ROCKE_MMA_ROLE_B, 8, 32, _wmma_gfx12_b_16x16}; -static const rocke_layout_map_t lm_wmma_gfx12_c - = {ROCKE_MMA_ROLE_ACC, 8, 32, _wmma_gfx12_acc_16x16}; +static const rocke_layout_map_t lm_mfma_32x32x16_src0 + = {ROCKE_MMA_ROLE_SRC0, 8, 64, _mfma_a_32x32x16}; +static const rocke_layout_map_t lm_mfma_32x32x16_src1 + = {ROCKE_MMA_ROLE_SRC1, 8, 64, _mfma_b_32x32x16}; +static const rocke_layout_map_t lm_mfma_32x32x16_src2 + = {ROCKE_MMA_ROLE_SRC2, 16, 64, _mfma_row_col_32x32}; +static const rocke_layout_map_t lm_mfma_32x32x16_dst + = {ROCKE_MMA_ROLE_DST, 16, 64, _mfma_row_col_32x32}; + +/* --- mfma_f32_16x16x4_f32: src0/src1/src2/dst (frag 1/1/4/4, wave64) (#8348) --- */ +static const rocke_layout_map_t lm_mfma_16x16x4_f32_src0 + = {ROCKE_MMA_ROLE_SRC0, 1, 64, _mfma_a_16x16x4_f32}; +static const rocke_layout_map_t lm_mfma_16x16x4_f32_src1 + = {ROCKE_MMA_ROLE_SRC1, 1, 64, _mfma_b_16x16x4_f32}; +static const rocke_layout_map_t lm_mfma_16x16x4_f32_src2 + = {ROCKE_MMA_ROLE_SRC2, 4, 64, _mfma_row_col_16x16}; +static const rocke_layout_map_t lm_mfma_16x16x4_f32_dst + = {ROCKE_MMA_ROLE_DST, 4, 64, _mfma_row_col_16x16}; + +/* --- mfma_f32_32x32x2_f32: src0/src1/src2/dst (frag 1/1/16/16, wave64) (#8348) --- */ +static const rocke_layout_map_t lm_mfma_32x32x2_f32_src0 + = {ROCKE_MMA_ROLE_SRC0, 1, 64, _mfma_a_32x32x2_f32}; +static const rocke_layout_map_t lm_mfma_32x32x2_f32_src1 + = {ROCKE_MMA_ROLE_SRC1, 1, 64, _mfma_b_32x32x2_f32}; +static const rocke_layout_map_t lm_mfma_32x32x2_f32_src2 + = {ROCKE_MMA_ROLE_SRC2, 16, 64, _mfma_row_col_32x32}; +static const rocke_layout_map_t lm_mfma_32x32x2_f32_dst + = {ROCKE_MMA_ROLE_DST, 16, 64, _mfma_row_col_32x32}; + +/* --- wmma_f32_16x16x16_{f16,bf16}: src0/src1/src2/dst (frag 16/16/8/8, wave32) --- */ +static const rocke_layout_map_t lm_wmma_16x16x16_src0 + = {ROCKE_MMA_ROLE_SRC0, 16, 32, _wmma_a_16x16}; +static const rocke_layout_map_t lm_wmma_16x16x16_src1 + = {ROCKE_MMA_ROLE_SRC1, 16, 32, _wmma_b_16x16}; +static const rocke_layout_map_t lm_wmma_16x16x16_src2 + = {ROCKE_MMA_ROLE_SRC2, 8, 32, _wmma_row_col_16x16}; +static const rocke_layout_map_t lm_wmma_16x16x16_dst + = {ROCKE_MMA_ROLE_DST, 8, 32, _wmma_row_col_16x16}; + +/* --- wmma_i32_16x16x16_iu8: src0/src1/src2/dst (frag 4/4/8/8, wave32) --- */ +static const rocke_layout_map_t lm_wmma_iu8_src0 = {ROCKE_MMA_ROLE_SRC0, 4, 32, _wmma_a_16x16_iu8}; +static const rocke_layout_map_t lm_wmma_iu8_src1 = {ROCKE_MMA_ROLE_SRC1, 4, 32, _wmma_b_16x16_iu8}; +static const rocke_layout_map_t lm_wmma_iu8_src2 + = {ROCKE_MMA_ROLE_SRC2, 8, 32, _wmma_row_col_16x16}; +static const rocke_layout_map_t lm_wmma_iu8_dst = {ROCKE_MMA_ROLE_DST, 8, 32, _wmma_row_col_16x16}; + +/* --- wmma_i32_16x16x16_iu4: src0/src1/src2/dst (frag 2/2/8/8, wave32) --- */ +static const rocke_layout_map_t lm_wmma_iu4_src0 = {ROCKE_MMA_ROLE_SRC0, 2, 32, _wmma_a_16x16_iu4}; +static const rocke_layout_map_t lm_wmma_iu4_src1 = {ROCKE_MMA_ROLE_SRC1, 2, 32, _wmma_b_16x16_iu4}; +static const rocke_layout_map_t lm_wmma_iu4_src2 + = {ROCKE_MMA_ROLE_SRC2, 8, 32, _wmma_row_col_16x16}; +static const rocke_layout_map_t lm_wmma_iu4_dst = {ROCKE_MMA_ROLE_DST, 8, 32, _wmma_row_col_16x16}; + +/* --- wmma_gfx12_f32_16x16x16_{f16,bf16}: src0/src1/src2/dst (frag 8/8/8/8, wave32) --- */ +static const rocke_layout_map_t lm_wmma_gfx12_src0 + = {ROCKE_MMA_ROLE_SRC0, 8, 32, _wmma_gfx12_a_16x16}; +static const rocke_layout_map_t lm_wmma_gfx12_src1 + = {ROCKE_MMA_ROLE_SRC1, 8, 32, _wmma_gfx12_b_16x16}; +static const rocke_layout_map_t lm_wmma_gfx12_src2 + = {ROCKE_MMA_ROLE_SRC2, 8, 32, _wmma_gfx12_row_col_16x16}; +static const rocke_layout_map_t lm_wmma_gfx12_dst + = {ROCKE_MMA_ROLE_DST, 8, 32, _wmma_gfx12_row_col_16x16}; /* gfx1250 WMMA 16x16x4 fp32 A operand (wave32): kABKLane=2, kAK1PerLane=2. * row = lane / 2 (M index 0..15); K = (lane % 2) * 2 + slot (slot in {0,1}). @@ -741,11 +779,11 @@ static void _wmma_gfx1250_b_16x16x4_f32(rocke_ir_builder_t* b, *out1 = col; } -/* --- wmma_gfx1250_f32_16x16x4_f32: a/b/c present (frag 2/2/8, wave32) --- */ -static const rocke_layout_map_t lm_wmma_gfx1250_f32_a - = {ROCKE_MMA_ROLE_A, 2, 32, _wmma_gfx1250_a_16x16x4_f32}; -static const rocke_layout_map_t lm_wmma_gfx1250_f32_b - = {ROCKE_MMA_ROLE_B, 2, 32, _wmma_gfx1250_b_16x16x4_f32}; +/* --- wmma_gfx1250_f32_16x16x4_f32: src0/src1 plus shared gfx12 src2/dst maps --- */ +static const rocke_layout_map_t lm_wmma_gfx1250_f32_src0 + = {ROCKE_MMA_ROLE_SRC0, 2, 32, _wmma_gfx1250_a_16x16x4_f32}; +static const rocke_layout_map_t lm_wmma_gfx1250_f32_src1 + = {ROCKE_MMA_ROLE_SRC1, 2, 32, _wmma_gfx1250_b_16x16x4_f32}; /* gfx1250 WMMA 16x16x32 A operand (wave32): row = lane % 16, * k = (lane / 16) * 16 + slot (slot in 0..15, a_frag_len=16). @@ -795,55 +833,55 @@ static void _wmma_gfx1250_b_16x16x32(rocke_ir_builder_t* b, *out1 = col; } -/* --- wmma_gfx1250_f32_16x16x32_{f16,bf16}: a/b/c present (frag 16/16/8, wave32) --- */ -static const rocke_layout_map_t lm_wmma_gfx1250_32_a - = {ROCKE_MMA_ROLE_A, 16, 32, _wmma_gfx1250_a_16x16x32}; -static const rocke_layout_map_t lm_wmma_gfx1250_32_b - = {ROCKE_MMA_ROLE_B, 16, 32, _wmma_gfx1250_b_16x16x32}; +/* --- wmma_gfx1250_f32_16x16x32_{f16,bf16}: src0/src1 plus shared gfx12 src2/dst maps --- */ +static const rocke_layout_map_t lm_wmma_gfx1250_32_src0 + = {ROCKE_MMA_ROLE_SRC0, 16, 32, _wmma_gfx1250_a_16x16x32}; +static const rocke_layout_map_t lm_wmma_gfx1250_32_src1 + = {ROCKE_MMA_ROLE_SRC1, 16, 32, _wmma_gfx1250_b_16x16x32}; /* ========================================================================= - * op_id -> accumulator fragment length (the c_frag_len projection of + * op_id -> dst fragment length (the dst.frag_len projection of * target.py::_MMA_FRAGMENT_INFO). * ========================================================================= * * This is the arch SSOT that rocke.core.ir.IRBuilder.mma (rocke_b_mma) consults * to size a tile.mma result vector from a bare op_id string; the ir bucket keeps - * no private copy. It mirrors _MMA_FRAGMENT_INFO's c_frag_len column exactly. + * no private copy. It mirrors _MMA_FRAGMENT_INFO's dst.frag_len exactly. * Only atoms this engine can emit are listed: this table mirrors the Python SSOT * including the gfx1250 WMMA atoms. * * `ati` = arch-target internal (see arch_target_internal.h): the naming prefix * reserved for storage private to this arch-target port, kept out of the public * rocke_ / rocke_arch_ namespace. This table is file-local static (only - * rocke_arch_mma_c_frag_len below reads it), so it is not extern'd in the + * rocke_arch_mma_dst_frag_len below reads it), so it is not extern'd in the * internal header like the cross-bucket rocke_ati_ symbols are. * * NOTE (drift): this is still a hand-maintained mirror of the Python - * _MMA_FRAGMENT_INFO c_frag_len column, not generated from it. + * _MMA_FRAGMENT_INFO dst.frag_len values, not generated from them. * TODO: codegen this table from the Python SSOT (_MMA_FRAGMENT_INFO) at build * time so the two engines share a single source of truth. Until then there is * no direct mirror-agreement test; each side's frag lengths are pinned * independently -- C via tests/core/mma_frag_ssot.cpp (rocke_b_mma) and Python * via tests/core/test_mma_frag_tables.py (_MMA_FRAGMENT_INFO) -- and the * byte-identity gate (tools/check_byte_identity.py) catches any cross-engine - * c_frag_len drift, since a wrong length changes the emitted tile.mma result + * dst-fragment drift, since a wrong length changes the emitted tile.mma result * vector width. */ typedef struct rocke_ati_mma_frag_row { const char* op_id; - int c_frag_len; + int dst_frag_len; } rocke_ati_mma_frag_row_t; -/* Layout guard for the rocke_arch_mma_c_frag_len lookup below, which derives its +/* Layout guard for the rocke_arch_mma_dst_frag_len lookup below, which derives its * row count from sizeof(table)/sizeof(row). That division is exact for any * element size, so padding of rocke_ati_mma_frag_row_t (12 bytes of data padded * out to 16 for op_id's 8-byte alignment) cannot desync the count. These asserts * pin the field order so a future reorder/resize is caught at compile time. */ static_assert(offsetof(rocke_ati_mma_frag_row_t, op_id) == 0, "op_id must be the first field of rocke_ati_mma_frag_row_t"); -static_assert(offsetof(rocke_ati_mma_frag_row_t, c_frag_len) == sizeof(const char*), - "c_frag_len must immediately follow op_id (no leading padding)"); +static_assert(offsetof(rocke_ati_mma_frag_row_t, dst_frag_len) == sizeof(const char*), + "dst_frag_len must immediately follow op_id (no leading padding)"); static const rocke_ati_mma_frag_row_t rocke_ati_mma_frag[] = { /* --- MFMA fp32 (wave64) --- */ @@ -887,13 +925,13 @@ static const rocke_ati_mma_frag_row_t rocke_ati_mma_frag[] = { {"wmma_gfx1250_f32_16x16x64_fp8_bf8", 8}, {"wmma_gfx1250_f32_16x16x64_bf8_fp8", 8}, {"wmma_gfx1250_f32_16x16x64_bf8_bf8", 8}, - {"wmma_scale_f32_16x16x128_fp8_fp8", 8}, - {"wmma_scale_f32_16x16x128_bf8_bf8", 8}, - {"wmma_scale16_f32_16x16x128_fp8_fp8", 8}, - {"wmma_scale16_f32_16x16x128_bf8_bf8", 8}, + {"wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32", 8}, + {"wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32", 8}, + {"wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32", 8}, + {"wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32", 8}, }; -int rocke_arch_mma_c_frag_len(const char* op_id) +int rocke_arch_mma_dst_frag_len(const char* op_id) { size_t i; if(!op_id) @@ -904,7 +942,7 @@ int rocke_arch_mma_c_frag_len(const char* op_id) { if(strcmp(rocke_ati_mma_frag[i].op_id, op_id) == 0) { - return rocke_ati_mma_frag[i].c_frag_len; + return rocke_ati_mma_frag[i].dst_frag_len; } } return 0; /* unknown atom: the zero-length _frag_info fallback */ @@ -916,13 +954,15 @@ int rocke_arch_mma_c_frag_len(const char* op_id) * ========================================================================= * * Frag lengths come from _MMA_FRAGMENT_INFO[op_id]; op_ids absent from that - * table carry (0,0,0,64) and NULL maps. dtype fields hold the normalised - * (canonical) catalog keys from rocke_normalize_dtype, exactly as - * normalize_dtype(o["a"/"b"/"c"]) would produce. + * table carry zero fragment lengths and NULL maps. dtype fields hold the normalised + * (canonical) catalog keys (interned strings from rocke_normalize_dtype), exactly as + * normalize_dtype() would produce for each indexed src and dst. * * Field order of rocke_mma_op_t: - * family, a_dtype, b_dtype, c_dtype, m, n, k, op_id, - * a_frag_len, b_frag_len, c_frag_len, wave_size, a_layout, b_layout, c_layout + * family, srcs[0], srcs[1], srcs[2], dst, m, n, k, op_id, wave_size. + * Each source stores dtype, frag_len, layout, optional scale value dtype, and + * scale block size along K (zero when unscaled); dst stores dtype, frag_len, + * and layout. Scale value dtype is independent of the packed register carrier. * * Low-bit spellings normalize to explicit exponent/mantissa catalog keys. */ @@ -933,805 +973,529 @@ int rocke_arch_mma_c_frag_len(const char* op_id) * bf16 arrived in CDNA2 with the same layout maps as fp16 for those shapes). */ static const rocke_mma_op_t k_mma_gfx90a[] = { {"mma", - "fp16", - "fp16", - "fp32", + {{"fp16", 4, &lm_mfma_16x16x16_src0}, + {"fp16", 4, &lm_mfma_16x16x16_src1}, + {"fp32", 4, &lm_mfma_16x16x16_src2}}, + {"fp32", 4, &lm_mfma_16x16x16_dst}, 16, 16, 16, "mfma_f32_16x16x16_f16", - 4, - 4, - 4, - 64, - &lm_mfma_16x16x16_a, - &lm_mfma_16x16x16_b, - &lm_mfma_16x16x16_c}, + 64}, {"mma", - "fp16", - "fp16", - "fp32", + {{"fp16", 4, &lm_mfma_32x32x8_src0}, + {"fp16", 4, &lm_mfma_32x32x8_src1}, + {"fp32", 16, &lm_mfma_32x32x8_src2}}, + {"fp32", 16, &lm_mfma_32x32x8_dst}, 32, 32, 8, "mfma_f32_32x32x8_f16", - 4, - 4, - 16, - 64, - &lm_mfma_32x32x8_a, - &lm_mfma_32x32x8_b, - &lm_mfma_32x32x8_c}, + 64}, {"mma", - "bf16", - "bf16", - "fp32", + {{"bf16", 4, &lm_mfma_16x16x16_src0}, + {"bf16", 4, &lm_mfma_16x16x16_src1}, + {"fp32", 4, &lm_mfma_16x16x16_src2}}, + {"fp32", 4, &lm_mfma_16x16x16_dst}, 16, 16, 16, "mfma_f32_16x16x16_bf16", - 4, - 4, - 4, - 64, - &lm_mfma_16x16x16_a, - &lm_mfma_16x16x16_b, - &lm_mfma_16x16x16_c}, + 64}, {"mma", - "bf16", - "bf16", - "fp32", + {{"bf16", 4, &lm_mfma_32x32x8_src0}, + {"bf16", 4, &lm_mfma_32x32x8_src1}, + {"fp32", 16, &lm_mfma_32x32x8_src2}}, + {"fp32", 16, &lm_mfma_32x32x8_dst}, 32, 32, 8, "mfma_f32_32x32x8_bf16", - 4, - 4, - 16, - 64, - &lm_mfma_32x32x8_a, - &lm_mfma_32x32x8_b, - &lm_mfma_32x32x8_c}, + 64}, }; /* ----------------------------- gfx942 (CDNA) ----------------------------- */ static const rocke_mma_op_t k_mma_gfx942[] = { {"mma", - "fp32", - "fp32", - "fp32", + {{"fp32", 1, &lm_mfma_16x16x4_f32_src0}, + {"fp32", 1, &lm_mfma_16x16x4_f32_src1}, + {"fp32", 4, &lm_mfma_16x16x4_f32_src2}}, + {"fp32", 4, &lm_mfma_16x16x4_f32_dst}, 16, 16, 4, "mfma_f32_16x16x4_f32", - 1, - 1, - 4, - 64, - &lm_mfma_16x16x4_f32_a, - &lm_mfma_16x16x4_f32_b, - &lm_mfma_16x16x4_f32_c}, + 64}, {"mma", - "fp32", - "fp32", - "fp32", + {{"fp32", 1, &lm_mfma_32x32x2_f32_src0}, + {"fp32", 1, &lm_mfma_32x32x2_f32_src1}, + {"fp32", 16, &lm_mfma_32x32x2_f32_src2}}, + {"fp32", 16, &lm_mfma_32x32x2_f32_dst}, 32, 32, 2, "mfma_f32_32x32x2_f32", - 1, - 1, - 16, - 64, - &lm_mfma_32x32x2_f32_a, - &lm_mfma_32x32x2_f32_b, - &lm_mfma_32x32x2_f32_c}, + 64}, {"mma", - "fp16", - "fp16", - "fp32", + {{"fp16", 4, &lm_mfma_16x16x16_src0}, + {"fp16", 4, &lm_mfma_16x16x16_src1}, + {"fp32", 4, &lm_mfma_16x16x16_src2}}, + {"fp32", 4, &lm_mfma_16x16x16_dst}, 16, 16, 16, "mfma_f32_16x16x16_f16", - 4, - 4, - 4, - 64, - &lm_mfma_16x16x16_a, - &lm_mfma_16x16x16_b, - &lm_mfma_16x16x16_c}, + 64}, {"mma", - "fp16", - "fp16", - "fp32", + {{"fp16", 4, &lm_mfma_32x32x8_src0}, + {"fp16", 4, &lm_mfma_32x32x8_src1}, + {"fp32", 16, &lm_mfma_32x32x8_src2}}, + {"fp32", 16, &lm_mfma_32x32x8_dst}, 32, 32, 8, "mfma_f32_32x32x8_f16", - 4, - 4, - 16, - 64, - &lm_mfma_32x32x8_a, - &lm_mfma_32x32x8_b, - &lm_mfma_32x32x8_c}, + 64}, {"mma", - "bf16", - "bf16", - "fp32", + {{"bf16", 4, &lm_mfma_16x16x16_src0}, + {"bf16", 4, &lm_mfma_16x16x16_src1}, + {"fp32", 4, &lm_mfma_16x16x16_src2}}, + {"fp32", 4, &lm_mfma_16x16x16_dst}, 16, 16, 16, "mfma_f32_16x16x16_bf16", - 4, - 4, - 4, - 64, - &lm_mfma_16x16x16_a, - &lm_mfma_16x16x16_b, - &lm_mfma_16x16x16_c}, + 64}, {"mma", - "bf16", - "bf16", - "fp32", + {{"bf16", 4, &lm_mfma_32x32x8_src0}, + {"bf16", 4, &lm_mfma_32x32x8_src1}, + {"fp32", 16, &lm_mfma_32x32x8_src2}}, + {"fp32", 16, &lm_mfma_32x32x8_dst}, 32, 32, 8, "mfma_f32_32x32x8_bf16", - 4, - 4, - 16, - 64, - &lm_mfma_32x32x8_a, - &lm_mfma_32x32x8_b, - &lm_mfma_32x32x8_c}, + 64}, {"mma", - "fp8e4m3", - "fp8e4m3", - "fp32", + {{"fp8e4m3", 8, &lm_mfma_16x16x32_src0}, + {"fp8e4m3", 8, &lm_mfma_16x16x32_src1}, + {"fp32", 4, &lm_mfma_16x16x32_src2}}, + {"fp32", 4, &lm_mfma_16x16x32_dst}, 16, 16, 32, "mfma_f32_16x16x32_fp8", - 8, - 8, - 4, - 64, - &lm_mfma_16x16x32_a, - &lm_mfma_16x16x32_b, - &lm_mfma_16x16x32_c}, + 64}, {"mma", - "fp8e4m3", - "fp8e4m3", - "fp32", + {{"fp8e4m3", 8, &lm_mfma_32x32x16_src0}, + {"fp8e4m3", 8, &lm_mfma_32x32x16_src1}, + {"fp32", 16, &lm_mfma_32x32x16_src2}}, + {"fp32", 16, &lm_mfma_32x32x16_dst}, 32, 32, 16, "mfma_f32_32x32x16_fp8", - 8, - 8, - 16, - 64, - &lm_mfma_32x32x16_a, - &lm_mfma_32x32x16_b, - &lm_mfma_32x32x16_c}, + 64}, {"mma", - "bf8e5m2", - "bf8e5m2", - "fp32", + {{"bf8e5m2", 8, &lm_mfma_16x16x32_src0}, + {"bf8e5m2", 8, &lm_mfma_16x16x32_src1}, + {"fp32", 4, &lm_mfma_16x16x32_src2}}, + {"fp32", 4, &lm_mfma_16x16x32_dst}, 16, 16, 32, "mfma_f32_16x16x32_bf8", - 8, - 8, - 4, - 64, - &lm_mfma_16x16x32_a, - &lm_mfma_16x16x32_b, - &lm_mfma_16x16x32_c}, + 64}, {"mma", - "bf8e5m2", - "bf8e5m2", - "fp32", + {{"bf8e5m2", 8, &lm_mfma_32x32x16_src0}, + {"bf8e5m2", 8, &lm_mfma_32x32x16_src1}, + {"fp32", 16, &lm_mfma_32x32x16_src2}}, + {"fp32", 16, &lm_mfma_32x32x16_dst}, 32, 32, 16, "mfma_f32_32x32x16_bf8", - 8, - 8, - 16, - 64, - &lm_mfma_32x32x16_a, - &lm_mfma_32x32x16_b, - &lm_mfma_32x32x16_c}, + 64}, }; /* ----------------------------- gfx950 (CDNA) ----------------------------- */ static const rocke_mma_op_t k_mma_gfx950[] = { {"mma", - "fp32", - "fp32", - "fp32", + {{"fp32", 1, &lm_mfma_16x16x4_f32_src0}, + {"fp32", 1, &lm_mfma_16x16x4_f32_src1}, + {"fp32", 4, &lm_mfma_16x16x4_f32_src2}}, + {"fp32", 4, &lm_mfma_16x16x4_f32_dst}, 16, 16, 4, "mfma_f32_16x16x4_f32", - 1, - 1, - 4, - 64, - &lm_mfma_16x16x4_f32_a, - &lm_mfma_16x16x4_f32_b, - &lm_mfma_16x16x4_f32_c}, + 64}, {"mma", - "fp32", - "fp32", - "fp32", + {{"fp32", 1, &lm_mfma_32x32x2_f32_src0}, + {"fp32", 1, &lm_mfma_32x32x2_f32_src1}, + {"fp32", 16, &lm_mfma_32x32x2_f32_src2}}, + {"fp32", 16, &lm_mfma_32x32x2_f32_dst}, 32, 32, 2, "mfma_f32_32x32x2_f32", - 1, - 1, - 16, - 64, - &lm_mfma_32x32x2_f32_a, - &lm_mfma_32x32x2_f32_b, - &lm_mfma_32x32x2_f32_c}, + 64}, {"mma", - "fp16", - "fp16", - "fp32", + {{"fp16", 4, &lm_mfma_16x16x16_src0}, + {"fp16", 4, &lm_mfma_16x16x16_src1}, + {"fp32", 4, &lm_mfma_16x16x16_src2}}, + {"fp32", 4, &lm_mfma_16x16x16_dst}, 16, 16, 16, "mfma_f32_16x16x16_f16", - 4, - 4, - 4, - 64, - &lm_mfma_16x16x16_a, - &lm_mfma_16x16x16_b, - &lm_mfma_16x16x16_c}, + 64}, {"mma", - "fp16", - "fp16", - "fp32", + {{"fp16", 8, &lm_mfma_16x16x32_src0}, + {"fp16", 8, &lm_mfma_16x16x32_src1}, + {"fp32", 4, &lm_mfma_16x16x32_src2}}, + {"fp32", 4, &lm_mfma_16x16x32_dst}, 16, 16, 32, "mfma_f32_16x16x32_f16", - 8, - 8, - 4, - 64, - &lm_mfma_16x16x32_a, - &lm_mfma_16x16x32_b, - &lm_mfma_16x16x32_c}, + 64}, {"mma", - "fp16", - "fp16", - "fp32", + {{"fp16", 4, &lm_mfma_32x32x8_src0}, + {"fp16", 4, &lm_mfma_32x32x8_src1}, + {"fp32", 16, &lm_mfma_32x32x8_src2}}, + {"fp32", 16, &lm_mfma_32x32x8_dst}, 32, 32, 8, "mfma_f32_32x32x8_f16", - 4, - 4, - 16, - 64, - &lm_mfma_32x32x8_a, - &lm_mfma_32x32x8_b, - &lm_mfma_32x32x8_c}, + 64}, {"mma", - "fp16", - "fp16", - "fp32", + {{"fp16", 8, &lm_mfma_32x32x16_src0}, + {"fp16", 8, &lm_mfma_32x32x16_src1}, + {"fp32", 16, &lm_mfma_32x32x16_src2}}, + {"fp32", 16, &lm_mfma_32x32x16_dst}, 32, 32, 16, "mfma_f32_32x32x16_f16", - 8, - 8, - 16, - 64, - &lm_mfma_32x32x16_a, - &lm_mfma_32x32x16_b, - &lm_mfma_32x32x16_c}, + 64}, {"mma", - "bf16", - "bf16", - "fp32", + {{"bf16", 4, &lm_mfma_16x16x16_src0}, + {"bf16", 4, &lm_mfma_16x16x16_src1}, + {"fp32", 4, &lm_mfma_16x16x16_src2}}, + {"fp32", 4, &lm_mfma_16x16x16_dst}, 16, 16, 16, "mfma_f32_16x16x16_bf16", - 4, - 4, - 4, - 64, - &lm_mfma_16x16x16_a, - &lm_mfma_16x16x16_b, - &lm_mfma_16x16x16_c}, + 64}, {"mma", - "bf16", - "bf16", - "fp32", + {{"bf16", 8, &lm_mfma_16x16x32_src0}, + {"bf16", 8, &lm_mfma_16x16x32_src1}, + {"fp32", 4, &lm_mfma_16x16x32_src2}}, + {"fp32", 4, &lm_mfma_16x16x32_dst}, 16, 16, 32, "mfma_f32_16x16x32_bf16", - 8, - 8, - 4, - 64, - &lm_mfma_16x16x32_a, - &lm_mfma_16x16x32_b, - &lm_mfma_16x16x32_c}, + 64}, {"mma", - "bf16", - "bf16", - "fp32", + {{"bf16", 4, &lm_mfma_32x32x8_src0}, + {"bf16", 4, &lm_mfma_32x32x8_src1}, + {"fp32", 16, &lm_mfma_32x32x8_src2}}, + {"fp32", 16, &lm_mfma_32x32x8_dst}, 32, 32, 8, "mfma_f32_32x32x8_bf16", - 4, - 4, - 16, - 64, - &lm_mfma_32x32x8_a, - &lm_mfma_32x32x8_b, - &lm_mfma_32x32x8_c}, + 64}, {"mma", - "bf16", - "bf16", - "fp32", + {{"bf16", 8, &lm_mfma_32x32x16_src0}, + {"bf16", 8, &lm_mfma_32x32x16_src1}, + {"fp32", 16, &lm_mfma_32x32x16_src2}}, + {"fp32", 16, &lm_mfma_32x32x16_dst}, 32, 32, 16, "mfma_f32_32x32x16_bf16", - 8, - 8, - 16, - 64, - &lm_mfma_32x32x16_a, - &lm_mfma_32x32x16_b, - &lm_mfma_32x32x16_c}, + 64}, {"mma", - "fp8e4m3", - "fp8e4m3", - "fp32", + {{"fp8e4m3", 8, &lm_mfma_16x16x32_src0}, + {"fp8e4m3", 8, &lm_mfma_16x16x32_src1}, + {"fp32", 4, &lm_mfma_16x16x32_src2}}, + {"fp32", 4, &lm_mfma_16x16x32_dst}, 16, 16, 32, "mfma_f32_16x16x32_fp8", - 8, - 8, - 4, - 64, - &lm_mfma_16x16x32_a, - &lm_mfma_16x16x32_b, - &lm_mfma_16x16x32_c}, + 64}, {"mma", - "fp8e4m3", - "fp8e4m3", - "fp32", + {{"fp8e4m3", 8, &lm_mfma_32x32x16_src0}, + {"fp8e4m3", 8, &lm_mfma_32x32x16_src1}, + {"fp32", 16, &lm_mfma_32x32x16_src2}}, + {"fp32", 16, &lm_mfma_32x32x16_dst}, 32, 32, 16, "mfma_f32_32x32x16_fp8", - 8, - 8, - 16, - 64, - &lm_mfma_32x32x16_a, - &lm_mfma_32x32x16_b, - &lm_mfma_32x32x16_c}, + 64}, {"mma", - "bf8e5m2", - "bf8e5m2", - "fp32", + {{"bf8e5m2", 8, &lm_mfma_16x16x32_src0}, + {"bf8e5m2", 8, &lm_mfma_16x16x32_src1}, + {"fp32", 4, &lm_mfma_16x16x32_src2}}, + {"fp32", 4, &lm_mfma_16x16x32_dst}, 16, 16, 32, "mfma_f32_16x16x32_bf8", - 8, - 8, - 4, - 64, - &lm_mfma_16x16x32_a, - &lm_mfma_16x16x32_b, - &lm_mfma_16x16x32_c}, + 64}, {"mma", - "bf8e5m2", - "bf8e5m2", - "fp32", + {{"bf8e5m2", 8, &lm_mfma_32x32x16_src0}, + {"bf8e5m2", 8, &lm_mfma_32x32x16_src1}, + {"fp32", 16, &lm_mfma_32x32x16_src2}}, + {"fp32", 16, &lm_mfma_32x32x16_dst}, 32, 32, 16, "mfma_f32_32x32x16_bf8", - 8, - 8, - 16, - 64, - &lm_mfma_32x32x16_a, - &lm_mfma_32x32x16_b, - &lm_mfma_32x32x16_c}, + 64}, /* Low-bit catalog keys use exponent/mantissa names; op_ids carry frag lengths * only (no verified maps), per _MMA_FRAGMENT_INFO. */ {"mma", - "fp4e2m1", - "fp4e2m1", - "fp32", + {{"fp4e2m1", 16, NULL}, {"fp4e2m1", 16, NULL}, {"fp32", 4, NULL}}, + {"fp32", 4, NULL}, 16, 16, 128, "mfma_f32_16x16x128_fp4", - 16, - 16, - 4, - 64, - NULL, - NULL, - NULL}, + 64}, {"mma", - "fp6e2m3", - "fp6e2m3", - "fp32", + {{"fp6e2m3", 12, NULL}, {"fp6e2m3", 12, NULL}, {"fp32", 4, NULL}}, + {"fp32", 4, NULL}, 16, 16, 96, "mfma_f32_16x16x96_fp6", - 12, - 12, - 4, - 64, - NULL, - NULL, - NULL}, + 64}, }; /* ----------------------------- gfx1151 (RDNA3.5) ------------------------- */ static const rocke_mma_op_t k_mma_gfx1151[] = { {"wmma", - "fp16", - "fp16", - "fp32", + {{"fp16", 16, &lm_wmma_16x16x16_src0}, + {"fp16", 16, &lm_wmma_16x16x16_src1}, + {"fp32", 8, &lm_wmma_16x16x16_src2}}, + {"fp32", 8, &lm_wmma_16x16x16_dst}, 16, 16, 16, "wmma_f32_16x16x16_f16", - 16, - 16, - 8, - 32, - &lm_wmma_16x16x16_a, - &lm_wmma_16x16x16_b, - &lm_wmma_16x16x16_c}, + 32}, {"wmma", - "bf16", - "bf16", - "fp32", + {{"bf16", 16, &lm_wmma_16x16x16_src0}, + {"bf16", 16, &lm_wmma_16x16x16_src1}, + {"fp32", 8, &lm_wmma_16x16x16_src2}}, + {"fp32", 8, &lm_wmma_16x16x16_dst}, 16, 16, 16, "wmma_f32_16x16x16_bf16", - 16, - 16, - 8, - 32, - &lm_wmma_16x16x16_a, - &lm_wmma_16x16x16_b, - &lm_wmma_16x16x16_c}, + 32}, {"wmma", - "iu8", - "iu8", - "i32", + {{"iu8", 4, &lm_wmma_iu8_src0}, {"iu8", 4, &lm_wmma_iu8_src1}, {"i32", 8, &lm_wmma_iu8_src2}}, + {"i32", 8, &lm_wmma_iu8_dst}, 16, 16, 16, "wmma_i32_16x16x16_iu8", - 4, - 4, - 8, - 32, - &lm_wmma_iu8_a, - &lm_wmma_iu8_b, - &lm_wmma_iu8_c}, + 32}, {"wmma", - "iu4", - "iu4", - "i32", + {{"iu4", 2, &lm_wmma_iu4_src0}, {"iu4", 2, &lm_wmma_iu4_src1}, {"i32", 8, &lm_wmma_iu4_src2}}, + {"i32", 8, &lm_wmma_iu4_dst}, 16, 16, 16, "wmma_i32_16x16x16_iu4", - 2, - 2, - 8, - 32, - &lm_wmma_iu4_a, - &lm_wmma_iu4_b, - &lm_wmma_iu4_c}, + 32}, }; /* ----------------------------- gfx1201 (RDNA4) -------------------------- */ static const rocke_mma_op_t k_mma_gfx1201[] = { {"wmma", - "fp16", - "fp16", - "fp32", + {{"fp16", 8, &lm_wmma_gfx12_src0}, + {"fp16", 8, &lm_wmma_gfx12_src1}, + {"fp32", 8, &lm_wmma_gfx12_src2}}, + {"fp32", 8, &lm_wmma_gfx12_dst}, 16, 16, 16, "wmma_gfx12_f32_16x16x16_f16", - 8, - 8, - 8, - 32, - &lm_wmma_gfx12_a, - &lm_wmma_gfx12_b, - &lm_wmma_gfx12_c}, + 32}, {"wmma", - "bf16", - "bf16", - "fp32", + {{"bf16", 8, &lm_wmma_gfx12_src0}, + {"bf16", 8, &lm_wmma_gfx12_src1}, + {"fp32", 8, &lm_wmma_gfx12_src2}}, + {"fp32", 8, &lm_wmma_gfx12_dst}, 16, 16, 16, "wmma_gfx12_f32_16x16x16_bf16", - 8, - 8, - 8, - 32, - &lm_wmma_gfx12_a, - &lm_wmma_gfx12_b, - &lm_wmma_gfx12_c}, + 32}, }; /* --------------------------- gfx11-generic (RDNA3) ---------------------- */ static const rocke_mma_op_t k_mma_gfx11_generic[] = { {"wmma", - "fp16", - "fp16", - "fp32", + {{"fp16", 16, &lm_wmma_16x16x16_src0}, + {"fp16", 16, &lm_wmma_16x16x16_src1}, + {"fp32", 8, &lm_wmma_16x16x16_src2}}, + {"fp32", 8, &lm_wmma_16x16x16_dst}, 16, 16, 16, "wmma_f32_16x16x16_f16", - 16, - 16, - 8, - 32, - &lm_wmma_16x16x16_a, - &lm_wmma_16x16x16_b, - &lm_wmma_16x16x16_c}, + 32}, {"wmma", - "bf16", - "bf16", - "fp32", + {{"bf16", 16, &lm_wmma_16x16x16_src0}, + {"bf16", 16, &lm_wmma_16x16x16_src1}, + {"fp32", 8, &lm_wmma_16x16x16_src2}}, + {"fp32", 8, &lm_wmma_16x16x16_dst}, 16, 16, 16, "wmma_f32_16x16x16_bf16", - 16, - 16, - 8, - 32, - &lm_wmma_16x16x16_a, - &lm_wmma_16x16x16_b, - &lm_wmma_16x16x16_c}, + 32}, {"wmma", - "iu8", - "iu8", - "i32", + {{"iu8", 4, &lm_wmma_iu8_src0}, {"iu8", 4, &lm_wmma_iu8_src1}, {"i32", 8, &lm_wmma_iu8_src2}}, + {"i32", 8, &lm_wmma_iu8_dst}, 16, 16, 16, "wmma_i32_16x16x16_iu8", - 4, - 4, - 8, - 32, - &lm_wmma_iu8_a, - &lm_wmma_iu8_b, - &lm_wmma_iu8_c}, + 32}, {"wmma", - "iu4", - "iu4", - "i32", + {{"iu4", 2, &lm_wmma_iu4_src0}, {"iu4", 2, &lm_wmma_iu4_src1}, {"i32", 8, &lm_wmma_iu4_src2}}, + {"i32", 8, &lm_wmma_iu4_dst}, 16, 16, 16, "wmma_i32_16x16x16_iu4", - 2, - 2, - 8, - 32, - &lm_wmma_iu4_a, - &lm_wmma_iu4_b, - &lm_wmma_iu4_c}, + 32}, }; /* ----------------------------- gfx1250 (CDNA/GFX12) ---------------------- */ -/* gfx1250: wave32, WMMA only (no MFMA). fp32 K=4 atom: A/B are scalar float - * per lane (same lane layout as mfma_f32_16x16x4_f32 on CDNA — reuses - * lm_mfma_16x16x4_f32_{a,b}); accumulator is <8 x float> in the gfx12 - * column-distributed layout (lm_wmma_gfx12_c, c_frag_len=8). - * Primary fp16/bf16 atom is 16x16x32 (K=32, not the gfx12 RDNA 16x16x16). - * A/B frag_len=16 (16 fp16/bf16 elements per lane); A/B layout maps are NULL - * pending the gfx1250 ISA backend port (rocke_ll_backend_for); the accumulator - * reuses lm_wmma_gfx12_c (same 16x16 tile layout). FP8/BF8 K=64 atoms carry - * frag_len=8 (8 i32 per lane) with NULL A/B maps. */ +/* gfx1250: wave32, WMMA only (no MFMA). The fp32 K=4 atom has two fp32 values + * per lane in src0 and src1; src2 and dst use separate + * descriptors with the same gfx12 column-distributed row/column function. + * The primary fp16/bf16 atom is 16x16x32 (K=32, not the gfx12 RDNA + * 16x16x16). Its src0 and src1 have 16 fp16/bf16 values per lane. FP8/BF8 + * K=64 atoms carry eight packed i32 values per lane in src0 and src1; those + * source layout maps remain NULL pending the gfx1250 ISA backend port + * (rocke_ll_backend_for). */ static const rocke_mma_op_t k_mma_gfx1250[] = { {"wmma", - "fp32", - "fp32", - "fp32", + {{"fp32", 2, &lm_wmma_gfx1250_f32_src0}, + {"fp32", 2, &lm_wmma_gfx1250_f32_src1}, + {"fp32", 8, &lm_wmma_gfx12_src2}}, + {"fp32", 8, &lm_wmma_gfx12_dst}, 16, 16, 4, "wmma_gfx1250_f32_16x16x4_f32", - 2, - 2, - 8, - 32, - &lm_wmma_gfx1250_f32_a, - &lm_wmma_gfx1250_f32_b, - &lm_wmma_gfx12_c}, + 32}, {"wmma", - "fp16", - "fp16", - "fp32", + {{"fp16", 16, &lm_wmma_gfx1250_32_src0}, + {"fp16", 16, &lm_wmma_gfx1250_32_src1}, + {"fp32", 8, &lm_wmma_gfx12_src2}}, + {"fp32", 8, &lm_wmma_gfx12_dst}, 16, 16, 32, "wmma_gfx1250_f32_16x16x32_f16", - 16, - 16, - 8, - 32, - &lm_wmma_gfx1250_32_a, - &lm_wmma_gfx1250_32_b, - &lm_wmma_gfx12_c}, + 32}, {"wmma", - "bf16", - "bf16", - "fp32", + {{"bf16", 16, &lm_wmma_gfx1250_32_src0}, + {"bf16", 16, &lm_wmma_gfx1250_32_src1}, + {"fp32", 8, &lm_wmma_gfx12_src2}}, + {"fp32", 8, &lm_wmma_gfx12_dst}, 16, 16, 32, "wmma_gfx1250_f32_16x16x32_bf16", - 16, - 16, - 8, - 32, - &lm_wmma_gfx1250_32_a, - &lm_wmma_gfx1250_32_b, - &lm_wmma_gfx12_c}, + 32}, {"wmma", - "fp8e4m3", - "fp8e4m3", - "fp32", + {{"fp8e4m3", 8, NULL}, {"fp8e4m3", 8, NULL}, {"fp32", 8, &lm_wmma_gfx12_src2}}, + {"fp32", 8, &lm_wmma_gfx12_dst}, 16, 16, 64, "wmma_gfx1250_f32_16x16x64_fp8_fp8", - 32, - 32, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c}, + 32}, {"wmma", - "fp8e4m3", - "bf8e5m2", - "fp32", + {{"fp8e4m3", 8, NULL}, {"bf8e5m2", 8, NULL}, {"fp32", 8, &lm_wmma_gfx12_src2}}, + {"fp32", 8, &lm_wmma_gfx12_dst}, 16, 16, 64, "wmma_gfx1250_f32_16x16x64_fp8_bf8", - 32, - 32, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c}, + 32}, {"wmma", - "bf8e5m2", - "fp8e4m3", - "fp32", + {{"bf8e5m2", 8, NULL}, {"fp8e4m3", 8, NULL}, {"fp32", 8, &lm_wmma_gfx12_src2}}, + {"fp32", 8, &lm_wmma_gfx12_dst}, 16, 16, 64, "wmma_gfx1250_f32_16x16x64_bf8_fp8", - 32, - 32, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c}, + 32}, {"wmma", - "bf8e5m2", - "bf8e5m2", - "fp32", + {{"bf8e5m2", 8, NULL}, {"bf8e5m2", 8, NULL}, {"fp32", 8, &lm_wmma_gfx12_src2}}, + {"fp32", 8, &lm_wmma_gfx12_dst}, 16, 16, 64, "wmma_gfx1250_f32_16x16x64_bf8_bf8", - 32, - 32, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c}, - {"wmma_scale", - "fp8e4m3", - "fp8e4m3", - "fp32", + 32}, + {"wmma_scaled", + {{"fp8e4m3", 16, NULL, "e8m0", 32}, + {"fp8e4m3", 16, NULL, "e8m0", 32}, + {"fp32", 8, &lm_wmma_gfx12_src2}}, + {"fp32", 8, &lm_wmma_gfx12_dst}, 16, 16, 128, - "wmma_scale_f32_16x16x128_fp8_fp8", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c}, - {"wmma_scale", - "bf8e5m2", - "bf8e5m2", - "fp32", + "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32", + 32}, + {"wmma_scaled", + {{"bf8e5m2", 16, NULL, "e8m0", 32}, + {"bf8e5m2", 16, NULL, "e8m0", 32}, + {"fp32", 8, &lm_wmma_gfx12_src2}}, + {"fp32", 8, &lm_wmma_gfx12_dst}, 16, 16, 128, - "wmma_scale_f32_16x16x128_bf8_bf8", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c}, - {"wmma_scale16", - "fp8e4m3", - "fp8e4m3", - "fp32", + "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32", + 32}, + {"wmma_scaled", + {{"fp8e4m3", 16, NULL, "e8m0", 16}, + {"fp8e4m3", 16, NULL, "e8m0", 16}, + {"fp32", 8, &lm_wmma_gfx12_src2}}, + {"fp32", 8, &lm_wmma_gfx12_dst}, 16, 16, 128, - "wmma_scale16_f32_16x16x128_fp8_fp8", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c}, - {"wmma_scale16", - "bf8e5m2", - "bf8e5m2", - "fp32", + "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32", + 32}, + {"wmma_scaled", + {{"bf8e5m2", 16, NULL, "e8m0", 16}, + {"bf8e5m2", 16, NULL, "e8m0", 16}, + {"fp32", 8, &lm_wmma_gfx12_src2}}, + {"fp32", 8, &lm_wmma_gfx12_dst}, 16, 16, 128, - "wmma_scale16_f32_16x16x128_bf8_bf8", - 16, - 16, - 8, - 32, - NULL, - NULL, - &lm_wmma_gfx12_c}, + "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32", + 32}, }; /* ========================================================================= diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/helpers.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/helpers.cpp index 5f74ed7ddc7d..9eadf48cc081 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/helpers.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/helpers.cpp @@ -40,17 +40,10 @@ const rocke_arch_layout_map_t* rocke_mmaop_b_layout(const rocke_mmaop_t* op, roc const rocke_arch_layout_map_t* rocke_mmaop_c_layout(const rocke_mmaop_t* op, rocke_ir_builder_t* b) { - /* MmaOp.c_layout(): accumulator (row, col) lane/slot map. */ + /* Historical MmaOp.c_layout(): result (row, col) lane/slot map. */ return rocke_mma_op_c_layout(op, b); } -const rocke_arch_layout_map_t* rocke_mmaop_acc_layout(const rocke_mmaop_t* op, - rocke_ir_builder_t* b) -{ - /* MmaOp.acc_layout(): alias for the accumulator (C) map. */ - return rocke_mma_op_acc_layout(op, b); -} - bool rocke_arch_layout_map_coord(const rocke_arch_layout_map_t* m, rocke_ir_builder_t* b, rocke_value_t* lane, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp index 9db10433f7ac..5ef09ff437d8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include "rocke/arch_target.h" #include "rocke/arch_target_internal.h" @@ -50,6 +51,34 @@ static const char* rocke_ati_family_or_default(const char* family) /* ============================== MMA atom getters ====================== */ +const char* rocke_mma_op_semantic_id(const rocke_mma_op_t* op, char* out, size_t cap) +{ + if(!op || !out || !cap) + return NULL; + std::string id = op->family; + for(char& c : id) + if(c == '_') + c = '.'; + id += "." + std::to_string(op->m) + "x" + std::to_string(op->n) + "x" + std::to_string(op->k); + char dtype_buf[64]; + for(int i = 0; i < 3; ++i) + { + const rocke_mma_src_t& src = op->srcs[i]; + id += ".src" + std::to_string(i) + "_" + + rocke_normalize_dtype(src.dtype, dtype_buf, sizeof(dtype_buf)); + if(src.scale_dtype) + { + const char* scale = strcmp(src.scale_dtype, "fp8e4m3") == 0 ? "e4m3" : src.scale_dtype; + id += std::string("_") + scale + "_b" + std::to_string(src.scale_block_size); + } + } + id += std::string(".dst_") + rocke_normalize_dtype(op->dst.dtype, dtype_buf, sizeof(dtype_buf)); + if(id.size() >= cap) + return NULL; + memcpy(out, id.c_str(), id.size() + 1); + return out; +} + void rocke_mma_op_shape(const rocke_mma_op_t* op, int* m, int* n, int* k) { if(!op) @@ -96,23 +125,34 @@ static const rocke_layout_map_t* rocke_ati_require_layout(const rocke_mma_op_t* const rocke_layout_map_t* rocke_mma_op_a_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b) { - return rocke_ati_require_layout(op, op ? op->a_layout : NULL, "a", b); + return rocke_mma_op_src_layout(op, 0, b); } const rocke_layout_map_t* rocke_mma_op_b_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b) { - return rocke_ati_require_layout(op, op ? op->b_layout : NULL, "b", b); + return rocke_mma_op_src_layout(op, 1, b); } const rocke_layout_map_t* rocke_mma_op_c_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b) { - return rocke_ati_require_layout(op, op ? op->c_layout : NULL, "c", b); + return rocke_mma_op_dst_layout(op, b); +} + +const rocke_layout_map_t* + rocke_mma_op_src_layout(const rocke_mma_op_t* op, int index, rocke_ir_builder_t* b) +{ + if(index < 0 || index >= 3) + { + rocke_ati_q_set_err(b, ROCKE_ERR_VALUE, "MMA src index %d is outside [0, 3)", index); + } + char role[8]; + (void)snprintf(role, sizeof(role), "src%d", index); + return rocke_ati_require_layout(op, op ? op->srcs[index].layout : NULL, role, b); } -const rocke_layout_map_t* rocke_mma_op_acc_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b) +const rocke_layout_map_t* rocke_mma_op_dst_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b) { - /* Python alias: acc_layout() == c_layout(). */ - return rocke_mma_op_c_layout(op, b); + return rocke_ati_require_layout(op, op ? op->dst.layout : NULL, "dst", b); } /* ============================== MMA catalog =========================== */ @@ -133,21 +173,60 @@ const rocke_mma_op_t* rocke_mma_catalog_ops(const rocke_mma_catalog_t* cat, int* /* Shared predicate matching MmaCatalog.enumerate's per-op filter. dtypes here * are already-normalised canonical keys (a/b/c). family is non-NULL. m/n < 0 * mean "any" (Python None). */ +static void rocke_ati_validate_scales(const rocke_mma_scale_operand_t* scales) +{ + if(!scales) + return; + for(int i = 0; i < 3; ++i) + { + const char* dtype = scales[i].dtype; + if(dtype && strcmp(dtype, "fp8e4m3") == 0) + dtype = "e4m3"; + if(dtype && strcmp(dtype, "e8m0") != 0 && strcmp(dtype, "e4m3") != 0 + && strcmp(dtype, "e5m3") != 0) + ckc::raise_status(ROCKE_ERR_VALUE, "MMA scale dtype must be e8m0, e4m3, or e5m3"); + if((dtype && scales[i].block_size != 16 && scales[i].block_size != 32) + || (!dtype && scales[i].block_size != 0)) + ckc::raise_status(ROCKE_ERR_VALUE, "invalid MMA scale block size"); + } +} + +static bool rocke_ati_scales_match(const rocke_mma_op_t* op, + const rocke_mma_scale_operand_t* scales) +{ + if(!scales) + return true; + for(int i = 0; i < 3; ++i) + { + const char* dtype = scales[i].dtype; + if(dtype && strcmp(dtype, "fp8e4m3") == 0) + dtype = "e4m3"; + const char* actual = op->srcs[i].scale_dtype; + if(bool(dtype) != bool(actual) || (dtype && strcmp(dtype, actual) != 0) + || scales[i].block_size != op->srcs[i].scale_block_size) + return false; + } + return true; +} + static bool rocke_ati_op_matches(const rocke_mma_op_t* op, const char* family, const char* a, const char* b, const char* c, + const char* d, int m, int n) { if(strcmp(op->family, family) != 0) return false; - if(strcmp(op->a_dtype, a) != 0) + if(strcmp(op->srcs[0].dtype, a) != 0) + return false; + if(strcmp(op->srcs[1].dtype, b) != 0) return false; - if(strcmp(op->b_dtype, b) != 0) + if(strcmp(op->srcs[2].dtype, c) != 0) return false; - if(strcmp(op->c_dtype, c) != 0) + if(strcmp(op->dst.dtype, d) != 0) return false; if(m >= 0 && op->m != m) return false; @@ -156,31 +235,34 @@ static bool rocke_ati_op_matches(const rocke_mma_op_t* op, return true; } -int rocke_mma_catalog_enumerate(const rocke_mma_catalog_t* cat, - const char* family, - const char* a_dtype, - const char* b_dtype, - const char* c_dtype, - int m, - int n, - const rocke_mma_op_t** out, - int cap) +int rocke_mma_catalog_enumerate_indexed(const rocke_mma_catalog_t* cat, + const char* family, + const char* const src_dtypes[3], + const char* dst_dtype, + const rocke_mma_scale_operand_t src_scales[3], + int m, + int n, + const rocke_mma_op_t** out, + int cap) { - char abuf[64], bbuf[64], cbuf[64]; - const char *a, *bd, *c, *fam; + char abuf[64], bbuf[64], cbuf[64], dbuf[64]; + const char *a, *bd, *c, *d, *fam; int i, total = 0; - if(!cat) + if(!cat || !src_dtypes) return 0; + rocke_ati_validate_scales(src_scales); fam = rocke_ati_family_or_default(family); - a = rocke_normalize_dtype(a_dtype, abuf, sizeof abuf); - bd = rocke_normalize_dtype(b_dtype, bbuf, sizeof bbuf); - c = rocke_normalize_dtype(c_dtype, cbuf, sizeof cbuf); + a = rocke_normalize_dtype(src_dtypes[0], abuf, sizeof abuf); + bd = rocke_normalize_dtype(src_dtypes[1], bbuf, sizeof bbuf); + c = rocke_normalize_dtype(src_dtypes[2], cbuf, sizeof cbuf); + d = rocke_normalize_dtype(dst_dtype, dbuf, sizeof dbuf); for(i = 0; i < cat->num_ops; ++i) { const rocke_mma_op_t* op = &cat->ops[i]; - if(!rocke_ati_op_matches(op, fam, a, bd, c, m, n)) + if(!rocke_ati_op_matches(op, fam, a, bd, c, d, m, n) + || !rocke_ati_scales_match(op, src_scales)) continue; if(out && total < cap) out[total] = op; @@ -189,31 +271,49 @@ int rocke_mma_catalog_enumerate(const rocke_mma_catalog_t* cat, return total; } -bool rocke_mma_catalog_has_shape(const rocke_mma_catalog_t* cat, - const char* family, - const char* a_dtype, - const char* b_dtype, - const char* c_dtype, - int m, - int n, - int k) +int rocke_mma_catalog_enumerate(const rocke_mma_catalog_t* cat, + const char* family, + const char* a_dtype, + const char* b_dtype, + const char* c_dtype, + int m, + int n, + const rocke_mma_op_t** out, + int cap) { - char abuf[64], bbuf[64], cbuf[64]; - const char *a, *bd, *c, *fam; + const char* src_dtypes[3] = {a_dtype, b_dtype, c_dtype}; + return rocke_mma_catalog_enumerate_indexed( + cat, family, src_dtypes, c_dtype, NULL, m, n, out, cap); +} + +bool rocke_mma_catalog_has_shape_indexed(const rocke_mma_catalog_t* cat, + const char* family, + const char* const src_dtypes[3], + const char* dst_dtype, + const rocke_mma_scale_operand_t src_scales[3], + int m, + int n, + int k) +{ + char abuf[64], bbuf[64], cbuf[64], dbuf[64]; + const char *a, *bd, *c, *d, *fam; int i; - if(!cat) + if(!cat || !src_dtypes) return false; + rocke_ati_validate_scales(src_scales); fam = rocke_ati_family_or_default(family); - a = rocke_normalize_dtype(a_dtype, abuf, sizeof abuf); - bd = rocke_normalize_dtype(b_dtype, bbuf, sizeof bbuf); - c = rocke_normalize_dtype(c_dtype, cbuf, sizeof cbuf); + a = rocke_normalize_dtype(src_dtypes[0], abuf, sizeof abuf); + bd = rocke_normalize_dtype(src_dtypes[1], bbuf, sizeof bbuf); + c = rocke_normalize_dtype(src_dtypes[2], cbuf, sizeof cbuf); + d = rocke_normalize_dtype(dst_dtype, dbuf, sizeof dbuf); /* Python enumerates with m=m, n=n then checks op.shape == (m, n, k). */ for(i = 0; i < cat->num_ops; ++i) { const rocke_mma_op_t* op = &cat->ops[i]; - if(!rocke_ati_op_matches(op, fam, a, bd, c, m, n)) + if(!rocke_ati_op_matches(op, fam, a, bd, c, d, m, n) + || !rocke_ati_scales_match(op, src_scales)) continue; if(op->m == m && op->n == n && op->k == k) return true; @@ -221,42 +321,80 @@ bool rocke_mma_catalog_has_shape(const rocke_mma_catalog_t* cat, return false; } -const rocke_mma_op_t* rocke_mma_catalog_select_largest_k(const rocke_mma_catalog_t* cat, - const char* family, - const char* a_dtype, - const char* b_dtype, - const char* c_dtype, - int m, - int n, - int k_max) +bool rocke_mma_catalog_has_shape(const rocke_mma_catalog_t* cat, + const char* family, + const char* a_dtype, + const char* b_dtype, + const char* c_dtype, + int m, + int n, + int k) +{ + const char* src_dtypes[3] = {a_dtype, b_dtype, c_dtype}; + return rocke_mma_catalog_has_shape_indexed(cat, family, src_dtypes, c_dtype, NULL, m, n, k); +} + +const rocke_mma_op_t* + rocke_mma_catalog_select_largest_k_indexed(const rocke_mma_catalog_t* cat, + const char* family, + const char* const src_dtypes[3], + const char* dst_dtype, + const rocke_mma_scale_operand_t src_scales[3], + int m, + int n, + int k_max) { - char abuf[64], bbuf[64], cbuf[64]; - const char *a, *bd, *c, *fam; + char abuf[64], bbuf[64], cbuf[64], dbuf[64]; + const char *a, *bd, *c, *d, *fam; const rocke_mma_op_t* best = NULL; + bool ambiguous = false; int i; - if(!cat) + if(!cat || !src_dtypes) return NULL; + rocke_ati_validate_scales(src_scales); fam = rocke_ati_family_or_default(family); - a = rocke_normalize_dtype(a_dtype, abuf, sizeof abuf); - bd = rocke_normalize_dtype(b_dtype, bbuf, sizeof bbuf); - c = rocke_normalize_dtype(c_dtype, cbuf, sizeof cbuf); + a = rocke_normalize_dtype(src_dtypes[0], abuf, sizeof abuf); + bd = rocke_normalize_dtype(src_dtypes[1], bbuf, sizeof bbuf); + c = rocke_normalize_dtype(src_dtypes[2], cbuf, sizeof cbuf); + d = rocke_normalize_dtype(dst_dtype, dbuf, sizeof dbuf); for(i = 0; i < cat->num_ops; ++i) { const rocke_mma_op_t* op = &cat->ops[i]; - if(!rocke_ati_op_matches(op, fam, a, bd, c, m, n)) + if(!rocke_ati_op_matches(op, fam, a, bd, c, d, m, n) + || !rocke_ati_scales_match(op, src_scales)) continue; if(k_max >= 0 && op->k > k_max) continue; /* Python: k_max is None || op.k <= k_max */ - /* max(cands, key=op.k): first op wins ties (Python max keeps the first - * maximal element when iterating in catalog order). */ if(best == NULL || op->k > best->k) + { best = op; + ambiguous = false; + } + else if(op->k == best->k) + ambiguous = true; } + if(ambiguous) + ckc::raise_status(ROCKE_ERR_VALUE, + "ambiguous MMA query; specify the full operand contract"); return best; } +const rocke_mma_op_t* rocke_mma_catalog_select_largest_k(const rocke_mma_catalog_t* cat, + const char* family, + const char* a_dtype, + const char* b_dtype, + const char* c_dtype, + int m, + int n, + int k_max) +{ + const char* src_dtypes[3] = {a_dtype, b_dtype, c_dtype}; + return rocke_mma_catalog_select_largest_k_indexed( + cat, family, src_dtypes, c_dtype, NULL, m, n, k_max); +} + const rocke_mma_op_t* rocke_mma_catalog_by_op_id(const rocke_mma_catalog_t* cat, const char* op_id) { int i; @@ -271,51 +409,74 @@ const rocke_mma_op_t* rocke_mma_catalog_by_op_id(const rocke_mma_catalog_t* cat, return NULL; } -const rocke_mma_op_t* rocke_mma_catalog_op_for_shape(const rocke_mma_catalog_t* cat, - const char* family, - const char* a_dtype, - const char* b_dtype, - const char* c_dtype, - int m, - int n, - int k) +const rocke_mma_op_t* + rocke_mma_catalog_op_for_shape_indexed(const rocke_mma_catalog_t* cat, + const char* family, + const char* const src_dtypes[3], + const char* dst_dtype, + const rocke_mma_scale_operand_t src_scales[3], + int m, + int n, + int k) { - char abuf[64], bbuf[64], cbuf[64]; - const char *a, *bd, *c, *fam; + char abuf[64], bbuf[64], cbuf[64], dbuf[64]; + const char *a, *bd, *c, *d, *fam; int i; + const rocke_mma_op_t* match = NULL; - if(!cat) + if(!cat || !src_dtypes) return NULL; + rocke_ati_validate_scales(src_scales); fam = rocke_ati_family_or_default(family); - a = rocke_normalize_dtype(a_dtype, abuf, sizeof abuf); - bd = rocke_normalize_dtype(b_dtype, bbuf, sizeof bbuf); - c = rocke_normalize_dtype(c_dtype, cbuf, sizeof cbuf); + a = rocke_normalize_dtype(src_dtypes[0], abuf, sizeof abuf); + bd = rocke_normalize_dtype(src_dtypes[1], bbuf, sizeof bbuf); + c = rocke_normalize_dtype(src_dtypes[2], cbuf, sizeof cbuf); + d = rocke_normalize_dtype(dst_dtype, dbuf, sizeof dbuf); - /* Python enumerates with m=m, n=n, then returns the first op whose k == k. */ + /* Exact selection rejects an underspecified contract. */ for(i = 0; i < cat->num_ops; ++i) { const rocke_mma_op_t* op = &cat->ops[i]; - if(!rocke_ati_op_matches(op, fam, a, bd, c, m, n)) + if(!rocke_ati_op_matches(op, fam, a, bd, c, d, m, n) + || !rocke_ati_scales_match(op, src_scales)) continue; if(op->k == k) - return op; + { + if(match) + ckc::raise_status(ROCKE_ERR_VALUE, + "ambiguous MMA query; specify the full operand contract"); + match = op; + } } - return NULL; + return match; +} + +const rocke_mma_op_t* rocke_mma_catalog_op_for_shape(const rocke_mma_catalog_t* cat, + const char* family, + const char* a_dtype, + const char* b_dtype, + const char* c_dtype, + int m, + int n, + int k) +{ + const char* src_dtypes[3] = {a_dtype, b_dtype, c_dtype}; + return rocke_mma_catalog_op_for_shape_indexed(cat, family, src_dtypes, c_dtype, NULL, m, n, k); } /* ===================== bare-op_id SSOT lookups ======================== */ -const char* rocke_arch_mma_op_id_c_dtype(const char* op_id) +const char* rocke_arch_mma_op_id_dst_dtype(const char* op_id) { int i; if(!op_id) { return NULL; } - /* Mirrors target._op_id_c_dtype()[op_id]: the accumulator dtype names a + /* Mirrors target._op_id_dst_dtype()[op_id]: the dst dtype names a * specific atom, so it is invariant across the arches that list op_id -- - * the first catalog hit wins. The catalog c_dtype is already the normalised - * (canonical) key, matching normalize_dtype(o["c"]). Op_ids absent from + * the first catalog hit wins. The catalog dst.dtype is already the normalised + * (canonical) key, matching normalize_dtype(o["dst"]["dtype"]). Op_ids absent from * every catalog return NULL. */ for(i = 0; i < rocke_ati_arch_registry_len; ++i) { @@ -323,7 +484,7 @@ const char* rocke_arch_mma_op_id_c_dtype(const char* op_id) const rocke_mma_op_t* op = t ? rocke_mma_catalog_by_op_id(&t->mma, op_id) : NULL; if(op) { - return op->c_dtype; + return op->dst.dtype; } } return NULL; @@ -365,13 +526,25 @@ bool rocke_arch_fits_lds(const rocke_arch_target_t* t, long bytes_in_use) return bytes_in_use <= (long)t->lds_capacity_bytes; } -bool rocke_arch_supports_dtype_combo( - const rocke_arch_target_t* t, const char* a, const char* b, const char* c, const char* family) +bool rocke_arch_supports_dtype_combo_indexed(const rocke_arch_target_t* t, + const char* const src_dtypes[3], + const char* dst_dtype, + const rocke_mma_scale_operand_t src_scales[3], + const char* family) { if(!t) return false; /* Python: len(enumerate(...)) > 0 (m/n omitted => None => "any"). */ - return rocke_mma_catalog_enumerate(&t->mma, family, a, b, c, -1, -1, NULL, 0) > 0; + return rocke_mma_catalog_enumerate_indexed( + &t->mma, family, src_dtypes, dst_dtype, src_scales, -1, -1, NULL, 0) + > 0; +} + +bool rocke_arch_supports_dtype_combo( + const rocke_arch_target_t* t, const char* a, const char* b, const char* c, const char* family) +{ + const char* src_dtypes[3] = {a, b, c}; + return rocke_arch_supports_dtype_combo_indexed(t, src_dtypes, c, NULL, family); } int rocke_arch_max_vector_load_dwords(const rocke_arch_target_t* t, const char* dtype) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp index b335e4b334a1..48aeb992b0c3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp @@ -25,10 +25,10 @@ /* * rocke_b_mma emits a single tile.mma op keyed by op_id; the ISA backend lowers * that op_id to the matching MFMA/WMMA call. To size the result vector it needs - * the accumulator fragment length and dtype for the atom. Both come from the - * arch SSOT (core/arch/target): frag lengths from rocke_arch_mma_c_frag_len (the - * _MMA_FRAGMENT_INFO projection) and the accumulator dtype from - * rocke_arch_mma_op_id_c_dtype (the JSON catalog aggregation). This bucket keeps + * the dst fragment length and dtype for the atom. Both come from the + * arch SSOT (core/arch/target): frag lengths from rocke_arch_mma_dst_frag_len + * (the _MMA_FRAGMENT_INFO projection) and the dst dtype from + * rocke_arch_mma_op_id_dst_dtype (the JSON catalog aggregation). This bucket keeps * NO private copy of that data, mirroring ir.py after it dropped _MMA_C_FRAG_LEN * / _MMA_C_INT_OP_IDS. * @@ -49,30 +49,27 @@ static const rocke_mma_hint_row_t ROCKE_MMA_RESULT_HINT[] = { {"mfma_f32_16x16x96_fp6", "acc6"}, {"mfma_f32_16x16x128_fp8", "acc128"}, {"mfma_scale_f32_16x16x128_f8f6f4", "mxacc"}, - {"wmma_scale_f32_16x16x128_fp8_fp8", "mxacc"}, - {"wmma_scale_f32_16x16x128_bf8_bf8", "mxacc"}, - {"wmma_scale16_f32_16x16x128_fp8_fp8", "mxacc"}, - {"wmma_scale16_f32_16x16x128_bf8_bf8", "mxacc"}, }; -/* Accumulator fragment length for op_id, from the arch SSOT - * (target._frag_info(op_id).c_frag_len). Returns <= 0 for an unknown atom (the +/* dst fragment length for op_id, from the arch SSOT + * (target._frag_info(op_id).dst.frag_len). Returns <= 0 for an unknown atom (the * zero-length _frag_info fallback), which rocke_b_mma reports as an error. */ -static int rocke_mma_c_frag_len(const char* op_id) +static int rocke_mma_dst_frag_len(const char* op_id) { - return rocke_arch_mma_c_frag_len(op_id); + return rocke_arch_mma_dst_frag_len(op_id); } -/* True when op_id accumulates in i32 (integer WMMA), from the arch catalog SSOT - * (target._op_id_c_dtype()[op_id] == "i32"). */ -static bool rocke_mma_is_int_acc(const char* op_id) +/* True when op_id produces i32, from the arch catalog SSOT. */ +static bool rocke_mma_dst_is_int(const char* op_id) { - const char* c_dtype = rocke_arch_mma_op_id_c_dtype(op_id); - return c_dtype != NULL && strcmp(c_dtype, ROCKE_DTYPE_I32) == 0; + const char* dst_dtype = rocke_arch_mma_op_id_dst_dtype(op_id); + return dst_dtype != NULL && strcmp(dst_dtype, ROCKE_DTYPE_I32) == 0; } static const char* rocke_mma_result_hint(const char* op_id) { + if(op_id && strncmp(op_id, "wmma.scaled.", 12) == 0) + return "mxacc"; size_t i; if(op_id) { @@ -553,9 +550,9 @@ rocke_value_t* rocke_b_mma(rocke_ir_builder_t* b, rocke_value_t* const* extra, int num_extra) { - int c_frag_len; - bool is_int_acc; - const rocke_type_t* c_elem; + int dst_frag_len; + bool is_int_dst; + const rocke_type_t* dst_elem; const rocke_type_t* vt; const char* hint; rocke_attr_map_t attrs; @@ -570,19 +567,19 @@ rocke_value_t* rocke_b_mma(rocke_ir_builder_t* b, { return (rocke_value_t*)rocke_i_set_err(b, ROCKE_ERR_VALUE, "mma op_id is NULL"); } - /* C has no MmaOp object; op_id is always a bare string, so the frag length - * and accumulator element are resolved from the arch SSOT (the Python + /* The C API has no MmaOp object; op_id is always a bare string, so the dst + * frag length and element type are resolved from the arch SSOT (the Python * bare-string code path). A <= 0 frag length means the op_id is unknown to * the SSOT (the zero-length _frag_info fallback). */ - c_frag_len = rocke_mma_c_frag_len(op_id); - if(c_frag_len <= 0) + dst_frag_len = rocke_mma_dst_frag_len(op_id); + if(dst_frag_len <= 0) { return (rocke_value_t*)rocke_i_set_err( b, ROCKE_ERR_VALUE, "unknown MMA op_id '%s'; pass a known mfma_*/wmma_* op_id", op_id); } - is_int_acc = rocke_mma_is_int_acc(op_id); - c_elem = is_int_acc ? rocke_i32() : rocke_f32(); - vt = rocke_vector_type(b, c_elem, c_frag_len); + is_int_dst = rocke_mma_dst_is_int(op_id); + dst_elem = is_int_dst ? rocke_i32() : rocke_f32(); + vt = rocke_vector_type(b, dst_elem, dst_frag_len); if(!vt) { return NULL; @@ -655,32 +652,6 @@ rocke_value_t* rocke_b_mfma_scale_f32_16x16x128_f8f6f4(rocke_ir_builder_t* b, return rocke_b_mma(b, "mfma_scale_f32_16x16x128_f8f6f4", a, bb, c, extra, 2); } -rocke_value_t* rocke_b_wmma_scale_f32_16x16x128_fp8_fp8(rocke_ir_builder_t* b, - rocke_value_t* a, - rocke_value_t* bb, - rocke_value_t* c, - rocke_value_t* a_scale, - rocke_value_t* b_scale) -{ - rocke_value_t* extra[2]; - extra[0] = a_scale; - extra[1] = b_scale; - return rocke_b_mma(b, "wmma_scale_f32_16x16x128_fp8_fp8", a, bb, c, extra, 2); -} - -rocke_value_t* rocke_b_wmma_scale16_f32_16x16x128_fp8_fp8(rocke_ir_builder_t* b, - rocke_value_t* a, - rocke_value_t* bb, - rocke_value_t* c, - rocke_value_t* a_scale, - rocke_value_t* b_scale) -{ - rocke_value_t* extra[2]; - extra[0] = a_scale; - extra[1] = b_scale; - return rocke_b_mma(b, "wmma_scale16_f32_16x16x128_fp8_fp8", a, bb, c, extra, 2); -} - /* ===================================================================== */ /* register-fragment reshape (P13) */ /* ===================================================================== */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mma.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mma.cpp index 5afb304eb25a..3abfd360a3c5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mma.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_mma.cpp @@ -78,14 +78,15 @@ static const char* h_elem_scalar(const rocke_type_t* t) * reusing the original attrs map reproduces the Python emission exactly. */ static rocke_status_t h_emit_gfx1250_scaled_wmma(rocke_h_lowerer_t* lw, const rocke_op_t* op) { - const rocke_scaled_wmma_op_t* spec = rocke_gfx1250_scaled_wmma_from_op(op); - if(!spec) + const rocke_mma_op_t* atom = rocke_gfx1250_scaled_wmma_from_op(op); + if(!atom) { return rocke_h_fail(lw, ROCKE_ERR_NOTIMPL, "unsupported scaled WMMA op '%s'", op->name); } + const rocke_scaled_wmma_op_t contract = rocke_scaled_wmma_contract(atom); + const rocke_scaled_wmma_op_t* spec = &contract; const char* op_id = spec->op_id; const bool scale16 = spec->scales.block_k == 16; - const int fmt = spec->matrix_format; const char* builtin = scale16 ? "__builtin_amdgcn_wmma_scale16_f32_16x16x128_f8f6f4" : "__builtin_amdgcn_wmma_scale_f32_16x16x128_f8f6f4"; if(!lw->arch.gfx || __builtin_strcmp(lw->arch.gfx, "gfx1250") != 0) @@ -102,15 +103,17 @@ static rocke_status_t h_emit_gfx1250_scaled_wmma(rocke_h_lowerer_t* lw, const ro } rocke_h_emitf(lw, "f32x8 %s = %s(%d, %s, %d, %s, (int16_t)0, %s, " - "0, 0, %s, 0, 0, %s, false, false);", + "0, %d, %s, 0, %d, %s, false, false);", rocke_h_name(lw, op->results[0]), builtin, - fmt, + spec->matrix_formats[0], rocke_h_name(lw, op->operands[0]), - fmt, + spec->matrix_formats[1], rocke_h_name(lw, op->operands[1]), rocke_h_name(lw, op->operands[2]), + spec->scale_formats[0], rocke_h_name(lw, op->operands[3]), + spec->scale_formats[1], rocke_h_name(lw, op->operands[4])); return lw->status; } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/data.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/data.cpp index 1ee6248e7f8d..e6fd436551cb 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/data.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/data.cpp @@ -213,11 +213,11 @@ const rocke_ll_decl_t ROCKE_LL_INTRINSIC_DECLS[] = { {"wmma.gfx1250.f32.16x16x64.bf8.bf8", "declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.bf8.v8f32.v8i32(<8 x i32>, <8 x i32>, " "i16 immarg, <8 x float>, i1 immarg, i1 immarg)"}, - {"wmma.scale.gfx1250.f32.16x16x128.fp8.fp8", + {"wmma.scale.block32.gfx1250.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32", "declare <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 " "immarg, <16 x i32>, i32 immarg, <16 x i32>, i16 immarg, <8 x float>, i32 immarg, i32 immarg, " "i32, i32 immarg, i32 immarg, i32, i1 immarg, i1 immarg)"}, - {"wmma.scale16.gfx1250.f32.16x16x128.fp8.fp8", + {"wmma.scale.block16.gfx1250.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32", "declare <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32(i32 " "immarg, <16 x i32>, i32 immarg, <16 x i32>, i16 immarg, <8 x float>, i32 immarg, i32 immarg, " "i64, i32 immarg, i32 immarg, i64, i1 immarg, i1 immarg)"}, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp index 93554dac6bc7..e3c9c980f346 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp @@ -170,9 +170,9 @@ static void _op_tile_mma(rocke_lower_t* L, const rocke_op_t* op) } } -/* gfx1250 native scaled FP8/FP4 WMMA. ROCm 7.13 introduced the LLVM 23 ABI: - * matrix operands are <16 x i32>; SCALE carries four packed E8M0 bytes in - * each i32 scale operand and SCALE16 carries eight in i64. */ +/* Emit the gfx1250 scaled-WMMA intrinsic using the resolved operand contract. + * The contract supplies packed carrier types, matrix and scale format selectors, + * and the declaration identity. This instruction family requires LLVM 23. */ static void _emit_wmma_scale(rocke_lower_t* L, const rocke_op_t* op) { const char* intrinsic; @@ -184,13 +184,13 @@ static void _emit_wmma_scale(rocke_lower_t* L, const rocke_op_t* op) { return; } - const rocke_scaled_wmma_op_t* spec = rocke_gfx1250_scaled_wmma_from_op(op); - if(!spec) + const rocke_mma_op_t* atom = rocke_gfx1250_scaled_wmma_from_op(op); + if(!atom) { rocke_ll_fail(L, ROCKE_ERR_NOTIMPL, "unsupported scaled WMMA op '%s'", op->name); } - const bool scale16 = spec->scales.block_k == 16; - const int fmt = spec->matrix_format; + const rocke_scaled_wmma_op_t contract = rocke_scaled_wmma_contract(atom); + const rocke_scaled_wmma_op_t* spec = &contract; char concrete_name[160]; snprintf(concrete_name, sizeof(concrete_name), "tile.%s", spec->op_id); op_name = concrete_name; @@ -231,31 +231,28 @@ static void _emit_wmma_scale(rocke_lower_t* L, const rocke_op_t* op) op->operands[4]->type->name); } - if(scale16) - { - decl_key = "wmma.scale16.gfx1250.f32.16x16x128.fp8.fp8"; - intrinsic = "llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32"; - } - else - { - decl_key = "wmma.scale.gfx1250.f32.16x16x128.fp8.fp8"; - intrinsic = "llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32"; - } + // Physical declaration identity comes from the resolved backend contract. + decl_key = spec->declaration_key; + intrinsic = spec->intrinsic; rocke_ll_need(L, decl_key); rocke_ll_emitf(L, " %s = call <8 x float> @%s(" - "i32 %d, <16 x i32> %s, i32 %d, <16 x i32> %s, " - "i16 0, <8 x float> %s, i32 0, i32 0, %s %s, " - "i32 0, i32 0, %s %s, i1 false, i1 false)", + "i32 %d, <%d x i32> %s, i32 %d, <%d x i32> %s, " + "i16 0, <8 x float> %s, i32 0, i32 %d, %s %s, " + "i32 0, i32 %d, %s %s, i1 false, i1 false)", mma_result_name(L, op), intrinsic, - fmt, + spec->matrix_formats[0], + spec->matrix_words[0], rocke_ll_operand(L, op->operands[0]), - fmt, + spec->matrix_formats[1], + spec->matrix_words[1], rocke_ll_operand(L, op->operands[1]), rocke_ll_operand(L, op->operands[2]), + spec->scale_formats[0], scale_ty, rocke_ll_operand(L, op->operands[3]), + spec->scale_formats[1], scale_ty, rocke_ll_operand(L, op->operands[4])); } @@ -444,23 +441,26 @@ static const int WMMA_INT_SPECS_N = (int)(sizeof(WMMA_INT_SPECS) / sizeof(WMMA_I static void _emit_wmma_int(rocke_lower_t* L, const rocke_op_t* op, const _wmma_int_spec_t* spec) { const rocke_value_t *a, *b, *c; + const char *src2_ty, *dst_ty; a = op->operands[0]; b = op->operands[1]; c = op->operands[2]; + src2_ty = rocke_ll_llvm_type(L, c->type); + dst_ty = rocke_ll_llvm_type(L, op->results[0]->type); rocke_ll_need(L, spec->decl_key); rocke_ll_emitf(L, - " %s = call <%d x i32> @%s(" + " %s = call %s @%s(" "i1 1, <%d x i32> %s, " "i1 1, <%d x i32> %s, " - "<%d x i32> %s, i1 0)", + "%s %s, i1 0)", mma_result_name(L, op), - spec->acc_vec, + dst_ty, spec->intrinsic, spec->op_vec, rocke_ll_operand(L, a), spec->op_vec, rocke_ll_operand(L, b), - spec->acc_vec, + src2_ty, rocke_ll_operand(L, c)); } @@ -468,7 +468,7 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id { const _wmma_spec_t* spec = NULL; const rocke_value_t *a, *b, *c; - const char *a_arg, *b_arg; + const char *a_arg, *b_arg, *src2_ty, *dst_ty; int w, i; if(!rocke_ll_live(L)) @@ -516,6 +516,8 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id a = op->operands[0]; b = op->operands[1]; c = op->operands[2]; + src2_ty = rocke_ll_llvm_type(L, c->type); + dst_ty = rocke_ll_llvm_type(L, op->results[0]->type); w = spec->frag_width; rocke_ll_need(L, spec->decl_key); @@ -554,12 +556,13 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id * i1 reuseA, i1 reuseB). The negate / format / reuse immediates are * pinned to the plain unscaled MMA. */ rocke_ll_emitf(L, - " %s = call <8 x float> @%s(" + " %s = call %s @%s(" "i1 false, <%d x %s> %s, " "i1 false, <%d x %s> %s, " - "i16 0, <8 x float> %s, " + "i16 0, %s %s, " "i1 false, i1 false)", mma_result_name(L, op), + dst_ty, spec->intrinsic, w, spec->call_elt, @@ -567,6 +570,7 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id w, spec->call_elt, b_arg, + src2_ty, rocke_ll_operand(L, c)); return; } @@ -575,11 +579,12 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id /* gfx1250 K=64 6-operand form: (A, B, i16 fmt, C, i1, i1), format and * reuse immediates pinned to 0 (plain unscaled MMA). */ rocke_ll_emitf(L, - " %s = call <8 x float> @%s(" + " %s = call %s @%s(" "<%d x %s> %s, <%d x %s> %s, " - "i16 0, <8 x float> %s, " + "i16 0, %s %s, " "i1 false, i1 false)", mma_result_name(L, op), + dst_ty, spec->intrinsic, w, spec->call_elt, @@ -587,13 +592,15 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id w, spec->call_elt, b_arg, + src2_ty, rocke_ll_operand(L, c)); return; } rocke_ll_emitf(L, - " %s = call <8 x float> @%s(" - "<%d x %s> %s, <%d x %s> %s, <8 x float> %s)", + " %s = call %s @%s(" + "<%d x %s> %s, <%d x %s> %s, %s %s)", mma_result_name(L, op), + dst_ty, spec->intrinsic, w, spec->call_elt, @@ -601,6 +608,7 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id w, spec->call_elt, b_arg, + src2_ty, rocke_ll_operand(L, c)); } @@ -612,16 +620,17 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id * guards (live + exactly 3 operands), tracks one intrinsic decl key, optionally * prepends a fixed two-line operand bitcast (the bf16 `_1k` atoms widen * <4 x bfloat> -> <4 x i16>), then emits a single MFMA call. They differ only - * by four literal strings (decl key, intrinsic, A/B element-vector spelling, - * accumulator/result vector spelling) plus the optional bitcast target. This + * by five literal strings (decl key, intrinsic, src0/src1 element-vector + * spelling, src2 spelling, dst spelling) plus the optional bitcast target. This * mirrors the WMMA_SPECS / _emit_wmma idiom already used in this file. */ typedef struct _mfma_spec { const char* op_id; /* the tile.mma op_id (no "tile." prefix) */ const char* decl_key; /* _need() key */ const char* intrinsic; /* fully-mangled @llvm.amdgcn.mfma.... */ - const char* ab_ty; /* A/B SSA operand type spelling */ - const char* acc_ty; /* accumulator/result vector spelling */ + const char* src01_ty; /* src0/src1 SSA operand type spelling */ + const char* src2_ty; /* src2 vector spelling */ + const char* dst_ty; /* dst vector spelling */ const char* bitcast_to; /* operand bitcast target (NULL = no bitcast) */ } _mfma_spec_t; @@ -631,12 +640,14 @@ static const _mfma_spec_t MFMA_SPECS[] = { "llvm.amdgcn.mfma.f32.16x16x16f16", "<4 x half>", "<4 x float>", + "<4 x float>", NULL}, {"mfma_f32_16x16x32_f16", "mfma.f32.16x16x32.f16", "llvm.amdgcn.mfma.f32.16x16x32.f16", "<8 x half>", "<4 x float>", + "<4 x float>", NULL}, /* bf16 `_1k`: bitcast <4 x bfloat> -> <4 x i16> before the call. */ {"mfma_f32_16x16x16_bf16", @@ -644,18 +655,21 @@ static const _mfma_spec_t MFMA_SPECS[] = { "llvm.amdgcn.mfma.f32.16x16x16bf16.1k", "<4 x bfloat>", "<4 x float>", + "<4 x float>", "<4 x i16>"}, {"mfma_f32_16x16x32_bf16", "mfma.f32.16x16x32.bf16", "llvm.amdgcn.mfma.f32.16x16x32.bf16", "<8 x bfloat>", "<4 x float>", + "<4 x float>", NULL}, {"mfma_f32_32x32x8_f16", "mfma.f32.32x32x8f16", "llvm.amdgcn.mfma.f32.32x32x8f16", "<4 x half>", "<16 x float>", + "<16 x float>", NULL}, /* bf16 `_1k`: bitcast <4 x bfloat> -> <4 x i16> exactly like 16x16x16. */ {"mfma_f32_32x32x8_bf16", @@ -663,6 +677,7 @@ static const _mfma_spec_t MFMA_SPECS[] = { "llvm.amdgcn.mfma.f32.32x32x8bf16.1k", "<4 x bfloat>", "<16 x float>", + "<16 x float>", "<4 x i16>"}, /* fp32 (TF32-class) scalar atoms: A/B are single floats per lane. */ {"mfma_f32_16x16x4_f32", @@ -670,30 +685,35 @@ static const _mfma_spec_t MFMA_SPECS[] = { "llvm.amdgcn.mfma.f32.16x16x4f32", "float", "<4 x float>", + "<4 x float>", NULL}, {"mfma_f32_32x32x2_f32", "mfma.f32.32x32x2f32", "llvm.amdgcn.mfma.f32.32x32x2f32", "float", "<16 x float>", + "<16 x float>", NULL}, {"mfma_f32_32x32x16_f16", "mfma.f32.32x32x16.f16", "llvm.amdgcn.mfma.f32.32x32x16.f16", "<8 x half>", "<16 x float>", + "<16 x float>", NULL}, {"mfma_f32_32x32x16_bf16", "mfma.f32.32x32x16.bf16", "llvm.amdgcn.mfma.f32.32x32x16.bf16", "<8 x bfloat>", "<16 x float>", + "<16 x float>", NULL}, {"mfma_f32_4x4x4_f16", "mfma.f32.4x4x4f16", "llvm.amdgcn.mfma.f32.4x4x4f16", "<4 x half>", "<4 x float>", + "<4 x float>", NULL}, }; static const int MFMA_SPECS_N = (int)(sizeof(MFMA_SPECS) / sizeof(MFMA_SPECS[0])); @@ -732,13 +752,13 @@ static void _emit_mfma(rocke_lower_t* L, const rocke_op_t* op, const _mfma_spec_ rocke_ll_emitf(L, " %s = bitcast %s %s to %s", a_cast, - spec->ab_ty, + spec->src01_ty, rocke_ll_operand(L, a), spec->bitcast_to); rocke_ll_emitf(L, " %s = bitcast %s %s to %s", b_cast, - spec->ab_ty, + spec->src01_ty, rocke_ll_operand(L, b), spec->bitcast_to); a_arg = a_cast; @@ -749,20 +769,20 @@ static void _emit_mfma(rocke_lower_t* L, const rocke_op_t* op, const _mfma_spec_ { a_arg = rocke_ll_operand(L, a); b_arg = rocke_ll_operand(L, b); - call_ty = spec->ab_ty; + call_ty = spec->src01_ty; } rocke_ll_emitf(L, " %s = call %s @%s(" "%s %s, %s %s, %s %s, i32 0, i32 0, i32 0)", mma_result_name(L, op), - spec->acc_ty, + spec->dst_ty, spec->intrinsic, call_ty, a_arg, call_ty, b_arg, - spec->acc_ty, + spec->src2_ty, rocke_ll_operand(L, c)); } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp index 152641849784..ed05b5e5b330 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp @@ -567,7 +567,7 @@ void rocke_cshuffle_epilogue_store(rocke_ir_builder_t* b, } /* ---- step 1: publish accs to LDS. ---- - * WMMA path: use op->c_layout().coord() for the scatter. + * WMMA path: use rocke_mma_op_dst_layout(op).coord() for the scatter. * MFMA path: use atom->lane_to_output(). */ for(mi = 0; mi < mfmas_m; ++mi) { @@ -589,7 +589,7 @@ void rocke_cshuffle_epilogue_store(rocke_ir_builder_t* b, if(epi->mma_op != NULL) { - c_per_lane = epi->mma_op->c_frag_len; + c_per_lane = epi->mma_op->dst.frag_len; atom_m = epi->mma_op->m; atom_n = epi->mma_op->n; } @@ -858,7 +858,7 @@ void rocke_cshuffle_epilogue_atomic_store(rocke_ir_builder_t* b, if(epi->mma_op != NULL) { - c_per_lane = epi->mma_op->c_frag_len; + c_per_lane = epi->mma_op->dst.frag_len; atom_m = epi->mma_op->m; atom_n = epi->mma_op->n; } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mfma_attention.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mfma_attention.cpp index b9102aa0cb03..d925455b3b78 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mfma_attention.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mfma_attention.cpp @@ -722,10 +722,10 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, return rocke_ir_builder_status(b); } - const rocke_layout_map_t* a_map = op->a_layout; - const rocke_layout_map_t* c_map = op->c_layout; - int a_frag = op->a_frag_len; - int c_frag = op->c_frag_len; + const rocke_layout_map_t* src0_map = op->srcs[0].layout; + const rocke_layout_map_t* dst_map = op->dst.layout; + int src0_frag = op->srcs[0].frag_len; + int dst_frag = op->dst.frag_len; int n_dk = head_size / 16; /* Python evaluates b.mod(b.thread_id_x(), b.const_i32(wave)) left-to-right: @@ -738,22 +738,22 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, rocke_value_t* a_row = NULL; rocke_value_t* dummy = NULL; - rocke_layout_map_coord(a_map, b, lane, 0, &a_row, &dummy); + rocke_layout_map_coord(src0_map, b, lane, 0, &a_row, &dummy); /* gfx12 (RDNA4) split-K: the 16 K-elements of one WMMA step are split across - * the two lane-halves, so each lane loads a_frag (=8) elements from K base - * (lane // 16) * a_frag. gfx11 (RDNA3/3.5) duplicates the full K row in every - * lane (a_frag=16, base 0). k_half_off==NULL keeps the gfx11 emission + * the two lane-halves, so each lane loads src0_frag (=8) elements from K base + * (lane // 16) * src0_frag. gfx11 (RDNA3/3.5) duplicates the full K row in every + * lane (src0_frag=16, base 0). k_half_off==NULL keeps the gfx11 emission * byte-identical (no half-offset add); mirrors Python's split_k handling. */ - bool split_k = (a_frag * 2 == 16); + bool split_k = (src0_frag * 2 == 16); rocke_value_t* k_half_off = NULL; if(split_k) { - /* Python: b.mul(b.div(lane, c16), b.const_i32(a_frag)) -- div created + /* Python: b.mul(b.div(lane, c16), b.const_i32(src0_frag)) -- div created * before the const (left-to-right). Hoist so the C arg-eval order (gcc is * right-to-left) matches the Python value-creation order. */ rocke_value_t* half = rocke_b_div(b, lane, c16); - rocke_value_t* c_frag_off = rocke_b_const_i32(b, a_frag); - k_half_off = rocke_b_mul(b, half, c_frag_off); + rocke_value_t* src0_frag_off = rocke_b_const_i32(b, src0_frag); + k_half_off = rocke_b_mul(b, half, src0_frag_off); } rocke_value_t* col = rocke_b_mod(b, lane, c16); @@ -779,7 +779,7 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, { q_addr = rocke_b_add(b, q_addr, k_half_off); } - q_frags[d] = rocke_b_global_load_vN(b, p->Q, q_addr, dtype_ir, a_frag, a_frag * 2); + q_frags[d] = rocke_b_global_load_vN(b, p->Q, q_addr, dtype_ir, src0_frag, src0_frag * 2); } /* ---- LDS staging tiles ---- */ @@ -796,7 +796,7 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, rocke_iter_arg_t iter_args[ROCKE_ATTN_MAX_ITER_ARGS]; char name_buf[ROCKE_ATTN_MAX_ITER_ARGS][16]; int n_ia = 0; - for(int r = 0; r < c_frag; ++r) + for(int r = 0; r < dst_frag; ++r) { snprintf(name_buf[n_ia], sizeof(name_buf[0]), "m%d", r); iter_args[n_ia].name = name_buf[n_ia]; @@ -811,7 +811,7 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, { snprintf(name_buf[n_ia], sizeof(name_buf[0]), "acc%d", d); iter_args[n_ia].name = name_buf[n_ia]; - iter_args[n_ia].init = rocke_b_zero_vec_f32(b, c_frag); + iter_args[n_ia].init = rocke_b_zero_vec_f32(b, dst_frag); ++n_ia; } @@ -829,14 +829,14 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, rocke_value_t* ms[ROCKE_ATTN_MAX_LANE]; rocke_value_t* ls[ROCKE_ATTN_MAX_LANE]; rocke_value_t* accs[ROCKE_ATTN_MAX_ATOMS]; - for(int r = 0; r < c_frag; ++r) + for(int r = 0; r < dst_frag; ++r) { ms[r] = kloop.iter_vars[2 * r]; ls[r] = kloop.iter_vars[2 * r + 1]; } for(int d = 0; d < n_dk; ++d) { - accs[d] = kloop.iter_vars[2 * c_frag + d]; + accs[d] = kloop.iter_vars[2 * dst_frag + d]; } rocke_value_t* effective_kt @@ -870,7 +870,7 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, } /* ---- QK^T WMMA chain ---- */ - rocke_value_t* score = rocke_b_zero_vec_f32(b, c_frag); + rocke_value_t* score = rocke_b_zero_vec_f32(b, dst_frag); for(int d = 0; d < n_dk; ++d) { rocke_value_t* k_addr = rocke_b_add(b, k_addr_row_base, rocke_b_const_i32(b, d * 16)); @@ -879,7 +879,7 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, k_addr = rocke_b_add(b, k_addr, k_half_off); } rocke_value_t* k_frag - = rocke_b_global_load_vN(b, p->K, k_addr, dtype_ir, a_frag, a_frag * 2); + = rocke_b_global_load_vN(b, p->K, k_addr, dtype_ir, src0_frag, src0_frag * 2); score = rocke_b_mma(b, op->op_id, q_frags[d], k_frag, score, NULL, 0); } @@ -893,11 +893,11 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, new_accs[d] = accs[d]; } rocke_value_t* q_pos_for_mask = (p->q_pos_base != NULL) ? p->q_pos_base : p->q_tile_base; - for(int r = 0; r < c_frag; ++r) + for(int r = 0; r < dst_frag; ++r) { rocke_value_t* row_rel = NULL; rocke_value_t* col_k = NULL; - rocke_layout_map_coord(c_map, b, lane, r, &row_rel, &col_k); + rocke_layout_map_coord(dst_map, b, lane, r, &row_rel, &col_k); rocke_value_t* s_r = rocke_b_fmul(b, rocke_b_vec_extract(b, score, r), p->scale_log2); rocke_value_t* row_q_pos = rocke_b_add(b, q_pos_for_mask, row_rel); rocke_value_t* k_col_pos = rocke_b_add(b, k_tile_base, col_k); @@ -964,23 +964,23 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, } /* ---- P staging through LDS ---- */ - for(int r = 0; r < c_frag; ++r) + for(int r = 0; r < dst_frag; ++r) { rocke_value_t* row_rel = NULL; rocke_value_t* col_k = NULL; - rocke_layout_map_coord(c_map, b, lane, r, &row_rel, &col_k); + rocke_layout_map_coord(dst_map, b, lane, r, &row_rel, &col_k); rocke_value_t* idx[2] = {row_rel, col_k}; rocke_b_smem_store_vN(b, P_lds, idx, 2, rocke_b_cast_f32_to(b, ps_arr[r], dtype_ir), 1); } rocke_b_sync(b); /* ---- V load + PV WMMA chain ---- */ - rocke_value_t* p_a = rocke_b_zero_vec(b, dtype_ir, a_frag); - for(int j = 0; j < a_frag; ++j) + rocke_value_t* p_a = rocke_b_zero_vec(b, dtype_ir, src0_frag); + for(int j = 0; j < src0_frag; ++j) { rocke_value_t* a_k = NULL; rocke_value_t* a_dummy = NULL; - rocke_layout_map_coord(a_map, b, lane, j, &a_dummy, &a_k); + rocke_layout_map_coord(src0_map, b, lane, j, &a_dummy, &a_k); rocke_value_t* idx[2] = {a_row, a_k}; rocke_value_t* p_v = rocke_b_vec_extract(b, rocke_b_smem_load_vN(b, P_lds, idx, 2, dtype_ir, 1), 0); @@ -990,12 +990,12 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, for(int d = 0; d < n_dk; ++d) { rocke_value_t* d_col = rocke_b_add(b, rocke_b_const_i32(b, d * 16), col); - rocke_value_t* v_b = rocke_b_zero_vec(b, dtype_ir, a_frag); - for(int j = 0; j < a_frag; ++j) + rocke_value_t* v_b = rocke_b_zero_vec(b, dtype_ir, src0_frag); + for(int j = 0; j < src0_frag; ++j) { /* B-operand K row this lane's slot j feeds: j on gfx11 (full K * per lane, byte-identical to the historical literal), or - * (lane // 16) * a_frag + j on gfx12 (split-K halves). Python: + * (lane // 16) * src0_frag + j on gfx12 (split-K halves). Python: * b.add(k_half_off, b.const_i32(j)) -- k_half_off created before * the const, so it is the first add operand. */ rocke_value_t* b_k = (k_half_off != NULL) @@ -1033,7 +1033,7 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, rocke_value_t* yields[ROCKE_ATTN_MAX_ITER_ARGS]; int ny = 0; - for(int r = 0; r < c_frag; ++r) + for(int r = 0; r < dst_frag; ++r) { yields[ny++] = new_ms[r]; yields[ny++] = new_ls[r]; @@ -1048,23 +1048,23 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, rocke_value_t* ls_final[ROCKE_ATTN_MAX_LANE]; rocke_value_t* accs_final[ROCKE_ATTN_MAX_ATOMS]; - for(int r = 0; r < c_frag; ++r) + for(int r = 0; r < dst_frag; ++r) { ls_final[r] = (kloop.op != NULL) ? kloop.op->results[2 * r + 1] : NULL; } for(int d = 0; d < n_dk; ++d) { - accs_final[d] = (kloop.op != NULL) ? kloop.op->results[2 * c_frag + d] : NULL; + accs_final[d] = (kloop.op != NULL) ? kloop.op->results[2 * dst_frag + d] : NULL; } /* ---- Epilogue ---- */ for(int d = 0; d < n_dk; ++d) { - for(int r = 0; r < c_frag; ++r) + for(int r = 0; r < dst_frag; ++r) { rocke_value_t* row_rel = NULL; rocke_value_t* col_n = NULL; - rocke_layout_map_coord(c_map, b, lane, r, &row_rel, &col_n); + rocke_layout_map_coord(dst_map, b, lane, r, &row_rel, &col_n); rocke_value_t* l_safe = ls_final[r]; rocke_value_t* zero_mask = rocke_b_fcmp(b, "oeq", l_safe, zero_f); rocke_value_t* inv_l = rocke_b_select(b, zero_mask, zero_f, rocke_b_rcp(b, l_safe)); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h index 672dfd40e2a8..e6aff19d876b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h @@ -17,7 +17,7 @@ * .fn closure rocke_lane_coord_fn (fn ptr + builder) * .coord(builder, lane, slot) rocke_layout_map_coord() * class MmaOp (frozen) rocke_mma_op_t - * .shape / .a_layout()/... rocke_mma_op_shape() / *_layout() getters + * .shape / .src_layout()/... rocke_mma_op_shape() / *_layout() getters * class MemoryCapabilities rocke_memory_caps_t * class ResourceLimits rocke_resource_limits_t * class MmaCatalog rocke_mma_catalog_t + rocke_mma_catalog_* () @@ -58,14 +58,14 @@ extern "C" { /* ============================== layout map ============================== */ -/* MMA fragment role. Mirrors LayoutMap.role ("acc"/"a"/"b"). The accumulator is - * spelled "c" in the C op_id maps (MmaOp.c_layout) but role text is "acc"/"a"/ - * "b" exactly as Python LayoutMap.role. */ +/* Machine-level MMA operand position. The stored contract is + * src0/src1/src2 -> dst. */ typedef enum rocke_mma_role { - ROCKE_MMA_ROLE_ACC = 0, /* accumulator C/D: coords (row, col) */ - ROCKE_MMA_ROLE_A, /* A operand: coords (row, k) */ - ROCKE_MMA_ROLE_B /* B operand: coords (k, col) */ + ROCKE_MMA_ROLE_SRC0 = 0, /* src0: coords (row, k) */ + ROCKE_MMA_ROLE_SRC1, /* src1: coords (k, col) */ + ROCKE_MMA_ROLE_SRC2, /* src2: coords (row, col) */ + ROCKE_MMA_ROLE_DST /* dst: coords (row, col) */ } rocke_mma_role_t; /* The lane/slot -> tile-coordinate emitter. Given the builder, a runtime i32 @@ -103,44 +103,70 @@ bool rocke_layout_map_coord(const rocke_layout_map_t* m, /* ============================== MMA atom =============================== */ -/* A single supported matrix-multiply-accumulate atom on a target. Frozen; lives - * in static storage. Mirrors MmaOp. `family` is "mma"|"wmma"; dtype fields hold - * canonical (normalised) catalog keys; op_id is the opaque backend handle (the - * ISA-named IRBuilder method, e.g. "mfma_f32_16x16x16_f16"). The *_layout - * pointers are NULL when no verified lane map is registered for the op_id (the - * Python `_*_layout is None` / NotImplementedError case). */ +/* Metadata shared by one matrix src or dst. */ +typedef struct rocke_mma_dst +{ + const char* dtype; + int frag_len; + const rocke_layout_map_t* layout; /* may be NULL */ +} rocke_mma_dst_t; + +/* A matrix source may have a distinct associated scale operand. NULL means + * that the source is unscaled. Scales are not matrix fragments and therefore + * are not flattened into the srcs[] array. */ +/* Query scale descriptor. A NULL query array leaves scales unconstrained; + * an array entry {NULL, 0} explicitly requests an unscaled source. */ +typedef struct rocke_mma_scale_operand +{ + const char* dtype; + int block_size; +} rocke_mma_scale_operand_t; + +typedef struct rocke_mma_src +{ + const char* dtype; + int frag_len; + const rocke_layout_map_t* layout; /* may be NULL */ + const char* scale_dtype; /* scale value format: e8m0/e4m3/e5m3; NULL if unscaled */ + int scale_block_size; /* source elements along K per scale: 16 or 32; 0 if unscaled */ +} rocke_mma_src_t; + typedef struct rocke_mma_op { const char* family; - const char* a_dtype; - const char* b_dtype; - const char* c_dtype; + rocke_mma_src_t srcs[3]; + rocke_mma_dst_t dst; int m; int n; int k; const char* op_id; - int a_frag_len; - int b_frag_len; - int c_frag_len; int wave_size; - const rocke_layout_map_t* a_layout; /* may be NULL */ - const rocke_layout_map_t* b_layout; /* may be NULL */ - const rocke_layout_map_t* c_layout; /* may be NULL */ } rocke_mma_op_t; +/* Format semantic identity; NULL if the output buffer is too small. */ +const char* rocke_mma_op_semantic_id(const rocke_mma_op_t* op, char* out, size_t cap); + /* MmaOp.shape -> (m, n, k) via out params. */ void rocke_mma_op_shape(const rocke_mma_op_t* op, int* m, int* n, int* k); -/* Physical-layout accessors. Each returns the verified map for the role, or NULL - * when none is registered. Unlike Python (which raises NotImplementedError), the - * C getters return NULL and -- when `b` is non-NULL -- set the builder's sticky - * error (ROCKE_ERR_NOTIMPL) with the same message text, so callers can either - * check NULL or rely on the sticky-fail builder. Pass b=NULL for a pure lookup. - * Mirrors MmaOp.a_layout / b_layout / c_layout / acc_layout. */ +/* Canonical physical-layout accessors. Each returns the verified map for the + * machine operand position, or NULL when none is registered. Source indices + * outside [0, 3) are rejected. */ +const rocke_layout_map_t* + rocke_mma_op_src_layout(const rocke_mma_op_t* op, int index, rocke_ir_builder_t* b); +const rocke_layout_map_t* rocke_mma_op_dst_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b); + +/* Historical mathematical-role compatibility accessors. A and B project src0 + * and src1; C projects the result/dst, matching the original operator-level + * surface. src2 is available only through the indexed + * accessor above. Each returns the verified map, or NULL when none is + * registered. Unlike Python (which raises NotImplementedError), the C getters + * return NULL and -- when `b` is non-NULL -- set the builder's sticky error + * (ROCKE_ERR_NOTIMPL) with the same message text, so callers can either check + * NULL or rely on the sticky-fail builder. Pass b=NULL for a pure lookup. */ const rocke_layout_map_t* rocke_mma_op_a_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b); const rocke_layout_map_t* rocke_mma_op_b_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b); const rocke_layout_map_t* rocke_mma_op_c_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b); -const rocke_layout_map_t* rocke_mma_op_acc_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b); /* ====================== memory caps / resource limits ================== */ @@ -172,11 +198,27 @@ typedef struct rocke_mma_catalog /* MmaCatalog.ops accessor (count + pointer). */ const rocke_mma_op_t* rocke_mma_catalog_ops(const rocke_mma_catalog_t* cat, int* num_out); -/* MmaCatalog.enumerate: filter atoms by family + (normalised) dtype combo and - * optional m/n. Writes up to `cap` matching op pointers into `out` and returns +/* Indexed MmaCatalog.enumerate: filter atoms by family + the exact three src + * dtypes and dst dtype. `src_dtypes` must contain three entries. + * `src_scales` is NULL for an unconstrained query, or three exact descriptors. + * Filter by optional m/n. Writes up to `cap` matching op pointers into `out` and returns + * the total number of matches (which may exceed `cap`). */ +int rocke_mma_catalog_enumerate_indexed(const rocke_mma_catalog_t* cat, + const char* family, + const char* const src_dtypes[3], + const char* dst_dtype, + const rocke_mma_scale_operand_t src_scales[3], + int m, + int n, + const rocke_mma_op_t** out, + int cap); + +/* Compatibility MmaCatalog.enumerate: filter atoms by family + A/B/C dtypes and + * `src_scales` is NULL for an unconstrained query, or three exact descriptors. + * Filter by optional m/n. Writes up to `cap` matching op pointers into `out` and returns * the total number of matches (which may exceed `cap` -- the caller can size a * buffer with cap=0 to count first). Pass m<0 / n<0 to mean "any" (Python None). - * `family` may be NULL => "mma". */ + * `family` may be NULL => "mma". The dst dtype defaults to C. */ int rocke_mma_catalog_enumerate(const rocke_mma_catalog_t* cat, const char* family, const char* a_dtype, @@ -187,7 +229,15 @@ int rocke_mma_catalog_enumerate(const rocke_mma_catalog_t* cat, const rocke_mma_op_t** out, int cap); -/* MmaCatalog.has_shape. family NULL => "mma". */ +/* Indexed and compatibility MmaCatalog.has_shape variants. */ +bool rocke_mma_catalog_has_shape_indexed(const rocke_mma_catalog_t* cat, + const char* family, + const char* const src_dtypes[3], + const char* dst_dtype, + const rocke_mma_scale_operand_t src_scales[3], + int m, + int n, + int k); bool rocke_mma_catalog_has_shape(const rocke_mma_catalog_t* cat, const char* family, const char* a_dtype, @@ -197,8 +247,16 @@ bool rocke_mma_catalog_has_shape(const rocke_mma_catalog_t* cat, int n, int k); -/* MmaCatalog.select_largest_k: the matching atom with the largest k (k <= k_max - * if k_max >= 0; pass k_max<0 for Python None). Returns NULL if none match. */ +/* Indexed and compatibility MmaCatalog.select_largest_k variants. */ +const rocke_mma_op_t* + rocke_mma_catalog_select_largest_k_indexed(const rocke_mma_catalog_t* cat, + const char* family, + const char* const src_dtypes[3], + const char* dst_dtype, + const rocke_mma_scale_operand_t src_scales[3], + int m, + int n, + int k_max); const rocke_mma_op_t* rocke_mma_catalog_select_largest_k(const rocke_mma_catalog_t* cat, const char* family, const char* a_dtype, @@ -211,8 +269,18 @@ const rocke_mma_op_t* rocke_mma_catalog_select_largest_k(const rocke_mma_catalog /* MmaCatalog.by_op_id. Returns NULL if absent. */ const rocke_mma_op_t* rocke_mma_catalog_by_op_id(const rocke_mma_catalog_t* cat, const char* op_id); -/* MmaCatalog.op_for_shape: the atom with exactly (m, n, k). NULL if absent. - * family NULL => "mma". */ +/* Indexed and compatibility MmaCatalog.op_for_shape variants. An ambiguous + * query raises ROCKE_ERR_VALUE, as does a tie at the largest eligible K in + * select_largest_k. Omitted scales never silently pick one contract. */ +const rocke_mma_op_t* + rocke_mma_catalog_op_for_shape_indexed(const rocke_mma_catalog_t* cat, + const char* family, + const char* const src_dtypes[3], + const char* dst_dtype, + const rocke_mma_scale_operand_t src_scales[3], + int m, + int n, + int k); const rocke_mma_op_t* rocke_mma_catalog_op_for_shape(const rocke_mma_catalog_t* cat, const char* family, const char* a_dtype, @@ -228,18 +296,17 @@ const rocke_mma_op_t* rocke_mma_catalog_op_for_shape(const rocke_mma_catalog_t* * any atom metadata. They mirror the target.py module-level helpers that * rocke.core.ir.IRBuilder.mma consults. */ -/* Accumulator fragment length for op_id -- the c_frag_len projection of - * target.py::_MMA_FRAGMENT_INFO (rocke_arch_mma_c_frag_len(op_id) == - * target._frag_info(op_id).c_frag_len). Returns 0 for an unknown atom (the +/* dst fragment length for op_id -- the dst.frag_len projection of + * target.py::_MMA_FRAGMENT_INFO. Returns 0 for an unknown atom (the * zero-length _FragInfo fallback). */ -int rocke_arch_mma_c_frag_len(const char* op_id); +int rocke_arch_mma_dst_frag_len(const char* op_id); /* Normalised accumulator dtype for op_id, aggregated across every arch's * catalog (mirrors target._op_id_c_dtype()[op_id]). The dtype is invariant * across the arches that list an op_id, so the first catalog hit wins. Returns * NULL for an op_id absent from every catalog. The returned string is an * interned canonical key (e.g. ROCKE_DTYPE_I32 / "fp32"); do not free. */ -const char* rocke_arch_mma_op_id_c_dtype(const char* op_id); +const char* rocke_arch_mma_op_id_dst_dtype(const char* op_id); /* ============================== arch target =========================== */ @@ -272,7 +339,12 @@ const char* rocke_arch_isa_triple(const rocke_arch_target_t* t, char* out, size_ /* ArchTarget.fits_lds. */ bool rocke_arch_fits_lds(const rocke_arch_target_t* t, long bytes_in_use); -/* ArchTarget.supports_dtype_combo. family NULL => "mma". */ +/* Indexed and compatibility ArchTarget.supports_dtype_combo variants. */ +bool rocke_arch_supports_dtype_combo_indexed(const rocke_arch_target_t* t, + const char* const src_dtypes[3], + const char* dst_dtype, + const rocke_mma_scale_operand_t src_scales[3], + const char* family); bool rocke_arch_supports_dtype_combo( const rocke_arch_target_t* t, const char* a, const char* b, const char* c, const char* family); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h index 1fc0f9462403..77f74d564742 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h @@ -63,16 +63,15 @@ typedef rocke_mma_role_t rocke_arch_mma_role_t; /* MmaOp.shape -> (m, n, k). Forwards to rocke_mma_op_shape. */ void rocke_mmaop_shape(const rocke_mmaop_t* op, int* m, int* n, int* k); -/* MmaOp.a_layout() / b_layout() / c_layout() / acc_layout(). Each returns the - * verified lane/slot -> coord map for the role, or NULL when none is registered - * for the op_id (Python raises NotImplementedError; here, when `b` is non-NULL, - * the builder's sticky error is set -- pass b=NULL for a pure lookup). Forwards - * to the canonical rocke_mma_op_*_layout accessors. */ +/* Historical MmaOp.a_layout() / b_layout() / c_layout() projections. C is the + * result/dst map; src2 is available through the canonical indexed + * API. Each returns the verified lane/slot -> coord map for the role, or NULL + * when none is registered for the op_id (Python raises NotImplementedError; + * here, when `b` is non-NULL, the builder's sticky error is set -- pass b=NULL + * for a pure lookup). */ const rocke_arch_layout_map_t* rocke_mmaop_a_layout(const rocke_mmaop_t* op, rocke_ir_builder_t* b); const rocke_arch_layout_map_t* rocke_mmaop_b_layout(const rocke_mmaop_t* op, rocke_ir_builder_t* b); const rocke_arch_layout_map_t* rocke_mmaop_c_layout(const rocke_mmaop_t* op, rocke_ir_builder_t* b); -const rocke_arch_layout_map_t* rocke_mmaop_acc_layout(const rocke_mmaop_t* op, - rocke_ir_builder_t* b); /* LayoutMap.coord(builder, lane, slot) -> (coord0, coord1). Forwards to * rocke_layout_map_coord (validates slot in [0, frag_len); emits arith via b). */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.epilogues.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.epilogues.h index 7f2e7f55f73a..2df8a195df13 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.epilogues.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.epilogues.h @@ -195,7 +195,7 @@ rocke_cshuffle_epilogue_t rocke_cshuffle_epilogue_from_grid(const rocke_mfma_ato int max_store_vec); /* CShuffleEpilogue.from_grid_op(op, grid, max_store_vec=8): WMMA variant. - * Uses op->c_frag_len / op->c_layout() for the LDS scatter in place of the + * Uses op->dst.frag_len / rocke_mma_op_dst_layout(op) for the LDS scatter in place of the * MFMA-specific atom.lane_to_output. */ rocke_cshuffle_epilogue_t rocke_cshuffle_epilogue_from_grid_op(const rocke_mmaop_t* op, const rocke_warp_grid_t* grid, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gemm_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gemm_internal.h index 57c09cff9b96..362a57c2cd57 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gemm_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gemm_internal.h @@ -256,7 +256,7 @@ typedef struct rocke_gemm_build_ctx /* _atom_frag_lengths(op) -> (a,b,c)_frag_len. Pure; fills out-params. */ void rocke_gemm_atom_frag_lengths(const rocke_mmaop_t* op, int* a_frag, int* b_frag, int* c_frag); -/* _emit_zero_acc_op(b, op): zero_vec_f32(op->c_frag_len). Used to build +/* _emit_zero_acc_op(b, op): zero_vec_f32(op->dst.frag_len). Used to build * acc_init. (The MFMA-only _emit_zero_acc / _mfma_atom_widths variants are not * reached by the contract-driven body; kept here for parity with the Python * module surface.) */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h index 092330b16a11..be5c3e83b22e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h @@ -985,7 +985,7 @@ rocke_value_t* rocke_b_vec_concat(rocke_ir_builder_t* b, rocke_value_t* a, rocke /* ----- ISA-named MMA wrappers (thin wrappers over rocke_b_mma; kept for parity * with the legacy Python helpers so emitters can call them by name). All take - * (a, b, c) and return . The scaled MX atom takes the + * (src0, src1, src2) and return . The scaled MX atom takes the * two extra E8M0 scale operands. */ rocke_value_t* rocke_b_mfma_f32_16x16x16_f16(rocke_ir_builder_t* b, rocke_value_t* a, @@ -1053,18 +1053,6 @@ rocke_value_t* rocke_b_mfma_scale_f32_16x16x128_f8f6f4(rocke_ir_builder_t* b, rocke_value_t* c, rocke_value_t* a_scale, rocke_value_t* b_scale); -rocke_value_t* rocke_b_wmma_scale_f32_16x16x128_fp8_fp8(rocke_ir_builder_t* b, - rocke_value_t* a, - rocke_value_t* bb, - rocke_value_t* c, - rocke_value_t* a_scale, - rocke_value_t* b_scale); -rocke_value_t* rocke_b_wmma_scale16_f32_16x16x128_fp8_fp8(rocke_ir_builder_t* b, - rocke_value_t* a, - rocke_value_t* bb, - rocke_value_t* c, - rocke_value_t* a_scale, - rocke_value_t* b_scale); rocke_value_t* rocke_b_wmma_f32_16x16x16_f16(rocke_ir_builder_t* b, rocke_value_t* a, rocke_value_t* bb, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h index 4b18e8331e9f..39f804f813db 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h @@ -8,8 +8,11 @@ #ifndef ROCKE_WMMA_SCALE_INTERNAL_H #define ROCKE_WMMA_SCALE_INTERNAL_H +#include #include +#include "rocke/arch_target.h" +#include "rocke/error.hpp" #include "rocke/ir.h" typedef struct rocke_e8m0_scale_packing @@ -22,39 +25,74 @@ typedef struct rocke_e8m0_scale_packing typedef struct rocke_scaled_wmma_op { const char* op_id; - int matrix_format; + int matrix_formats[2]; + int scale_formats[2]; + int matrix_words[2]; + char declaration_key[160]; + char intrinsic[160]; rocke_e8m0_scale_packing_t scales; } rocke_scaled_wmma_op_t; -static inline const rocke_scaled_wmma_op_t* rocke_gfx1250_scaled_wmma(const char* op_id) +static inline const rocke_mma_op_t* rocke_gfx1250_scaled_wmma(const char* op_id) { - static const rocke_e8m0_scale_packing_t scale = {/*count=*/4, /*block_k=*/32}; - static const rocke_e8m0_scale_packing_t scale16 = {/*count=*/8, /*block_k=*/16}; - static const rocke_scaled_wmma_op_t ops[] = { - {"wmma_scale_f32_16x16x128_fp8_fp8", 0, scale}, - {"wmma_scale_f32_16x16x128_bf8_bf8", 1, scale}, - {"wmma_scale16_f32_16x16x128_fp8_fp8", 0, scale16}, - {"wmma_scale16_f32_16x16x128_bf8_bf8", 1, scale16}, - }; if(!op_id) - { return NULL; - } if(strncmp(op_id, "tile.", 5) == 0) - { op_id += 5; - } - for(size_t i = 0; i < sizeof(ops) / sizeof(ops[0]); ++i) + const rocke_arch_target_t* target = rocke_arch_target_from_gfx("gfx1250"); + const rocke_mma_op_t* atom = rocke_mma_catalog_by_op_id(&target->mma, op_id); + return atom && strcmp(atom->family, "wmma_scaled") == 0 ? atom : NULL; +} + +static inline rocke_scaled_wmma_op_t rocke_scaled_wmma_contract(const rocke_mma_op_t* atom) +{ + rocke_scaled_wmma_op_t spec = {}; + spec.op_id = atom->op_id; + for(int i = 0; i < 2; ++i) { - if(strcmp(op_id, ops[i].op_id) == 0) - { - return &ops[i]; - } + const rocke_mma_src_t* src = &atom->srcs[i]; + if(strcmp(src->dtype, "fp8e4m3") == 0) + spec.matrix_formats[i] = 0; + else if(strcmp(src->dtype, "bf8e5m2") == 0) + spec.matrix_formats[i] = 1; + else + ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA matrix format"); + if(!src->scale_dtype || strcmp(src->scale_dtype, "e8m0") != 0 + || (src->scale_block_size != 16 && src->scale_block_size != 32)) + ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA scale format"); + spec.scale_formats[i] = 0; // E8M0. + spec.matrix_words[i] = src->frag_len; } - return NULL; + if(atom->srcs[0].scale_block_size != atom->srcs[1].scale_block_size || atom->srcs[2].scale_dtype + || strcmp(atom->srcs[2].dtype, "fp32") != 0 || strcmp(atom->dst.dtype, "fp32") != 0 + || atom->m != 16 || atom->n != 16 || atom->k != 128) + ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA backend contract"); + spec.scales.block_k = atom->srcs[0].scale_block_size; + spec.scales.count = atom->k / spec.scales.block_k; + char suffix[96]; + snprintf(suffix, + sizeof(suffix), + "f32.%dx%dx%d.f8f6f4.v%df32.v%di32.v%di32", + atom->m, + atom->n, + atom->k, + atom->dst.frag_len, + spec.matrix_words[0], + spec.matrix_words[1]); + snprintf(spec.intrinsic, + sizeof(spec.intrinsic), + "llvm.amdgcn.wmma.%s.%s", + spec.scales.block_k == 16 ? "scale16" : "scale", + suffix); + snprintf(spec.declaration_key, + sizeof(spec.declaration_key), + "wmma.scale.block%d.gfx1250.%s", + spec.scales.block_k, + suffix); + return spec; } -static inline const rocke_scaled_wmma_op_t* rocke_gfx1250_scaled_wmma_from_op(const rocke_op_t* op) +static inline const rocke_mma_op_t* rocke_gfx1250_scaled_wmma_from_op(const rocke_op_t* op) { const char* op_id = rocke_attr_get_str(&op->attrs, "op_id"); return rocke_gfx1250_scaled_wmma(op_id ? op_id : op->name); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_build_glue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_build_glue.cpp index c4c8a1929394..eebb5653ce27 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_build_glue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_build_glue.cpp @@ -236,9 +236,9 @@ bool rocke_conv_build_ctx_init(rocke_conv_build_ctx_t* ctx, ? NULL : rocke_mfma_atom("f16", spec->warp_tile_m, spec->warp_tile_n, spec->warp_tile_k); - ctx->a_per_lane = ctx->op->a_frag_len; - ctx->b_per_lane = ctx->op->b_frag_len; - ctx->c_per_lane = ctx->op->c_frag_len; + ctx->a_per_lane = ctx->op->srcs[0].frag_len; + ctx->b_per_lane = ctx->op->srcs[1].frag_len; + ctx->c_per_lane = ctx->op->dst.frag_len; /* ---- block tile dims ---- (817) */ ctx->block_m = spec->tile_m; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_epilogue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_epilogue.cpp index b7389eb0b7dc..23d266352638 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_epilogue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_epilogue.cpp @@ -321,7 +321,7 @@ void rocke_conv_emit_direct_epilogue_wmma(rocke_ir_builder_t* b, n_const = rocke_b_const_i32(b, ni * spec->warp_tile_n); atom_n_off = rocke_b_add(b, n_inner, n_const); - for(i = 0; i < op->c_frag_len; ++i) + for(i = 0; i < op->dst.frag_len; ++i) { rocke_value_t* row_off = NULL; rocke_value_t* col_off = NULL; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp index 52f30b6e97e9..51dfe0498ede 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp @@ -1016,7 +1016,7 @@ static void _emit_dgrad_direct_epilogue_wmma(rocke_ir_builder_t* b, = rocke_b_add(b, _bwn, rocke_b_const_i32(b, ni * spec->warp_tile_n)); const rocke_arch_layout_map_t* c_map = rocke_mmaop_c_layout(op, b); - for(int i = 0; i < op->c_frag_len; i++) + for(int i = 0; i < op->dst.frag_len; i++) { rocke_value_t* row_off; rocke_value_t* col_off; @@ -1501,7 +1501,7 @@ static void _emit_dgrad_tilde_direct_epilogue_wmma(rocke_ir_builder_t* b, rocke_b_add(b, block_n_off, warp_n_off), rocke_b_const_i32(b, ni * spec->warp_tile_n)); - for(int i = 0; i < op->c_frag_len; i++) + for(int i = 0; i < op->dst.frag_len; i++) { rocke_value_t* row_off; rocke_value_t* col_off; @@ -1807,9 +1807,9 @@ static rocke_kernel_def_t* = is_wmma ? NULL : rocke_mfma_atom( spec->dtype_a, spec->warp_tile_m, spec->warp_tile_n, spec->warp_tile_k); - int a_per_lane = op->a_frag_len; - int b_per_lane = op->b_frag_len; - int c_per_lane = op->c_frag_len; + int a_per_lane = op->srcs[0].frag_len; + int b_per_lane = op->srcs[1].frag_len; + int c_per_lane = op->dst.frag_len; // ---- 1D grid: block_id_x covers all sub-GEMMs' tiles ---- rocke_value_t* flat_block_id = rocke_b_block_id_x(b); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_wgrad.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_wgrad.cpp index b56df3714918..8e54b541affe 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_wgrad.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_wgrad.cpp @@ -1605,7 +1605,7 @@ static void wgrad_emit_direct_epilogue(rocke_ir_builder_t* b, rocke_value_t* n_const = rocke_b_const_i32(b, ni * spec->warp_tile_n); rocke_value_t* atom_n_off = rocke_b_add(b, n_inner, n_const); - for(int i = 0; i < op->c_frag_len; ++i) + for(int i = 0; i < op->dst.frag_len; ++i) { rocke_value_t* row_off = NULL; rocke_value_t* col_off = NULL; @@ -1801,9 +1801,9 @@ static bool wgrad_build_ctx_init(rocke_conv_build_ctx_t* ctx, spec->warp_tile_n, spec->warp_tile_k); } - ctx->a_per_lane = ctx->op->a_frag_len; - ctx->b_per_lane = ctx->op->b_frag_len; - ctx->c_per_lane = ctx->op->c_frag_len; + ctx->a_per_lane = ctx->op->srcs[0].frag_len; + ctx->b_per_lane = ctx->op->srcs[1].frag_len; + ctx->c_per_lane = ctx->op->dst.frag_len; /* Block tile dims */ ctx->block_m = spec->tile_m; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/deep_fused_conv_pool_conv1_gemm_and_lds_staging.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/deep_fused_conv_pool_conv1_gemm_and_lds_staging.cpp index 58ca7fbc758c..c5c7448ebe46 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/deep_fused_conv_pool_conv1_gemm_and_lds_staging.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/deep_fused_conv_pool_conv1_gemm_and_lds_staging.cpp @@ -132,7 +132,7 @@ static rocke_value_t* dfcp_load_smem_frag_contiguous_f16(rocke_ir_builder_t* b, * _stage_accumulators_to_cshuffle_lds (py 388-456) * * Publish MMA accumulators to a row-major [tile_m, tile_n] LDS tile, fully - * op-driven via op->c_frag_len + op->c_layout()->coord + the per-warp tile + * op-driven via op->dst.frag_len + rocke_mma_op_dst_layout(op)->coord + the per-warp tile * geometry. sync=false defers the trailing barrier so the caller can merge it * with the disjoint W1 producer barrier. * ================================================================== */ @@ -160,7 +160,7 @@ rocke_value_t* rocke_dfcp_stage_accumulators_to_cshuffle_lds(rocke_ir_builder_t* return NULL; } - c_frag_len = op->c_frag_len; + c_frag_len = op->dst.frag_len; mfmas_m = rocke_warp_grid_mfmas_per_warp_m(b, grid); mfmas_n = rocke_warp_grid_mfmas_per_warp_n(b, grid); if(!rocke_ir_builder_ok(b)) @@ -360,7 +360,7 @@ rocke_value_t* rocke_dfcp_load_conv1_weights_to_lds(rocke_ir_builder_t* b, * _emit_conv1_1x1 (py 816-910) * * Compute conv1 as a 1x1 GEMM over the staged conv0 activations, op-driven: - * operand lane->(mn,k) decomposition from op->a_layout()/b_layout()->coord, + * operand lane->(mn,k) decomposition from rocke_mma_op_src_layout(op, 0/1)->coord, * fragment widths op->{a,b}_frag_len, matmul via the target-neutral mma. The * K-tail mask is handled by load_smem_frag_contiguous_f16. defer_epilogue=true * returns the raw fp32 accumulators; else applies spec.conv1_epilogue. @@ -428,8 +428,8 @@ rocke_status_t rocke_dfcp_emit_conv1_1x1(rocke_ir_builder_t* b, return rocke_ir_builder_status(b); } - a_frag = op->a_frag_len; - b_frag = op->b_frag_len; + a_frag = op->srcs[0].frag_len; + b_frag = op->srcs[1].frag_len; mfmas_m = rocke_warp_grid_mfmas_per_warp_m(b, grid); mfmas_n = rocke_warp_grid_mfmas_per_warp_n(b, grid); if(!rocke_ir_builder_ok(b)) @@ -476,7 +476,7 @@ rocke_status_t rocke_dfcp_emit_conv1_1x1(rocke_ir_builder_t* b, int idx; for(idx = 0; idx < num_accs; ++idx) { - accs[idx] = rocke_b_zero_vec_f32(b, op->c_frag_len); + accs[idx] = rocke_b_zero_vec_f32(b, op->srcs[2].frag_len); } } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_build_compute.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_build_compute.cpp index b99998146b46..d858f214cb08 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_build_compute.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_build_compute.cpp @@ -30,7 +30,7 @@ * atom step inside this K-tile. On WMMA (RDNA) this delegates to the fully * contract-driven _emit_wmma_phase. On MFMA (CDNA) it keeps the byte-identical * arch-formula fragment load; the matmul + accumulator length are - * contract-driven (rocke_gemm_emit_mma / op->c_frag_len). + * contract-driven (rocke_gemm_emit_mma / op->dst.frag_len). * * lds_parity is split (per the internal-header convention) into a compile-time * int (parity_imm) OR a runtime Value (parity_v != NULL == Python isinstance diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_mma.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_mma.cpp index f057d592a199..f403ec3fe83a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_mma.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_mma.cpp @@ -135,11 +135,11 @@ rocke_value_t* rocke_gemm_emit_zero_acc(rocke_ir_builder_t* b, void rocke_gemm_atom_frag_lengths(const rocke_mmaop_t* op, int* a_frag, int* b_frag, int* c_frag) { if(a_frag) - *a_frag = op->a_frag_len; + *a_frag = op->srcs[0].frag_len; if(b_frag) - *b_frag = op->b_frag_len; + *b_frag = op->srcs[1].frag_len; if(c_frag) - *c_frag = op->c_frag_len; + *c_frag = op->dst.frag_len; } /* ====================================================================== * @@ -163,7 +163,7 @@ rocke_value_t* rocke_gemm_emit_mma(rocke_ir_builder_t* b, * ====================================================================== */ rocke_value_t* rocke_gemm_emit_zero_acc_op(rocke_ir_builder_t* b, const rocke_mmaop_t* op) { - return rocke_b_zero_vec_f32(b, op->c_frag_len); + return rocke_b_zero_vec_f32(b, op->srcs[2].frag_len); } /* ====================================================================== * diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_scatter_maxpool_finalization_phases.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_scatter_maxpool_finalization_phases.cpp index 85672b4c693c..0b9afe6484e5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_scatter_maxpool_finalization_phases.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_scatter_maxpool_finalization_phases.cpp @@ -54,7 +54,7 @@ void rocke_gfx1151_dfcp_scatter_codes_to_lds(rocke_gfx1151_dfcp_build_ctx_t* ctx const rocke_layout_map_t* c_map = rocke_mma_op_c_layout(op, b); rocke_value_t* warp_m_off = rocke_warp_grid_warp_m_off(b, grid); rocke_value_t* warp_n_off = rocke_warp_grid_warp_n_off(b, grid); - int frag_len = op->c_frag_len; + int frag_len = op->dst.frag_len; int flat = 0; int mi, ni, i; (void)num_accs; @@ -105,7 +105,7 @@ void rocke_gfx1151_dfcp_scatter_codes_to_i8_lds(rocke_gfx1151_dfcp_build_ctx_t* const rocke_layout_map_t* c_map = rocke_mma_op_c_layout(op, b); rocke_value_t* warp_m_off = rocke_warp_grid_warp_m_off(b, grid); rocke_value_t* warp_n_off = rocke_warp_grid_warp_n_off(b, grid); - int frag_len = op->c_frag_len; + int frag_len = op->dst.frag_len; int flat = 0; int mi, ni, i; (void)num_accs; @@ -156,7 +156,7 @@ void rocke_gfx1151_dfcp_scatter_vec_codes_to_i8_lds(rocke_gfx1151_dfcp_build_ctx const rocke_layout_map_t* c_map = rocke_mma_op_c_layout(op, b); rocke_value_t* warp_m_off = rocke_warp_grid_warp_m_off(b, grid); rocke_value_t* warp_n_off = rocke_warp_grid_warp_n_off(b, grid); - int frag_len = op->c_frag_len; + int frag_len = op->dst.frag_len; int flat = 0; int mi, ni, i; (void)num_accs; @@ -208,7 +208,7 @@ void rocke_gfx1151_dfcp_scatter_packed_i4_codes_to_lds(rocke_gfx1151_dfcp_build_ const rocke_layout_map_t* c_map = rocke_mma_op_c_layout(op, b); rocke_value_t* warp_m_off = rocke_warp_grid_warp_m_off(b, grid); rocke_value_t* warp_n_off = rocke_warp_grid_warp_n_off(b, grid); - int frag_len = op->c_frag_len; + int frag_len = op->dst.frag_len; int flat = 0; int mi, ni, i; rocke_value_t* c0 = rocke_b_const_i32(b, 0); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_wmma_gemm_fragment_loaders_register_handoff.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_wmma_gemm_fragment_loaders_register_handoff.cpp index bcc283f605fb..13e71fef557b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_wmma_gemm_fragment_loaders_register_handoff.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_wmma_gemm_fragment_loaders_register_handoff.cpp @@ -372,12 +372,12 @@ void rocke_gfx1151_dfcp_wmma_gemm_from_lds(rocke_gfx1151_dfcp_build_ctx_t* ctx, dfcp_layout_coord0(b, b_map, grid->lane, &b_k, &b_col); rocke_value_t* warp_m_off = rocke_warp_grid_warp_m_off(b, grid); rocke_value_t* warp_n_off = rocke_warp_grid_warp_n_off(b, grid); - int ds_per_frag = (op->a_frag_len + 7) / 8; + int ds_per_frag = (op->srcs[0].frag_len + 7) / 8; int n_ds = ds_per_frag * (mfmas_m + mfmas_n); int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec_f32(b, op->c_frag_len); + out_accs[i] = rocke_b_zero_vec_f32(b, op->srcs[2].frag_len); for(int kk = 0; kk < k_atoms; ++kk) { @@ -389,14 +389,14 @@ void rocke_gfx1151_dfcp_wmma_gemm_from_lds(rocke_gfx1151_dfcp_build_ctx_t* ctx, rocke_value_t* atom_row = rocke_b_add(b, warp_m_off, rocke_b_const_i32(b, mi * DFCP_WMMA)); a_rows[mi] = rocke_conv_emit_frag_smem_load( - b, a_smem, a_row, a_k, atom_row, k_tile_base, op->a_frag_len); + b, a_smem, a_row, a_k, atom_row, k_tile_base, op->srcs[0].frag_len); } for(int ni = 0; ni < mfmas_n; ++ni) { rocke_value_t* atom_row = rocke_b_add(b, warp_n_off, rocke_b_const_i32(b, ni * DFCP_WMMA)); b_cols[ni] = rocke_conv_emit_frag_smem_load( - b, b_smem, b_col, b_k, atom_row, k_tile_base, op->b_frag_len); + b, b_smem, b_col, b_k, atom_row, k_tile_base, op->srcs[1].frag_len); } int flat = 0; for(int mi = 0; mi < mfmas_m; ++mi) @@ -436,7 +436,7 @@ void rocke_gfx1151_dfcp_wmma_gemm_from_lds_int(rocke_gfx1151_dfcp_build_ctx_t* c int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->c_frag_len); + out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->srcs[2].frag_len); for(int kk = 0; kk < k_atoms; ++kk) { @@ -490,11 +490,11 @@ void rocke_gfx1151_dfcp_wmma_gemm_conv0_direct(rocke_gfx1151_dfcp_build_ctx_t* c dfcp_layout_coord0(b, b_map, grid->lane, &b_k, &b_col); rocke_value_t* warp_m_off = rocke_warp_grid_warp_m_off(b, grid); rocke_value_t* warp_n_off = rocke_warp_grid_warp_n_off(b, grid); - int n_ds = op->a_frag_len * mfmas_m + ((op->b_frag_len + 7) / 8) * mfmas_n; + int n_ds = op->srcs[0].frag_len * mfmas_m + ((op->srcs[1].frag_len + 7) / 8) * mfmas_n; int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec_f32(b, op->c_frag_len); + out_accs[i] = rocke_b_zero_vec_f32(b, op->srcs[2].frag_len); for(int kk = 0; kk < k_atoms; ++kk) { @@ -508,14 +508,14 @@ void rocke_gfx1151_dfcp_wmma_gemm_conv0_direct(rocke_gfx1151_dfcp_build_ctx_t* c = rocke_b_add(b, warp_m_off, rocke_b_const_i32(b, mi * DFCP_WMMA)); rocke_value_t* m_row = rocke_b_add(b, atom_row, a_row); a_rows[mi] = rocke_gfx1151_dfcp_load_conv0_a_frag_from_footprint( - ctx, inp_smem, m_row, k_base, op->a_frag_len); + ctx, inp_smem, m_row, k_base, op->srcs[0].frag_len); } for(int ni = 0; ni < mfmas_n; ++ni) { rocke_value_t* atom_row = rocke_b_add(b, warp_n_off, rocke_b_const_i32(b, ni * DFCP_WMMA)); b_cols[ni] = rocke_conv_emit_frag_smem_load( - b, w0_smem, b_col, b_k, atom_row, k_tile_base, op->b_frag_len); + b, w0_smem, b_col, b_k, atom_row, k_tile_base, op->srcs[1].frag_len); } int flat = 0; for(int mi = 0; mi < mfmas_m; ++mi) @@ -555,7 +555,7 @@ void rocke_gfx1151_dfcp_wmma_gemm_conv0_direct_int(rocke_gfx1151_dfcp_build_ctx_ int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->c_frag_len); + out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->srcs[2].frag_len); for(int kk = 0; kk < k_atoms; ++kk) { @@ -626,7 +626,7 @@ void rocke_gfx1151_dfcp_wmma_gemm_conv1_i4_from_lds(rocke_gfx1151_dfcp_build_ctx int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->c_frag_len); + out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->srcs[2].frag_len); const void* per_step = sched_fuse ? NULL : policy; @@ -734,7 +734,7 @@ void rocke_gfx1151_dfcp_wmma_gemm_conv1_i4_packed_from_lds(rocke_gfx1151_dfcp_bu int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->c_frag_len); + out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->srcs[2].frag_len); for(int kk = 0; kk < k_atoms; ++kk) { @@ -796,7 +796,7 @@ void rocke_gfx1151_dfcp_wmma_gemm_conv1_i4_from_regs(rocke_gfx1151_dfcp_build_ct int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->c_frag_len); + out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->srcs[2].frag_len); const void* per_step = sched_fuse ? NULL : policy; @@ -882,7 +882,7 @@ void rocke_gfx1151_dfcp_wmma_gemm_conv1_i8_from_regs(rocke_gfx1151_dfcp_build_ct int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->c_frag_len); + out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->srcs[2].frag_len); const void* per_step = sched_fuse ? NULL : policy; @@ -968,11 +968,11 @@ void rocke_gfx1151_dfcp_fuse_c0_to_conv1_a_regs(rocke_gfx1151_dfcp_build_ctx_t* rocke_value_t* acc = accs0[mi * mfmas_n + kk]; /* codes = [code_fn(b.vec_extract(acc, s)) for s in range(op0.c_frag_len)] */ rocke_value_t* codes[ROCKE_GFX1151_DFCP_MAX_ACCS]; - for(int s = 0; s < op0->c_frag_len; ++s) + for(int s = 0; s < op0->dst.frag_len; ++s) codes[s] = rocke_gfx1151_dfcp_apply_code_fn( ctx, code_fn, rocke_b_vec_extract(b, acc, s)); /* words = b.bitcast(b.vec_pack(codes, I8), VectorType(I32, 2)) */ - rocke_value_t* packed_i8 = rocke_b_vec_pack(b, codes, op0->c_frag_len, rocke_i8()); + rocke_value_t* packed_i8 = rocke_b_vec_pack(b, codes, op0->dst.frag_len, rocke_i8()); rocke_value_t* words = rocke_b_bitcast(b, packed_i8, rocke_vector_type(b, rocke_i32(), 2)); rocke_value_t* lo = rocke_b_vec_extract(b, words, 0); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md index 197f7782e90b..bbc313f159f7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md @@ -31,6 +31,46 @@ Matrix-operation choices start from the exact gfx target. Resolve and validate the operation's wave and layout contract. The exact catalog is listed in [`kernel_taxonomy.md`](./kernel_taxonomy.md). +MMA metadata uses `srcs[0]`, `srcs[1]`, `srcs[2]`, and `dst`: the first two +sources are multiplicands, the third is the accumulator input, and `dst` is +the result. A source's optional `MmaScaleOperand(dtype, block_size)` describes +its scale values independently of the matrix dtype and packed register type. + +For scaled operations, query the complete contract and pass the selected atom +to `IRBuilder.mma`: + +```python +from rocke.core.arch import ArchTarget, MmaScaleOperand + +scale = MmaScaleOperand("e8m0", 32) +atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( + family="wmma_scaled", + src_dtypes=("fp8", "fp8", "fp32"), + dst_dtype="fp32", + src_scales=(scale, scale, None), + m=16, n=16, k=128, +) +assert atom is not None +# result = builder.mma(atom, src0, src1, src2, scale0, scale1) +``` + +Omitting `src_scales` leaves scales unconstrained. Passing a three-entry tuple +matches every source exactly; `None` in that tuple means unscaled. Enumeration +and existence queries may match several records, while `op_for_shape` and +`select_largest_k` reject an ambiguous selection. The C indexed query functions +use the same rules: a NULL scale array is unconstrained, and `{NULL, 0}` requests +an unscaled source. Scale value formats are `e8m0`, `e4m3`, and `e5m3`, with +`fp8e4m3` accepted as an alias for `e4m3`; actual target support comes from the +catalog. + +Scaled-WMMA IDs have the form +`wmma.scaled..src0___b.src1___b.src2_.dst_`. +They describe a semantic contract independently of LLVM intrinsic names and +packed carriers. Existing `wmma_scale*_f32_*` IDs and their dedicated builder +wrappers are retired; serialized IR using those IDs must be regenerated. +Use `tile.mma` with a resolved catalog atom. Other MMA operation IDs retain +their existing spelling. + Kernel definitions in [`/python/rocke/instances/`](../../python/rocke/instances/) and [`/kernels/`](../../../library/kernels/) follow the same diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py index 9b0194a1a566..42e940009075 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py @@ -13,7 +13,10 @@ LayoutMap, MemoryCapabilities, MmaCatalog, + MmaDst, MmaOp, + MmaScaleOperand, + MmaSrc, ResourceLimits, arch_from_isa, base_arch_from_target_id, @@ -28,7 +31,10 @@ "LayoutMap", "MemoryCapabilities", "MmaCatalog", + "MmaDst", "MmaOp", + "MmaScaleOperand", + "MmaSrc", "ResourceLimits", "arch_from_isa", "base_arch_from_target_id", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json index 4269fb380b17..7170e799ad49 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json @@ -1,6 +1,6 @@ { - "version": 1, - "_comment": "ROCKE-owned architecture SSOT. Hardware facts only (no pipeline/scheduler vocabulary). MFMA atom sets are hardware-verified on real hardware (see dsl_docs/architecture/multi_arch_data_layout.md). Each MMA row: {family,a,b,c,m,n,k,op_id}; op_id is the opaque backend handle (matches IRBuilder.mfma_* today).", + "version": 2, + "_comment": "ROCKE-owned architecture SSOT. Hardware facts only (no pipeline/scheduler vocabulary). MFMA atom sets are hardware-verified on real hardware (see dsl_docs/architecture/multi_arch_data_layout.md). Each MMA row has three matrix srcs and one dst; a src may optionally carry a scale descriptor with dtype (scale value format: e8m0/e4m3/e5m3; fp8e4m3 aliases e4m3) and block_size (16 or 32 source elements along K per scale). op_id is the opaque backend handle (matches IRBuilder.mfma_* today).", "arches": { "gfx90a": { "_comment": "CDNA2 (gfx90a). Wave64, 64 KB LDS. No fp8/bf8 native MFMA (those arrived with gfx940/gfx942). No async_lds. bf16 32x32x8 is hardware-verified on gfx90a series. Hardware-verified MMA set matches AMD ISA manual v3 (GFX9 MFMA section).", @@ -21,10 +21,10 @@ "sgprs": 102 }, "mma": [ - {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_f16"}, - {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_f16"}, - {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_bf16"}, - {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_bf16"} + {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_f16"}, + {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_f16"}, + {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_bf16"}, + {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_bf16"} ] }, "gfx942": { @@ -45,16 +45,16 @@ "sgprs": 102 }, "mma": [ - {"family": "mma", "a": "fp32", "b": "fp32", "c": "fp32", "m": 16, "n": 16, "k": 4, "op_id": "mfma_f32_16x16x4_f32"}, - {"family": "mma", "a": "fp32", "b": "fp32", "c": "fp32", "m": 32, "n": 32, "k": 2, "op_id": "mfma_f32_32x32x2_f32"}, - {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_f16"}, - {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_f16"}, - {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_bf16"}, - {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_bf16"}, - {"family": "mma", "a": "fp8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_fp8"}, - {"family": "mma", "a": "fp8", "b": "fp8", "c": "fp32", "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_fp8"}, - {"family": "mma", "a": "bf8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_bf8"}, - {"family": "mma", "a": "bf8", "b": "bf8", "c": "fp32", "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_bf8"} + {"family": "mma", "srcs": [{"dtype": "fp32"}, {"dtype": "fp32"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 4, "op_id": "mfma_f32_16x16x4_f32"}, + {"family": "mma", "srcs": [{"dtype": "fp32"}, {"dtype": "fp32"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 2, "op_id": "mfma_f32_32x32x2_f32"}, + {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_f16"}, + {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_f16"}, + {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_bf16"}, + {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_bf16"}, + {"family": "mma", "srcs": [{"dtype": "fp8"}, {"dtype": "fp8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_fp8"}, + {"family": "mma", "srcs": [{"dtype": "fp8"}, {"dtype": "fp8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_fp8"}, + {"family": "mma", "srcs": [{"dtype": "bf8"}, {"dtype": "bf8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_bf8"}, + {"family": "mma", "srcs": [{"dtype": "bf8"}, {"dtype": "bf8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_bf8"} ] }, "gfx950": { @@ -75,22 +75,22 @@ "sgprs": 102 }, "mma": [ - {"family": "mma", "a": "fp32", "b": "fp32", "c": "fp32", "m": 16, "n": 16, "k": 4, "op_id": "mfma_f32_16x16x4_f32"}, - {"family": "mma", "a": "fp32", "b": "fp32", "c": "fp32", "m": 32, "n": 32, "k": 2, "op_id": "mfma_f32_32x32x2_f32"}, - {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_f16"}, - {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_f16"}, - {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_f16"}, - {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_f16"}, - {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_bf16"}, - {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_bf16"}, - {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_bf16"}, - {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_bf16"}, - {"family": "mma", "a": "fp8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_fp8"}, - {"family": "mma", "a": "fp8", "b": "fp8", "c": "fp32", "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_fp8"}, - {"family": "mma", "a": "bf8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_bf8"}, - {"family": "mma", "a": "bf8", "b": "bf8", "c": "fp32", "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_bf8"}, - {"family": "mma", "a": "fp4", "b": "fp4", "c": "fp32", "m": 16, "n": 16, "k": 128, "op_id": "mfma_f32_16x16x128_fp4"}, - {"family": "mma", "a": "fp6", "b": "fp6", "c": "fp32", "m": 16, "n": 16, "k": 96, "op_id": "mfma_f32_16x16x96_fp6"} + {"family": "mma", "srcs": [{"dtype": "fp32"}, {"dtype": "fp32"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 4, "op_id": "mfma_f32_16x16x4_f32"}, + {"family": "mma", "srcs": [{"dtype": "fp32"}, {"dtype": "fp32"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 2, "op_id": "mfma_f32_32x32x2_f32"}, + {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_f16"}, + {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_f16"}, + {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_f16"}, + {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_f16"}, + {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_bf16"}, + {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_bf16"}, + {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_bf16"}, + {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_bf16"}, + {"family": "mma", "srcs": [{"dtype": "fp8"}, {"dtype": "fp8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_fp8"}, + {"family": "mma", "srcs": [{"dtype": "fp8"}, {"dtype": "fp8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_fp8"}, + {"family": "mma", "srcs": [{"dtype": "bf8"}, {"dtype": "bf8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_bf8"}, + {"family": "mma", "srcs": [{"dtype": "bf8"}, {"dtype": "bf8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_bf8"}, + {"family": "mma", "srcs": [{"dtype": "fp4"}, {"dtype": "fp4"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 128, "op_id": "mfma_f32_16x16x128_fp4"}, + {"family": "mma", "srcs": [{"dtype": "fp6"}, {"dtype": "fp6"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 96, "op_id": "mfma_f32_16x16x96_fp6"} ] }, "gfx1151": { @@ -112,10 +112,10 @@ "sgprs": 106 }, "mma": [ - {"family": "wmma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_f16"}, - {"family": "wmma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_bf16"}, - {"family": "wmma", "a": "iu8", "b": "iu8", "c": "i32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu8"}, - {"family": "wmma", "a": "iu4", "b": "iu4", "c": "i32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu4"} + {"family": "wmma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_f16"}, + {"family": "wmma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_bf16"}, + {"family": "wmma", "srcs": [{"dtype": "iu8"}, {"dtype": "iu8"}, {"dtype": "i32"}], "dst": {"dtype": "i32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu8"}, + {"family": "wmma", "srcs": [{"dtype": "iu4"}, {"dtype": "iu4"}, {"dtype": "i32"}], "dst": {"dtype": "i32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu4"} ] }, "gfx1201": { @@ -137,12 +137,12 @@ "sgprs": 106 }, "mma": [ - {"family": "wmma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_gfx12_f32_16x16x16_f16"}, - {"family": "wmma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_gfx12_f32_16x16x16_bf16"} + {"family": "wmma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_gfx12_f32_16x16x16_f16"}, + {"family": "wmma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_gfx12_f32_16x16x16_bf16"} ] }, "gfx1250": { - "_comment": "gfx1250-class CDNA multi-chip device on the GFX12 programming model (wave32, WMMA, no MFMA). Primary fp16/bf16 WMMA atom is 16x16x32 (NOT gfx1201's 16x16x16). ABI empirically confirmed on ROCm 7.13 clang 23 (gfx1250 device target): __builtin_amdgcn_wmma_f32_16x16x32_f16 -> llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1,<16 x half>,i1,<16 x half>,i16,<8 x float>,i1,i1); bf16 uses v16bf16 (<16 x bfloat>) directly, not the gfx11 <16 x i16> bitcast. Datalayout/triple byte-identical to gfx950/gfx1201. This catalog ships validated fp16/bf16 and fp8/bf8 WMMA atoms plus the FP8xFP8 SCALE/SCALE16 prerequisite slice; FP4xFP4 SCALE/SCALE16 accepts prepacked E2M1 bytes; FP6/BF6 forms remain omitted. Lane maps are a HYPOTHESIS verified by examples/gfx1250/wmma_probe.py. MEMORY: gfx1250 HAS async global->LDS DMA (its own GFX12 family: global_load_async_to_lds_b128 + s_wait_asynccnt -- NOT the gfx9 buffer_load_lds/global_load_lds, which do not select), so has_async_global_lds=true (GPU-validated via the DTLA path). It also HAS a hardware transpose-LDS read, ds_load_tr16_b128 (GPU-validated via compute_pv_dstr). The has_async_lds and has_ds_read_tr flags stay false on purpose: in this catalog they denote the gfx9/gfx950 ds_read_*_tr_* / async-LDS *ABI* used to admit the gfx950 MFMA tiled builder (attention_arch.py), which gfx1250 does NOT have -- its async/transpose use distinct GFX12 opcodes gated directly in the ISA backend, not via these MFMA-pipeline bits. has_tdm=false: TDM not verified on this part.", + "_comment": "gfx1250-class CDNA multi-chip device on the GFX12 programming model (wave32, WMMA, no MFMA). Primary fp16/bf16 WMMA atom is 16x16x32 (NOT gfx1201's 16x16x16). ABI empirically confirmed on ROCm 7.13 clang 23 (gfx1250 device target): __builtin_amdgcn_wmma_f32_16x16x32_f16 -> llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1,<16 x half>,i1,<16 x half>,i16,<8 x float>,i1,i1); bf16 uses v16bf16 (<16 x bfloat>) directly, not the gfx11 <16 x i16> bitcast. Datalayout/triple byte-identical to gfx950/gfx1201. The MMA entries below enumerate supported operand formats and shapes. Dtype-name recognition alone does not imply an available atom. Lane maps are a HYPOTHESIS verified by examples/gfx1250/wmma_probe.py. MEMORY: gfx1250 HAS async global->LDS DMA (its own GFX12 family: global_load_async_to_lds_b128 + s_wait_asynccnt -- NOT the gfx9 buffer_load_lds/global_load_lds, which do not select), so has_async_global_lds=true (GPU-validated via the DTLA path). It also HAS a hardware transpose-LDS read, ds_load_tr16_b128 (GPU-validated via compute_pv_dstr). The has_async_lds and has_ds_read_tr flags stay false on purpose: in this catalog they denote the gfx9/gfx950 ds_read_*_tr_* / async-LDS *ABI* used to admit the gfx950 MFMA tiled builder (attention_arch.py), which gfx1250 does NOT have -- its async/transpose use distinct GFX12 opcodes gated directly in the ISA backend, not via these MFMA-pipeline bits. has_tdm=false: TDM not verified on this part.", "family": "cdna", "target_family": "gfx12_cdna", "matrix_path": "wmma", @@ -170,17 +170,17 @@ "sgprs": 106 }, "mma": [ - {"family": "wmma", "a": "fp32", "b": "fp32", "c": "fp32", "m": 16, "n": 16, "k": 4, "op_id": "wmma_gfx1250_f32_16x16x4_f32"}, - {"family": "wmma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "wmma_gfx1250_f32_16x16x32_f16"}, - {"family": "wmma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "wmma_gfx1250_f32_16x16x32_bf16"}, - {"family": "wmma", "a": "fp8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_fp8_fp8"}, - {"family": "wmma", "a": "fp8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_fp8_bf8"}, - {"family": "wmma", "a": "bf8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_fp8"}, - {"family": "wmma", "a": "bf8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_bf8"}, - {"family": "wmma_scale", "a": "fp8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 128, "op_id": "wmma_scale_f32_16x16x128_fp8_fp8"}, - {"family": "wmma_scale", "a": "bf8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 128, "op_id": "wmma_scale_f32_16x16x128_bf8_bf8"}, - {"family": "wmma_scale16", "a": "fp8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 128, "op_id": "wmma_scale16_f32_16x16x128_fp8_fp8"}, - {"family": "wmma_scale16", "a": "bf8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 128, "op_id": "wmma_scale16_f32_16x16x128_bf8_bf8"} + {"family": "wmma", "srcs": [{"dtype": "fp32"}, {"dtype": "fp32"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 4, "op_id": "wmma_gfx1250_f32_16x16x4_f32"}, + {"family": "wmma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "wmma_gfx1250_f32_16x16x32_f16"}, + {"family": "wmma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "wmma_gfx1250_f32_16x16x32_bf16"}, + {"family": "wmma", "srcs": [{"dtype": "fp8"}, {"dtype": "fp8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_fp8_fp8"}, + {"family": "wmma", "srcs": [{"dtype": "fp8"}, {"dtype": "bf8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_fp8_bf8"}, + {"family": "wmma", "srcs": [{"dtype": "bf8"}, {"dtype": "fp8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_fp8"}, + {"family": "wmma", "srcs": [{"dtype": "bf8"}, {"dtype": "bf8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_bf8"}, + {"family": "wmma_scaled", "srcs": [{"dtype": "fp8", "scale": {"dtype": "e8m0", "block_size": 32}}, {"dtype": "fp8", "scale": {"dtype": "e8m0", "block_size": 32}}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32"}, + {"family": "wmma_scaled", "srcs": [{"dtype": "bf8", "scale": {"dtype": "e8m0", "block_size": 32}}, {"dtype": "bf8", "scale": {"dtype": "e8m0", "block_size": 32}}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32"}, + {"family": "wmma_scaled", "srcs": [{"dtype": "fp8", "scale": {"dtype": "e8m0", "block_size": 16}}, {"dtype": "fp8", "scale": {"dtype": "e8m0", "block_size": 16}}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32"}, + {"family": "wmma_scaled", "srcs": [{"dtype": "bf8", "scale": {"dtype": "e8m0", "block_size": 16}}, {"dtype": "bf8", "scale": {"dtype": "e8m0", "block_size": 16}}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32"} ] }, "gfx11-generic": { @@ -202,10 +202,10 @@ "sgprs": 106 }, "mma": [ - {"family": "wmma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_f16"}, - {"family": "wmma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_bf16"}, - {"family": "wmma", "a": "iu8", "b": "iu8", "c": "i32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu8"}, - {"family": "wmma", "a": "iu4", "b": "iu4", "c": "i32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu4"} + {"family": "wmma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_f16"}, + {"family": "wmma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_bf16"}, + {"family": "wmma", "srcs": [{"dtype": "iu8"}, {"dtype": "iu8"}, {"dtype": "i32"}], "dst": {"dtype": "i32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu8"}, + {"family": "wmma", "srcs": [{"dtype": "iu4"}, {"dtype": "iu4"}, {"dtype": "i32"}], "dst": {"dtype": "i32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu4"} ] } } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py index d209de876a80..044c5914d0d1 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py @@ -32,12 +32,12 @@ # A callable that, given an :class:`~rocke.core.ir.IRBuilder`, a runtime lane # ``Value`` (0..wave_size-1) and a compile-time fragment slot index, emits the # index arithmetic for the two tile coordinates of that fragment element and -# returns them as a ``(Value, Value)`` pair. The coordinate *meaning* depends on -# the operand role (see :class:`LayoutMap.role`): +# returns them as a ``(Value, Value)`` pair. The coordinate meaning depends on +# the machine operand position (see :class:`LayoutMap.role`): # -# * accumulator (C/D) -> ``(row, col)`` in the atom's M x N output tile -# * A operand -> ``(row, k)`` in the atom's M x K input tile -# * B operand -> ``(k, col)`` in the atom's K x N input tile +# * src0 -> ``(row, k)`` in the atom's M x K input tile +# * src1 -> ``(k, col)`` in the atom's K x N input tile +# * src2/dst -> ``(row, col)`` in the atom's M x N tile # # The map is *pure* with respect to the IR: it only emits ``arith.*`` ops via the # builder, so the same map drives both fragment loads (compute the source @@ -60,9 +60,9 @@ class LayoutMap: Attributes ---------- role - ``"acc"`` (accumulator C/D, coords are ``(row, col)``), ``"a"`` (A - operand, coords are ``(row, k)``) or ``"b"`` (B operand, coords are - ``(k, col)``). + ``"src0"``, ``"src1"``, ``"src2"``, or ``"dst"``. ``src0`` uses + ``(row, k)``, ``src1`` uses ``(k, col)``, and ``src2``/``dst`` use + ``(row, col)`` coordinates. frag_len Number of fragment slots per lane for this role (the per-lane vector length: e.g. 4 for an MFMA 16x16x16 accumulator, 8 for the WMMA @@ -98,72 +98,167 @@ def coord(self, builder: Any, lane: Any, slot: int) -> Tuple[Any, Any]: return self.fn(builder, lane, slot) +@dataclass(frozen=True) +class MmaScaleOperand: + """Scale value format and granularity for one matrix source. + + ``dtype`` is one of ``e8m0``, ``e4m3``, or ``e5m3``; ``fp8e4m3`` is an + alias for ``e4m3``. It describes the scale values independently of the + source dtype and the backend's packed register carrier. ``block_size`` + is the number of source elements along K sharing one scale value. Only + 16 and 32 are valid: SCALE and SCALE16 use 32 and 16, respectively. + """ + + dtype: str + block_size: int + + def __post_init__(self) -> None: + if self.dtype == "fp8e4m3": + object.__setattr__(self, "dtype", "e4m3") + if self.dtype not in ("e8m0", "e4m3", "e5m3"): + raise ValueError("MMA scale dtype must be e8m0, e4m3, or e5m3") + if type(self.block_size) is not int or self.block_size not in (16, 32): + raise ValueError( + "MMA scale block_size must be an integer equal to 16 or 32" + ) + + +@dataclass(frozen=True) +class MmaDst: + """Metadata for the matrix ``dst`` operand.""" + + dtype: str + frag_len: int = 0 + layout: Optional[LayoutMap] = field(default=None, repr=False, compare=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "dtype", normalize_dtype(self.dtype)) + + def require_layout(self, name: str, op: "MmaOp") -> LayoutMap: + if self.layout is None: + raise NotImplementedError( + f"no verified {name!r} layout map for MMA op_id {op.op_id!r} " + f"({op.m}x{op.n}x{op.k}); add one to " + f"_MMA_FRAGMENT_INFO before consuming it" + ) + return self.layout + + +@dataclass(frozen=True) +class MmaSrc: + """Metadata for one matrix ``src``, with an optional scale operand.""" + + dtype: str + frag_len: int = 0 + layout: Optional[LayoutMap] = field(default=None, repr=False, compare=False) + scale: Optional[MmaScaleOperand] = None + + def __post_init__(self) -> None: + object.__setattr__(self, "dtype", normalize_dtype(self.dtype)) + + def require_layout(self, name: str, op: "MmaOp") -> LayoutMap: + if self.layout is None: + raise NotImplementedError( + f"no verified {name!r} layout map for MMA op_id {op.op_id!r} " + f"({op.m}x{op.n}x{op.k}); add one to " + f"_MMA_FRAGMENT_INFO before consuming it" + ) + return self.layout + + @dataclass(frozen=True) class MmaOp: """A single supported matrix-multiply-accumulate atom on a target. - ``op_id`` is the opaque handle the backend consumes; today it matches the - ISA-named ``IRBuilder`` method (e.g. ``mfma_f32_16x16x16_f16``). It is **not** - LLVM intrinsic text — the backend maps ``op_id`` to an intrinsic. - - The ``*_frag_len`` fields and the layout-map accessors describe the - *physical* register fragmentation of the atom: how many values of each - operand a single lane holds, and where (in the logical M/N/K tile) each of - those values lives. They are the cross-arch data-layout contract that lets - one kernel body drive both MFMA (wave64) and WMMA (wave32). They are derived - from ``(op_id, family, shape)`` at load time via :data:`_MMA_FRAGMENT_INFO`; - op_ids without verified layout maps still carry frag lengths but raise from - the accessor until a map is added. + ``op_id`` identifies a concrete operand contract. Scaled WMMA uses semantic + IDs including source and result dtypes and per-source scales. The backend + maps this identity to an intrinsic and format selectors. + + ``srcs`` contains ``src0``, ``src1``, and ``src2`` in machine order. The + first two are multiplicands, ``src2`` is the accumulator input, and ``dst`` + is the matrix result. + Optional scale operands belong to the source they scale; they are not extra + matrix fragments in ``srcs``. """ - family: str # "mma" | "wmma" - a_dtype: str - b_dtype: str - c_dtype: str + family: str # "mma" | "wmma" | "wmma_scaled" + srcs: Tuple[MmaSrc, MmaSrc, MmaSrc] + dst: MmaDst m: int n: int k: int op_id: str - a_frag_len: int = 0 - b_frag_len: int = 0 - c_frag_len: int = 0 wave_size: int = 64 - # Layout maps are kept off the public field list (repr=False, compare=False) - # so two MmaOps with the same shape still compare equal regardless of the - # closure identity of their maps. - _a_layout: Optional[LayoutMap] = field(default=None, repr=False, compare=False) - _b_layout: Optional[LayoutMap] = field(default=None, repr=False, compare=False) - _c_layout: Optional[LayoutMap] = field(default=None, repr=False, compare=False) + + def __post_init__(self) -> None: + if len(self.srcs) != 3: + raise ValueError(f"MMA op {self.op_id!r} requires exactly 3 matrix sources") + + def semantic_id(self) -> str: + """Stable semantic identity, independent of backend packing and names.""" + fields = [self.family.replace("_", "."), f"{self.m}x{self.n}x{self.k}"] + for i, src in enumerate(self.srcs): + field = f"src{i}_{normalize_dtype(src.dtype)}" + if src.scale is not None: + field += f"_{src.scale.dtype}_b{src.scale.block_size}" + fields.append(field) + fields.append(f"dst_{normalize_dtype(self.dst.dtype)}") + return ".".join(fields) @property def shape(self) -> Tuple[int, int, int]: return (self.m, self.n, self.k) - # --- physical-layout accessors --------------------------------------- + def src(self, index: int) -> MmaSrc: + if not 0 <= index < len(self.srcs): + raise IndexError(f"MMA src index {index} is outside [0, {len(self.srcs)})") + return self.srcs[index] + + def src_layout(self, index: int) -> LayoutMap: + return self.src(index).require_layout(f"src{index}", self) + + def dst_layout(self) -> LayoutMap: + return self.dst.require_layout("dst", self) + + # Compatibility projections for existing mathematical A/B/C callers. The + # historical C surface described the accumulator result, so C projects dst; + # src2 is available only through the canonical indexed interface. + @property + def a_dtype(self) -> str: + return self.srcs[0].dtype + + @property + def b_dtype(self) -> str: + return self.srcs[1].dtype + + @property + def c_dtype(self) -> str: + return self.dst.dtype + + @property + def a_frag_len(self) -> int: + return self.srcs[0].frag_len + + @property + def b_frag_len(self) -> int: + return self.srcs[1].frag_len + + @property + def c_frag_len(self) -> int: + return self.dst.frag_len + + # --- physical-layout compatibility accessors ------------------------- def a_layout(self) -> LayoutMap: """The A-operand ``(row, k)`` lane/slot -> coordinate map.""" - return self._require_layout(self._a_layout, "a") + return self.src_layout(0) def b_layout(self) -> LayoutMap: """The B-operand ``(k, col)`` lane/slot -> coordinate map.""" - return self._require_layout(self._b_layout, "b") + return self.src_layout(1) def c_layout(self) -> LayoutMap: - """The accumulator (C/D) ``(row, col)`` lane/slot -> coordinate map.""" - return self._require_layout(self._c_layout, "c") - - # Convenience aliases for the accumulator (the most-used map). - def acc_layout(self) -> LayoutMap: - return self.c_layout() - - def _require_layout(self, layout: Optional[LayoutMap], role: str) -> LayoutMap: - if layout is None: - raise NotImplementedError( - f"no verified {role!r} layout map for MMA op_id {self.op_id!r} " - f"({self.m}x{self.n}x{self.k}); add one to " - f"_MMA_FRAGMENT_INFO before consuming it" - ) - return layout + """Legacy result ``(row, col)`` lane/slot -> coordinate map.""" + return self.dst_layout() # --------------------------------------------------------------------------- @@ -181,8 +276,8 @@ def _require_layout(self, layout: Optional[LayoutMap], role: str) -> LayoutMap: # single LayoutMap covers all slots of the fragment. -def _mfma_acc_16x16(builder, lane, slot): - """MFMA 16x16 accumulator: slot ``i`` -> ``(m_blk*4 + i, lane % 16)``. +def _mfma_row_col_16x16(builder, lane, slot): + """MFMA 16x16 src2/dst row-column map. Mirrors ``MfmaAtom.lane_to_output`` for the (16, 16) case with ``c_per_lane == 4`` (``row = m_blk * 4 + i``, ``col = lane % 16``). @@ -194,8 +289,8 @@ def _mfma_acc_16x16(builder, lane, slot): return row, n_in_atom -def _mfma_acc_32x32(builder, lane, slot): - """MFMA 32x32 accumulator (``c_per_lane == 16``). +def _mfma_row_col_32x32(builder, lane, slot): + """MFMA 32x32 src2/dst row-column map. Mirrors ``MfmaAtom.lane_to_output`` for the (32, 32) case: ``row = (i // 4) * 8 + (lane // 32) * 4 + (i % 4)``, ``col = lane % 32``. @@ -372,8 +467,8 @@ def _mfma_b_32x32x16(builder, lane, slot): return k, n_in_atom -def _wmma_acc_16x16(builder, lane, slot): - """WMMA 16x16x16 accumulator (wave32, hardware-verified gfx1151). +def _wmma_row_col_16x16(builder, lane, slot): + """WMMA 16x16x16 src2/dst row-column map. ``<8 x float>`` per lane; slot ``i`` -> ``(row 2*i + lane // 16, col lane % 16)``. Returns ``(row, col)``. @@ -408,7 +503,7 @@ def _wmma_b_16x16(builder, lane, slot): # fragment: slot j (0..3) is one i32 holding the four int8 K-values # [4j, 4j+1, 4j+2, 4j+3]. The lane map therefore returns the K *base* of the # slot (4*j); the kernel/staging code packs the four consecutive K bytes from -# there. The accumulator is identical to f16 WMMA (_wmma_acc_16x16), only the +# there. The accumulator is identical to f16 WMMA (_wmma_row_col_16x16), only the # element type is i32 instead of f32. def _wmma_a_16x16_iu8(builder, lane, slot): """iu8 WMMA A operand (wave32): lane ``l`` holds row ``l % 16``; A fragment @@ -452,8 +547,8 @@ def _wmma_b_16x16_iu4(builder, lane, slot): # accumulator is column-distributed (CDNA/MFMA-style): lanes index columns, # registers index rows. These maps are the *hypothesis* verified empirically by # examples/gfx1201/wmma_probe.py before matmul_nbits is trusted on gfx1201. -def _wmma_gfx12_acc_16x16(builder, lane, slot): - """RDNA4 WMMA 16x16x16 accumulator (wave32): ``<8 x float>`` per lane; +def _wmma_gfx12_row_col_16x16(builder, lane, slot): + """RDNA4 WMMA 16x16x16 src2/dst row-column map: slot ``i`` -> ``(row (lane // 16) * 8 + i, col lane % 16)``. Returns ``(row, col)``.""" c16 = builder.const_i32(16) @@ -537,18 +632,53 @@ def _wmma_gfx1250_b_16x16x32(builder, lane, slot): return k, col +@dataclass(frozen=True) +class _FragOperand: + """Physical fragment metadata for one machine operand position.""" + + frag_len: int + fn: _LaneCoordFn = None + + @dataclass(frozen=True) class _FragInfo: - """Per-op_id fragment metadata: per-lane vector lengths, wave size, and the - (optional) lane/slot coordinate functions for A / B / accumulator.""" + """Per-op_id physical metadata for ``srcs[0..2]`` and ``dst``.""" - a_frag_len: int - b_frag_len: int - c_frag_len: int + srcs: Tuple[_FragOperand, _FragOperand, _FragOperand] + dst: _FragOperand wave_size: int - a_fn: _LaneCoordFn = None - b_fn: _LaneCoordFn = None - c_fn: _LaneCoordFn = None + + def __post_init__(self) -> None: + if len(self.srcs) != 3: + raise ValueError("MMA fragment metadata requires exactly 3 sources") + + +def _shared_src2_dst_frag_info( + src0_frag_len: int, + src1_frag_len: int, + row_col_frag_len: int, + wave_size: int, + src0_fn: _LaneCoordFn = None, + src1_fn: _LaneCoordFn = None, + row_col_fn: _LaneCoordFn = None, +) -> _FragInfo: + """Build metadata whose ``src2`` and ``dst`` layouts currently match. + + ``src2`` and ``dst`` get separate descriptors even when the current ISA + gives them the same fragment width and row/column coordinate function. + This is a catalog-construction convenience, not a constraint on + :class:`_FragInfo`; distinct contracts use that type directly. + """ + + return _FragInfo( + srcs=( + _FragOperand(src0_frag_len, src0_fn), + _FragOperand(src1_frag_len, src1_fn), + _FragOperand(row_col_frag_len, row_col_fn), + ), + dst=_FragOperand(row_col_frag_len, row_col_fn), + wave_size=wave_size, + ) # op_id -> physical fragment metadata. Frag lengths are populated for every @@ -559,57 +689,57 @@ class _FragInfo: # --- MFMA fp32 (wave64) ----------------------------------------------- # A/B are scalar float per lane (a_frag_len=b_frag_len=1); accumulator # shares the standard 16x16 / 32x32 layout (c_frag_len=4 / 16). - "mfma_f32_16x16x4_f32": _FragInfo( - 1, 1, 4, 64, _mfma_a_16x16x4_f32, _mfma_b_16x16x4_f32, _mfma_acc_16x16 + "mfma_f32_16x16x4_f32": _shared_src2_dst_frag_info( + 1, 1, 4, 64, _mfma_a_16x16x4_f32, _mfma_b_16x16x4_f32, _mfma_row_col_16x16 ), - "mfma_f32_32x32x2_f32": _FragInfo( - 1, 1, 16, 64, _mfma_a_32x32x2_f32, _mfma_b_32x32x2_f32, _mfma_acc_32x32 + "mfma_f32_32x32x2_f32": _shared_src2_dst_frag_info( + 1, 1, 16, 64, _mfma_a_32x32x2_f32, _mfma_b_32x32x2_f32, _mfma_row_col_32x32 ), # --- MFMA f16 (wave64) ------------------------------------------------ - "mfma_f32_16x16x16_f16": _FragInfo( - 4, 4, 4, 64, _mfma_a_16x16, _mfma_b_16x16, _mfma_acc_16x16 + "mfma_f32_16x16x16_f16": _shared_src2_dst_frag_info( + 4, 4, 4, 64, _mfma_a_16x16, _mfma_b_16x16, _mfma_row_col_16x16 ), - "mfma_f32_16x16x32_f16": _FragInfo( - 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_acc_16x16 + "mfma_f32_16x16x32_f16": _shared_src2_dst_frag_info( + 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_row_col_16x16 ), - "mfma_f32_32x32x8_f16": _FragInfo( - 4, 4, 16, 64, _mfma_a_32x32x8, _mfma_b_32x32x8, _mfma_acc_32x32 + "mfma_f32_32x32x8_f16": _shared_src2_dst_frag_info( + 4, 4, 16, 64, _mfma_a_32x32x8, _mfma_b_32x32x8, _mfma_row_col_32x32 ), - "mfma_f32_32x32x16_f16": _FragInfo( - 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_acc_32x32 + "mfma_f32_32x32x16_f16": _shared_src2_dst_frag_info( + 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_row_col_32x32 ), - "mfma_f32_4x4x4_f16": _FragInfo(4, 4, 4, 64), + "mfma_f32_4x4x4_f16": _shared_src2_dst_frag_info(4, 4, 4, 64), # --- MFMA bf16 (wave64) ---------------------------------------------- - "mfma_f32_16x16x16_bf16": _FragInfo( - 4, 4, 4, 64, _mfma_a_16x16, _mfma_b_16x16, _mfma_acc_16x16 + "mfma_f32_16x16x16_bf16": _shared_src2_dst_frag_info( + 4, 4, 4, 64, _mfma_a_16x16, _mfma_b_16x16, _mfma_row_col_16x16 ), - "mfma_f32_16x16x32_bf16": _FragInfo( - 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_acc_16x16 + "mfma_f32_16x16x32_bf16": _shared_src2_dst_frag_info( + 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_row_col_16x16 ), - "mfma_f32_32x32x8_bf16": _FragInfo( - 4, 4, 16, 64, _mfma_a_32x32x8, _mfma_b_32x32x8, _mfma_acc_32x32 + "mfma_f32_32x32x8_bf16": _shared_src2_dst_frag_info( + 4, 4, 16, 64, _mfma_a_32x32x8, _mfma_b_32x32x8, _mfma_row_col_32x32 ), - "mfma_f32_32x32x16_bf16": _FragInfo( - 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_acc_32x32 + "mfma_f32_32x32x16_bf16": _shared_src2_dst_frag_info( + 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_row_col_32x32 ), # --- MFMA fp8 / bf8 (wave64) ----------------------------------------- # fp8/bf8 share the f16 operand lane layout (a_per_lane=8, same K-packing); # only the element type / intrinsic mangling differ. - "mfma_f32_16x16x32_fp8": _FragInfo( - 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_acc_16x16 + "mfma_f32_16x16x32_fp8": _shared_src2_dst_frag_info( + 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_row_col_16x16 ), - "mfma_f32_16x16x32_bf8": _FragInfo( - 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_acc_16x16 + "mfma_f32_16x16x32_bf8": _shared_src2_dst_frag_info( + 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_row_col_16x16 ), - "mfma_f32_32x32x16_fp8": _FragInfo( - 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_acc_32x32 + "mfma_f32_32x32x16_fp8": _shared_src2_dst_frag_info( + 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_row_col_32x32 ), - "mfma_f32_32x32x16_bf8": _FragInfo( - 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_acc_32x32 + "mfma_f32_32x32x16_bf8": _shared_src2_dst_frag_info( + 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_row_col_32x32 ), # --- MFMA MX (wave64), frag lengths only ----------------------------- - "mfma_f32_16x16x128_fp4": _FragInfo(16, 16, 4, 64), - "mfma_f32_16x16x96_fp6": _FragInfo(12, 12, 4, 64), + "mfma_f32_16x16x128_fp4": _shared_src2_dst_frag_info(16, 16, 4, 64), + "mfma_f32_16x16x96_fp6": _shared_src2_dst_frag_info(12, 12, 4, 64), # Unscaled fp8 K=128 hero atom (lowers through the f8f6f4 scale-MFMA # intrinsic; there is no dense plain fp8 K=128). A/B are 32 fp8 bytes per # lane (<8 x i32> at the intrinsic boundary), accumulator <4 x float> -- @@ -617,37 +747,37 @@ class _FragInfo: # op_id resolves to correct fragment lengths (this table is the SSOT that # ir.IRBuilder.mma consults) even before it is added to the JSON catalog, # instead of hitting the zero-length _frag_info fallback. - "mfma_f32_16x16x128_fp8": _FragInfo(32, 32, 4, 64), - "mfma_scale_f32_16x16x128_f8f6f4": _FragInfo(32, 32, 4, 64), + "mfma_f32_16x16x128_fp8": _shared_src2_dst_frag_info(32, 32, 4, 64), + "mfma_scale_f32_16x16x128_f8f6f4": _shared_src2_dst_frag_info(32, 32, 4, 64), # --- WMMA f16 / bf16 (wave32, RDNA) ---------------------------------- - "wmma_f32_16x16x16_f16": _FragInfo( - 16, 16, 8, 32, _wmma_a_16x16, _wmma_b_16x16, _wmma_acc_16x16 + "wmma_f32_16x16x16_f16": _shared_src2_dst_frag_info( + 16, 16, 8, 32, _wmma_a_16x16, _wmma_b_16x16, _wmma_row_col_16x16 ), # bf16 shares the f16 fragment layout (same 16x16x16 lane math; only the # element type / intrinsic mangling differ — operands lower as <16 x i16>). - "wmma_f32_16x16x16_bf16": _FragInfo( - 16, 16, 8, 32, _wmma_a_16x16, _wmma_b_16x16, _wmma_acc_16x16 + "wmma_f32_16x16x16_bf16": _shared_src2_dst_frag_info( + 16, 16, 8, 32, _wmma_a_16x16, _wmma_b_16x16, _wmma_row_col_16x16 ), # --- WMMA iu8 (wave32, RDNA3/3.5) ------------------------------------------ # A/B fragments are <4 x i32> (16 int8 packed 4-per-i32); accumulator is # <8 x i32> with the same lane math as the f16 WMMA accumulator. - "wmma_i32_16x16x16_iu8": _FragInfo( - 4, 4, 8, 32, _wmma_a_16x16_iu8, _wmma_b_16x16_iu8, _wmma_acc_16x16 + "wmma_i32_16x16x16_iu8": _shared_src2_dst_frag_info( + 4, 4, 8, 32, _wmma_a_16x16_iu8, _wmma_b_16x16_iu8, _wmma_row_col_16x16 ), # --- WMMA iu4 (wave32, RDNA3/3.5) ------------------------------------------ # A/B fragments are <2 x i32> (16 int4 packed 8-per-i32); accumulator is # <8 x i32> with the same lane math as the f16 WMMA accumulator. - "wmma_i32_16x16x16_iu4": _FragInfo( - 2, 2, 8, 32, _wmma_a_16x16_iu4, _wmma_b_16x16_iu4, _wmma_acc_16x16 + "wmma_i32_16x16x16_iu4": _shared_src2_dst_frag_info( + 2, 2, 8, 32, _wmma_a_16x16_iu4, _wmma_b_16x16_iu4, _wmma_row_col_16x16 ), # --- WMMA f16 / bf16 (wave32, RDNA4 / gfx12) ------------------------------- # No cross-half duplication: A/B are <8 x half> per lane; column-distributed # accumulator. Lane maps verified by examples/gfx1201/wmma_probe.py. - "wmma_gfx12_f32_16x16x16_f16": _FragInfo( - 8, 8, 8, 32, _wmma_gfx12_a_16x16, _wmma_gfx12_b_16x16, _wmma_gfx12_acc_16x16 + "wmma_gfx12_f32_16x16x16_f16": _shared_src2_dst_frag_info( + 8, 8, 8, 32, _wmma_gfx12_a_16x16, _wmma_gfx12_b_16x16, _wmma_gfx12_row_col_16x16 ), - "wmma_gfx12_f32_16x16x16_bf16": _FragInfo( - 8, 8, 8, 32, _wmma_gfx12_a_16x16, _wmma_gfx12_b_16x16, _wmma_gfx12_acc_16x16 + "wmma_gfx12_f32_16x16x16_bf16": _shared_src2_dst_frag_info( + 8, 8, 8, 32, _wmma_gfx12_a_16x16, _wmma_gfx12_b_16x16, _wmma_gfx12_row_col_16x16 ), # --- WMMA fp32 (wave32, gfx1250, CDNA) -------------------------------- # K=4 atom: A/B are <2 x fp32> per lane (a_frag_len=b_frag_len=2). @@ -655,116 +785,116 @@ class _FragInfo: # Accumulator is the gfx12 column-distributed <8 x float> (c_frag_len=8). # Intrinsic: __builtin_amdgcn_wmma_f32_16x16x4_f32 (verified in CK Tile # wmma_gfx12.hpp lines 355-373; amdgcn_mma_base kABKPerLane=2, kCMPerLane=8). - "wmma_gfx1250_f32_16x16x4_f32": _FragInfo( + "wmma_gfx1250_f32_16x16x4_f32": _shared_src2_dst_frag_info( 2, 2, 8, 32, _wmma_gfx1250_a_16x16x4_f32, _wmma_gfx1250_b_16x16x4_f32, - _wmma_gfx12_acc_16x16, + _wmma_gfx12_row_col_16x16, ), # --- WMMA f16 / bf16 (wave32, gfx1250, CDNA) ---------------------- # K=32 atom: A/B are <16 x half> per lane (K split across lane-halves, 16 # each); accumulator is the same 16x16 column-distributed <8 x float> as # gfx12. Lane maps verified by examples/gfx1250/wmma_probe.py. - "wmma_gfx1250_f32_16x16x32_f16": _FragInfo( + "wmma_gfx1250_f32_16x16x32_f16": _shared_src2_dst_frag_info( 16, 16, 8, 32, _wmma_gfx1250_a_16x16x32, _wmma_gfx1250_b_16x16x32, - _wmma_gfx12_acc_16x16, + _wmma_gfx12_row_col_16x16, ), # gfx1250 FP8/BF8 K=64 WMMA. A/B carry 32 low-bit bytes per lane presented # as <8 x i32>; accumulator is the same 16x16 column-distributed <8 x float> # as the f16/bf16 K=32 atom. The block-scaled GEMM kernel computes operand # offsets directly (no LayoutMap), so only frag lengths + wave size are # registered here; the accumulator map is shared with the f16 path. - "wmma_gfx1250_f32_16x16x64_fp8_fp8": _FragInfo( + "wmma_gfx1250_f32_16x16x64_fp8_fp8": _shared_src2_dst_frag_info( 8, 8, 8, 32, None, None, - _wmma_gfx12_acc_16x16, + _wmma_gfx12_row_col_16x16, ), - "wmma_gfx1250_f32_16x16x64_fp8_bf8": _FragInfo( + "wmma_gfx1250_f32_16x16x64_fp8_bf8": _shared_src2_dst_frag_info( 8, 8, 8, 32, None, None, - _wmma_gfx12_acc_16x16, + _wmma_gfx12_row_col_16x16, ), - "wmma_gfx1250_f32_16x16x64_bf8_fp8": _FragInfo( + "wmma_gfx1250_f32_16x16x64_bf8_fp8": _shared_src2_dst_frag_info( 8, 8, 8, 32, None, None, - _wmma_gfx12_acc_16x16, + _wmma_gfx12_row_col_16x16, ), - "wmma_gfx1250_f32_16x16x64_bf8_bf8": _FragInfo( + "wmma_gfx1250_f32_16x16x64_bf8_bf8": _shared_src2_dst_frag_info( 8, 8, 8, 32, None, None, - _wmma_gfx12_acc_16x16, + _wmma_gfx12_row_col_16x16, ), # Native gfx1250 scaled WMMA. FP8/BF8 use 64 bytes per lane as <16 x i32>. # SCALE packs four K=32 E8M0 bytes in i32 and SCALE16 packs eight K=16 # bytes in i64. Both share the gfx12 column-distributed accumulator. - "wmma_scale_f32_16x16x128_fp8_fp8": _FragInfo( + "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32": _shared_src2_dst_frag_info( 16, 16, 8, 32, None, None, - _wmma_gfx12_acc_16x16, + _wmma_gfx12_row_col_16x16, ), - "wmma_scale_f32_16x16x128_bf8_bf8": _FragInfo( + "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32": _shared_src2_dst_frag_info( 16, 16, 8, 32, None, None, - _wmma_gfx12_acc_16x16, + _wmma_gfx12_row_col_16x16, ), - "wmma_scale16_f32_16x16x128_fp8_fp8": _FragInfo( + "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32": _shared_src2_dst_frag_info( 16, 16, 8, 32, None, None, - _wmma_gfx12_acc_16x16, + _wmma_gfx12_row_col_16x16, ), - "wmma_scale16_f32_16x16x128_bf8_bf8": _FragInfo( + "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32": _shared_src2_dst_frag_info( 16, 16, 8, 32, None, None, - _wmma_gfx12_acc_16x16, + _wmma_gfx12_row_col_16x16, ), - "wmma_gfx1250_f32_16x16x32_bf16": _FragInfo( + "wmma_gfx1250_f32_16x16x32_bf16": _shared_src2_dst_frag_info( 16, 16, 8, 32, _wmma_gfx1250_a_16x16x32, _wmma_gfx1250_b_16x16x32, - _wmma_gfx12_acc_16x16, + _wmma_gfx12_row_col_16x16, ), } @@ -774,7 +904,7 @@ def _frag_info(op_id: str) -> _FragInfo: if info is None: # Unknown atoms still load (e.g. a future JSON-only op_id); they carry # zero frag lengths and no maps until registered here. - return _FragInfo(0, 0, 0, 64) + return _shared_src2_dst_frag_info(0, 0, 0, 64) return info @@ -796,7 +926,7 @@ class ResourceLimits: class MmaCatalog: - """The arch-selected set of MMA atoms, with enumeration + best-K selection.""" + """Query concrete operand contracts; omitted scales are a wildcard.""" def __init__(self, ops: List[MmaOp]) -> None: self._ops = tuple(ops) @@ -809,22 +939,39 @@ def enumerate( self, *, family: str = "mma", - a_dtype: str, - b_dtype: str, - c_dtype: str, + src_dtypes: Optional[Tuple[str, str, str]] = None, + dst_dtype: Optional[str] = None, + src_scales: Optional[Tuple[Optional[MmaScaleOperand], ...]] = None, + a_dtype: Optional[str] = None, + b_dtype: Optional[str] = None, + c_dtype: Optional[str] = None, m: Optional[int] = None, n: Optional[int] = None, ) -> List[MmaOp]: - a, b, c = ( - normalize_dtype(a_dtype), - normalize_dtype(b_dtype), - normalize_dtype(c_dtype), - ) + if src_dtypes is None: + if a_dtype is None or b_dtype is None or c_dtype is None: + raise TypeError("enumerate requires src_dtypes or a/b/c_dtype") + src_dtypes = (a_dtype, b_dtype, c_dtype) + if len(src_dtypes) != 3: + raise ValueError("src_dtypes must contain exactly 3 entries") + if src_scales is not None and len(src_scales) != 3: + raise ValueError("src_scales must contain exactly 3 entries") + src_keys = tuple(normalize_dtype(dtype) for dtype in src_dtypes) + # Preserve the historical three-dtype query by defaulting dst to src2. + # Indexed callers can state a distinct result. + dst_key = normalize_dtype(dst_dtype or src_dtypes[2]) out = [] for op in self._ops: if op.family != family: continue - if (op.a_dtype, op.b_dtype, op.c_dtype) != (a, b, c): + if ( + tuple(src.dtype for src in op.srcs) != src_keys + or op.dst.dtype != dst_key + ): + continue + if src_scales is not None and tuple(src.scale for src in op.srcs) != tuple( + src_scales + ): continue if m is not None and op.m != m: continue @@ -837,9 +984,12 @@ def has_shape( self, *, family: str = "mma", - a_dtype: str, - b_dtype: str, - c_dtype: str, + src_dtypes: Optional[Tuple[str, str, str]] = None, + a_dtype: Optional[str] = None, + b_dtype: Optional[str] = None, + c_dtype: Optional[str] = None, + dst_dtype: Optional[str] = None, + src_scales: Optional[Tuple[Optional[MmaScaleOperand], ...]] = None, m: int, n: int, k: int, @@ -848,9 +998,12 @@ def has_shape( op.shape == (m, n, k) for op in self.enumerate( family=family, + src_dtypes=src_dtypes, + src_scales=src_scales, a_dtype=a_dtype, b_dtype=b_dtype, c_dtype=c_dtype, + dst_dtype=dst_dtype, m=m, n=n, ) @@ -860,9 +1013,12 @@ def select_largest_k( self, *, family: str = "mma", - a_dtype: str, - b_dtype: str, - c_dtype: str, + src_dtypes: Optional[Tuple[str, str, str]] = None, + a_dtype: Optional[str] = None, + b_dtype: Optional[str] = None, + c_dtype: Optional[str] = None, + dst_dtype: Optional[str] = None, + src_scales: Optional[Tuple[Optional[MmaScaleOperand], ...]] = None, m: int, n: int, k_max: Optional[int] = None, @@ -871,9 +1027,12 @@ def select_largest_k( op for op in self.enumerate( family=family, + src_dtypes=src_dtypes, + src_scales=src_scales, a_dtype=a_dtype, b_dtype=b_dtype, c_dtype=c_dtype, + dst_dtype=dst_dtype, m=m, n=n, ) @@ -881,7 +1040,14 @@ def select_largest_k( ] if not cands: return None - return max(cands, key=lambda op: op.k) + largest_k = max(op.k for op in cands) + return self._unique([op for op in cands if op.k == largest_k]) + + @staticmethod + def _unique(ops: List[MmaOp]) -> Optional[MmaOp]: + if len(ops) > 1: + raise ValueError("ambiguous MMA query; specify the full operand contract") + return ops[0] if ops else None def by_op_id(self, op_id: str) -> Optional[MmaOp]: """Look up an atom by its ``op_id`` handle (the backend's MMA key).""" @@ -894,24 +1060,28 @@ def op_for_shape( self, *, family: str = "mma", - a_dtype: str, - b_dtype: str, - c_dtype: str, + src_dtypes: Optional[Tuple[str, str, str]] = None, + a_dtype: Optional[str] = None, + b_dtype: Optional[str] = None, + c_dtype: Optional[str] = None, + dst_dtype: Optional[str] = None, + src_scales: Optional[Tuple[Optional[MmaScaleOperand], ...]] = None, m: int, n: int, k: int, ) -> Optional[MmaOp]: - for op in self.enumerate( + candidates = self.enumerate( family=family, + src_dtypes=src_dtypes, + src_scales=src_scales, a_dtype=a_dtype, b_dtype=b_dtype, c_dtype=c_dtype, + dst_dtype=dst_dtype, m=m, n=n, - ): - if op.k == k: - return op - return None + ) + return self._unique([op for op in candidates if op.k == k]) @dataclass(frozen=True) @@ -947,10 +1117,27 @@ def fits_lds(self, bytes_in_use: int) -> bool: return bytes_in_use <= self.lds_capacity_bytes def supports_dtype_combo( - self, a: str, b: str, c: str, *, family: str = "mma" + self, + a: str, + b: str, + c: str, + dst: Optional[str] = None, + *, + family: str = "mma", + src_scales: Optional[Tuple[Optional[MmaScaleOperand], ...]] = None, ) -> bool: return ( - len(self.mma.enumerate(family=family, a_dtype=a, b_dtype=b, c_dtype=c)) > 0 + len( + self.mma.enumerate( + family=family, + a_dtype=a, + b_dtype=b, + c_dtype=c, + dst_dtype=dst, + src_scales=src_scales, + ) + ) + > 0 ) def max_vector_load_dwords(self, dtype: str) -> int: @@ -981,21 +1168,21 @@ def _load_specs() -> Dict[str, dict]: @lru_cache(maxsize=1) -def _op_id_c_dtype() -> Dict[str, str]: - """``op_id -> normalized accumulator dtype``, aggregated across every arch +def _op_id_dst_dtype() -> Dict[str, str]: + """``op_id -> normalized dst dtype``, aggregated across every arch row in the JSON catalog. - An ``op_id`` names a specific atom, so its accumulator dtype is invariant + An ``op_id`` names a specific atom, so its ``dst`` dtype is invariant across the arches that list it. This lets a caller that only has a bare ``op_id`` string (e.g. :meth:`rocke.core.ir.IRBuilder.mma`) recover the - accumulator dtype from the SSOT without holding an :class:`ArchTarget`. + ``dst`` dtype from the SSOT without holding an :class:`ArchTarget`. Op_ids that are frag-registered but not yet in the JSON catalog are absent - (callers should treat a miss as the default f32 accumulator). + (callers should treat a miss as the default f32 ``dst``). The **first** catalog hit for an op_id wins, matching the C implementation - (``rocke_arch_mma_op_id_c_dtype`` in ``query.cpp``, which returns the first + (``rocke_arch_mma_op_id_dst_dtype`` in ``query.cpp``, which returns the first registry hit). If a later arch lists the same op_id with a *different* - accumulator dtype we raise instead of silently overwriting: that would be + ``dst`` dtype we raise instead of silently overwriting: that would be SSOT drift (the invariant above is broken) and must be fixed in the catalog, not masked. Raising here also keeps the Python and C engines deterministic and byte-identical rather than diverging on catalog ordering. @@ -1004,21 +1191,26 @@ def _op_id_c_dtype() -> Dict[str, str]: for row in _load_specs().values(): for o in row["mma"]: op_id = o["op_id"] - c = normalize_dtype(o["c"]) + dst_row = o["dst"] if "dst" in o else {"dtype": o["c"]} + c = normalize_dtype(dst_row["dtype"]) prev = out.get(op_id) if prev is None: out[op_id] = c # first hit wins elif prev != c: raise ValueError( f"arch SSOT drift in {_DATA_FILE.name}: op_id {op_id!r} has " - f"inconsistent accumulator dtype across arches " + f"inconsistent dst dtype across arches " f"({prev!r} vs {c!r}). An op_id names a specific atom, so its " - f"accumulator dtype must be invariant across the arches that " + f"dst dtype must be invariant across the arches that " f"list it; fix the catalog so every row agrees." ) return out +# Compatibility for callers that used C to mean the instruction result. +_op_id_c_dtype = _op_id_dst_dtype + + def _build_mma_op(o: dict) -> MmaOp: """Construct an :class:`MmaOp` from one catalog JSON row, attaching the physical fragment lengths and layout maps registered for its op_id.""" @@ -1030,23 +1222,55 @@ def _mk(role: str, frag_len: int, fn: _LaneCoordFn) -> Optional[LayoutMap]: return None return LayoutMap(role=role, frag_len=frag_len, wave_size=info.wave_size, fn=fn) - return MmaOp( + src_rows = o.get("srcs") + if src_rows is None: + src_rows = ({"dtype": o["a"]}, {"dtype": o["b"]}, {"dtype": o["c"]}) + if len(src_rows) != 3: + raise ValueError( + f"MMA catalog op {op_id!r} must define exactly 3 matrix sources" + ) + + def _source(row: dict, role: str, frag_len: int, fn: _LaneCoordFn) -> MmaSrc: + scale_row = row.get("scale") + scale = ( + MmaScaleOperand( + dtype=normalize_dtype(scale_row["dtype"]), + block_size=scale_row["block_size"], + ) + if scale_row is not None + else None + ) + return MmaSrc( + dtype=normalize_dtype(row["dtype"]), + frag_len=frag_len, + layout=_mk(role, frag_len, fn), + scale=scale, + ) + + dst_row = o["dst"] if "dst" in o else {"dtype": o["c"]} + op = MmaOp( family=o["family"], - a_dtype=normalize_dtype(o["a"]), - b_dtype=normalize_dtype(o["b"]), - c_dtype=normalize_dtype(o["c"]), + srcs=( + _source(src_rows[0], "src0", info.srcs[0].frag_len, info.srcs[0].fn), + _source(src_rows[1], "src1", info.srcs[1].frag_len, info.srcs[1].fn), + _source(src_rows[2], "src2", info.srcs[2].frag_len, info.srcs[2].fn), + ), + dst=MmaDst( + dtype=normalize_dtype(dst_row["dtype"]), + frag_len=info.dst.frag_len, + layout=_mk("dst", info.dst.frag_len, info.dst.fn), + ), m=o["m"], n=o["n"], k=o["k"], op_id=op_id, - a_frag_len=info.a_frag_len, - b_frag_len=info.b_frag_len, - c_frag_len=info.c_frag_len, wave_size=info.wave_size, - _a_layout=_mk("a", info.a_frag_len, info.a_fn), - _b_layout=_mk("b", info.b_frag_len, info.b_fn), - _c_layout=_mk("c", info.c_frag_len, info.c_fn), ) + if op.family == "wmma_scaled" and op.op_id != op.semantic_id(): + raise ValueError( + f"scaled MMA catalog ID does not match its contract: {op.op_id}" + ) + return op @lru_cache(maxsize=None) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py index 3a80ab24c451..78f86df448ed 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py @@ -6,6 +6,8 @@ from dataclasses import dataclass +from .target import ArchTarget, MmaOp + @dataclass(frozen=True) class E8M0ScalePacking: @@ -33,31 +35,69 @@ def llvm_type(self) -> str: @dataclass(frozen=True) class ScaledWmmaOp: - """One supported matrix format and its packed E8M0 scale contract.""" + """Backend packing derived from a supported catalog operand contract.""" - op_id: str - matrix_dtype: str + atom: MmaOp + matrix_formats: tuple[int, int] + scale_formats: tuple[int, int] scales: E8M0ScalePacking + @property + def op_id(self) -> str: + return self.atom.op_id + @property def scale16(self) -> bool: return self.scales.block_k == 16 @property - def matrix_format(self) -> int: - return {"fp8": 0, "bf8": 1}[self.matrix_dtype] + def matrix_llvm_types(self) -> tuple[str, str]: + return tuple(f"<{src.frag_len} x i32>" for src in self.atom.srcs[:2]) + @property + def intrinsic_suffix(self) -> str: + atom = self.atom + return ( + f"f32.{atom.m}x{atom.n}x{atom.k}.f8f6f4.v{atom.dst.frag_len}f32." + f"v{atom.srcs[0].frag_len}i32.v{atom.srcs[1].frag_len}i32" + ) -_SCALE = E8M0ScalePacking(count=4, block_k=32) -_SCALE16 = E8M0ScalePacking(count=8, block_k=16) -_GFX1250_WMMA_SCALE = { - op_id: ScaledWmmaOp(op_id=op_id, matrix_dtype=dtype, scales=packing) - for family, packing in (("wmma_scale", _SCALE), ("wmma_scale16", _SCALE16)) - for dtype in ("fp8", "bf8") - for op_id in (f"{family}_f32_16x16x128_{dtype}_{dtype}",) -} + @property + def intrinsic(self) -> str: + mode = "scale16" if self.scale16 else "scale" + return f"llvm.amdgcn.wmma.{mode}.{self.intrinsic_suffix}" + + @property + def declaration_key(self) -> str: + return f"wmma.scale.block{self.scales.block_k}.gfx1250.{self.intrinsic_suffix}" def gfx1250_scaled_wmma(op_id: str) -> ScaledWmmaOp | None: - """Resolve a catalog ID or concrete tile op name; reject unknown variants.""" - return _GFX1250_WMMA_SCALE.get(op_id.removeprefix("tile.")) + """Resolve a catalog contract; LLVM selectors are backend details.""" + atom = ArchTarget.from_gfx("gfx1250").mma.by_op_id(op_id.removeprefix("tile.")) + if atom is None or atom.family != "wmma_scaled": + return None + formats = {"fp8e4m3": 0, "bf8e5m2": 1} + src0, src1, src2 = atom.srcs + # The current backend signatures require matching E8M0 scale packing. + # Keep these restrictions here, independently of the catalog query model. + if ( + src0.dtype not in formats + or src1.dtype not in formats + or src0.scale is None + or src0.scale != src1.scale + or src0.scale.dtype != "e8m0" + or src2.scale is not None + or src2.dtype != "fp32" + or atom.dst.dtype != "fp32" + or atom.shape != (16, 16, 128) + ): + raise ValueError(f"unsupported scaled WMMA backend contract: {atom.op_id}") + return ScaledWmmaOp( + atom=atom, + matrix_formats=(formats[src0.dtype], formats[src1.dtype]), + scale_formats=(0, 0), # E8M0 for each source. + scales=E8M0ScalePacking( + count=atom.k // src0.scale.block_size, block_k=src0.scale.block_size + ), + ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py index 8065f6107117..1f1eb493526a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py @@ -85,10 +85,6 @@ def __repr__(self) -> str: "mfma_f32_16x16x96_fp6": "acc6", "mfma_f32_16x16x128_fp8": "acc128", "mfma_scale_f32_16x16x128_f8f6f4": "mxacc", - "wmma_scale_f32_16x16x128_fp8_fp8": "mxacc", - "wmma_scale_f32_16x16x128_bf8_bf8": "mxacc", - "wmma_scale16_f32_16x16x128_fp8_fp8": "mxacc", - "wmma_scale16_f32_16x16x128_bf8_bf8": "mxacc", } @@ -134,8 +130,8 @@ def _check_cachepolicy(op: str, value: int) -> int: return v -def _mma_c_frag_len(op_id: str) -> int: - """Accumulator fragment length for ``op_id`` from the arch SSOT. +def _mma_dst_frag_len(op_id: str) -> int: + """``dst`` fragment length for ``op_id`` from the arch SSOT. Resolved through ``core/arch/target._MMA_FRAGMENT_INFO`` (imported lazily); ir.py holds no private copy. Unknown op_ids (frag length 0) raise, matching @@ -143,7 +139,7 @@ def _mma_c_frag_len(op_id: str) -> int: """ from rocke.core.arch import target as _arch - frag_len = _arch._frag_info(op_id).c_frag_len + frag_len = _arch._frag_info(op_id).dst.frag_len if frag_len <= 0: raise ValueError( f"unknown MMA op_id {op_id!r}; pass an MmaOp or one of " @@ -152,16 +148,21 @@ def _mma_c_frag_len(op_id: str) -> int: return frag_len -def _mma_c_is_int(op_id: str) -> bool: - """True when ``op_id`` accumulates in i32 (integer WMMA). +def _mma_dst_is_int(op_id: str) -> bool: + """True when ``op_id`` produces i32 (integer WMMA). - Sourced from the arch catalog's accumulator dtype - (``core/arch/data/arch_specs.json`` via ``target._op_id_c_dtype``), imported - lazily. Op_ids absent from the catalog default to the f32 accumulator. + Sourced from the arch catalog's ``dst`` dtype + (``core/arch/data/arch_specs.json`` via ``target._op_id_dst_dtype``), imported + lazily. Op_ids absent from the catalog default to an f32 ``dst``. """ from rocke.core.arch import target as _arch - return _arch._op_id_c_dtype().get(op_id) == "i32" + return _arch._op_id_dst_dtype().get(op_id) == "i32" + + +# Compatibility aliases for code that historically used C as the result role. +_mma_c_frag_len = _mma_dst_frag_len +_mma_c_is_int = _mma_dst_is_int @dataclass(frozen=True) @@ -1830,15 +1831,15 @@ def mma( """Target-neutral matrix-multiply-accumulate: ``D = A * B + C``. ``op`` is either an :class:`~rocke.core.arch.MmaOp` (preferred — its - ``op_id`` and ``c_frag_len`` drive the lowering) or a raw ``op_id`` + ``op_id`` and ``dst`` fragment metadata drive the lowering) or a raw ``op_id`` string. This emits a single ``tile.mma`` op carrying the ``op_id`` as an attribute; the LLVM lowering dispatches that ``op_id`` through the ISA backend (:meth:`rocke.core.isa.ISABackend.emit_mma`), which emits the matching MFMA call on CDNA or the WMMA call on RDNA. **One kernel body, two ISAs.** - The result vector type is ```` (the per-lane - accumulator length the atom produces). When ``op`` is an ``op_id`` + The result vector type is ```` (the per-lane + result length the atom produces). When ``op`` is an ``op_id`` string the frag length is resolved from the static MMA fragment table. The ISA-named helpers (:meth:`mfma_f32_16x16x16_f16`, @@ -1850,22 +1851,28 @@ def mma( (``a_scale``, ``b_scale``); ordinary atoms take exactly ``a, b, c``. """ op_id = op.op_id if hasattr(op, "op_id") else str(op) - c_frag_len = ( - op.c_frag_len - if hasattr(op, "c_frag_len") and op.c_frag_len - else _mma_c_frag_len(op_id) + dst_frag_len = ( + op.dst.frag_len + if hasattr(op, "dst") and op.dst.frag_len + else _mma_dst_frag_len(op_id) ) # Accumulator element type: integer WMMA atoms (iu8/iu4) accumulate in - # i32; everything else in f32. Prefer the atom's own c_dtype when ``op`` + # i32; everything else in f32. Prefer the atom's own dst dtype when ``op`` # is an MmaOp, else resolve from the arch SSOT via op_id. - c_dtype = getattr(op, "c_dtype", None) - is_int_acc = c_dtype == "i32" if c_dtype is not None else _mma_c_is_int(op_id) - c_elem = I32 if is_int_acc else F32 - hint = _MMA_RESULT_HINT.get(op_id, "acc") + dst_dtype = op.dst.dtype if hasattr(op, "dst") else None + is_int_acc = ( + dst_dtype == "i32" if dst_dtype is not None else _mma_dst_is_int(op_id) + ) + dst_elem = I32 if is_int_acc else F32 + hint = ( + "mxacc" + if op_id.startswith("wmma.scaled.") + else _MMA_RESULT_HINT.get(op_id, "acc") + ) return self._op( "tile.mma", [a, b, c, *extra], - [VectorType(c_elem, c_frag_len)], + [VectorType(dst_elem, dst_frag_len)], attrs={"op_id": op_id}, result_name_hint=hint, ).result @@ -1965,38 +1972,6 @@ def wmma_gfx1250_f32_16x16x64_bf8_bf8(self, a: Value, b: Value, c: Value) -> Val """gfx1250 (gfx1250) BF8 K=64 WMMA. Thin wrapper over :meth:`mma`.""" return self.mma("wmma_gfx1250_f32_16x16x64_bf8_bf8", a, b, c) - def wmma_scale_f32_16x16x128_fp8_fp8( - self, - a: Value, - b: Value, - c: Value, - a_scale: Value, - b_scale: Value, - ) -> Value: - """gfx1250 native SCALE FP8 WMMA with packed E8M0 scale operands. - - A and B are ``<16 x i32>`` fragments (64 FP8 bytes per lane), C is - ``<8 x f32>``, and each scale operand is one i32 packing four E8M0 - bytes for the instruction's four K=32 scale blocks. - """ - return self.mma("wmma_scale_f32_16x16x128_fp8_fp8", a, b, c, a_scale, b_scale) - - def wmma_scale16_f32_16x16x128_fp8_fp8( - self, - a: Value, - b: Value, - c: Value, - a_scale: Value, - b_scale: Value, - ) -> Value: - """gfx1250 native SCALE16 FP8 WMMA with eight packed E8M0 scales. - - The matrix and accumulator fragments match - :meth:`wmma_scale_f32_16x16x128_fp8_fp8`; each scale operand is i64 - because SCALE16 carries eight K=16 E8M0 scale bytes. - """ - return self.mma("wmma_scale16_f32_16x16x128_fp8_fp8", a, b, c, a_scale, b_scale) - def mfma_f32_16x16x16_f16(self, a: Value, b: Value, c: Value) -> Value: return self.mma("mfma_f32_16x16x16_f16", a, b, c) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/isa/backend.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/isa/backend.py index ae92acb1de08..e4a657583bc0 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/isa/backend.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/isa/backend.py @@ -612,7 +612,7 @@ def _emit_wmma_fp8(self, lowerer, op, spec) -> None: ) def _emit_wmma_scale(self, lowerer, op) -> None: - """Emit the ROCm 7.13+ gfx1250 SCALE/SCALE16 FP8/FP4 call.""" + """Emit the gfx1250 SCALE/SCALE16 intrinsic using the resolved operand contract.""" if lowerer._flavor != "llvm23": raise NotImplementedError( f"{op.name} requires llvm23 (ROCm 7.13+), got {lowerer._flavor}" @@ -622,12 +622,11 @@ def _emit_wmma_scale(self, lowerer, op) -> None: spec = gfx1250_scaled_wmma(op.name) if spec is None: raise NotImplementedError(f"unsupported scaled WMMA op {op.name!r}") - mode = "scale16" if spec.scale16 else "scale" - # Declaration keys are shared across matrix dtypes: the ABI is identical. - decl_key = f"wmma.{mode}.gfx1250.f32.16x16x128.fp8.fp8" - intrinsic = f"llvm.amdgcn.wmma.{mode}.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32" + # Declarations describe physical signatures, independently of matrix encodings. + decl_key = spec.declaration_key + intrinsic = spec.intrinsic scale_ty = spec.scales.llvm_type - fmt = spec.matrix_format + fmt0, fmt1 = spec.matrix_formats a, b, c, a_scale, b_scale = op.operands if a_scale.type.name != scale_ty or b_scale.type.name != scale_ty: raise ValueError( @@ -637,11 +636,11 @@ def _emit_wmma_scale(self, lowerer, op) -> None: lowerer._need(decl_key) lowerer._current().emit( f" {op.result.name} = call <8 x float> @{intrinsic}(" - f"i32 {fmt}, <16 x i32> {lowerer._operand(a)}, " - f"i32 {fmt}, <16 x i32> {lowerer._operand(b)}, " + f"i32 {fmt0}, {spec.matrix_llvm_types[0]} {lowerer._operand(a)}, " + f"i32 {fmt1}, {spec.matrix_llvm_types[1]} {lowerer._operand(b)}, " f"i16 0, <8 x float> {lowerer._operand(c)}, " - f"i32 0, i32 0, {scale_ty} {lowerer._operand(a_scale)}, " - f"i32 0, i32 0, {scale_ty} {lowerer._operand(b_scale)}, " + f"i32 0, i32 {spec.scale_formats[0]}, {scale_ty} {lowerer._operand(a_scale)}, " + f"i32 0, i32 {spec.scale_formats[1]}, {scale_ty} {lowerer._operand(b_scale)}, " f"i1 false, i1 false)" ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py index 1a5ca2a85620..0b1ddb1511db 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py @@ -310,6 +310,12 @@ def _pop_indent(self) -> None: self._indent -= 1 def lower_op(self, op: Op) -> None: + if ( + op.name.startswith("tile.wmma.") + and gfx1250_scaled_wmma(op.name) is not None + ): + self._emit_wmma_gfx1250_scaled(op) + return method = getattr(self, f"_op_{op.name.replace('.', '_')}", None) if method is None: raise NotImplementedError(f"no HIP lowering for op {op.name!r}") @@ -527,7 +533,7 @@ def _op_tile_mma(self, op: Op) -> None: The HIP source path has no ISA backend (it emits ``__builtin_amdgcn_*`` directly), so the ``op_id`` is mapped back to the concrete - ``tile.`` op and dispatched through the existing per-op handler. + ``tile.`` op and passed through the shared operation dispatcher. This keeps the HIP emission identical to the legacy ISA-named path while the IRBuilder helpers route through :meth:`IRBuilder.mma`. """ @@ -653,18 +659,6 @@ def _op_tile_wmma_gfx1250_f32_16x16x64_bf8_fp8(self, op: Op) -> None: def _op_tile_wmma_gfx1250_f32_16x16x64_bf8_bf8(self, op: Op) -> None: self._emit_wmma_gfx1250_fp8(op, "bf8_bf8") - def _op_tile_wmma_scale_f32_16x16x128_fp8_fp8(self, op: Op) -> None: - self._emit_wmma_gfx1250_scaled(op) - - def _op_tile_wmma_scale_f32_16x16x128_bf8_bf8(self, op: Op) -> None: - self._emit_wmma_gfx1250_scaled(op) - - def _op_tile_wmma_scale16_f32_16x16x128_fp8_fp8(self, op: Op) -> None: - self._emit_wmma_gfx1250_scaled(op) - - def _op_tile_wmma_scale16_f32_16x16x128_bf8_bf8(self, op: Op) -> None: - self._emit_wmma_gfx1250_scaled(op) - def _emit_wmma_gfx1250_fp8(self, op: Op, ab: str) -> None: # gfx1250 K=64 FP8/BF8 builtin: A/B are <8 x i32> (32 low-bit # bytes per lane), 6-operand form (A, B, fmt, C, reuseA, reuseB). @@ -681,7 +675,7 @@ def _emit_wmma_gfx1250_scaled(self, op: Op) -> None: if spec is None: raise NotImplementedError(f"unsupported scaled WMMA op {op.name!r}") op_id = spec.op_id - fmt = spec.matrix_format + fmt0, fmt1 = spec.matrix_formats self._require_wmma_arch(op_id) a, b, c, a_scale, b_scale = op.operands builtin = ( @@ -691,8 +685,9 @@ def _emit_wmma_gfx1250_scaled(self, op: Op) -> None: ) self._emit( f"f32x8 {_name(op.result)} = {builtin}(" - f"{fmt}, {_name(a)}, {fmt}, {_name(b)}, (int16_t)0, {_name(c)}, " - f"0, 0, {_name(a_scale)}, 0, 0, {_name(b_scale)}, false, false);" + f"{fmt0}, {_name(a)}, {fmt1}, {_name(b)}, (int16_t)0, {_name(c)}, " + f"0, {spec.scale_formats[0]}, {_name(a_scale)}, " + f"0, {spec.scale_formats[1]}, {_name(b_scale)}, false, false);" ) def _op_tile_wmma_gfx1250_f32_16x16x32_bf16(self, op: Op) -> None: diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py index 88e82f17efbd..4743a1335ea8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py @@ -608,13 +608,13 @@ def _resolve_llvm_flavor() -> str: "declare <8 x float> @llvm.amdgcn.wmma.f32.16x16x64.bf8.bf8.v8f32.v8i32(" "<8 x i32>, <8 x i32>, i16 immarg, <8 x float>, i1 immarg, i1 immarg)" ), - "wmma.scale.gfx1250.f32.16x16x128.fp8.fp8": ( + "wmma.scale.block32.gfx1250.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32": ( "declare <8 x float> @llvm.amdgcn.wmma.scale.f32.16x16x128.f8f6f4." "v8f32.v16i32.v16i32(i32 immarg, <16 x i32>, i32 immarg, " "<16 x i32>, i16 immarg, <8 x float>, i32 immarg, i32 immarg, i32, " "i32 immarg, i32 immarg, i32, i1 immarg, i1 immarg)" ), - "wmma.scale16.gfx1250.f32.16x16x128.fp8.fp8": ( + "wmma.scale.block16.gfx1250.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32": ( "declare <8 x float> @llvm.amdgcn.wmma.scale16.f32.16x16x128.f8f6f4." "v8f32.v16i32.v16i32(i32 immarg, <16 x i32>, i32 immarg, " "<16 x i32>, i16 immarg, <8 x float>, i32 immarg, i32 immarg, i64, " diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md index e78a6be4c966..6c085c1ad0db 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md @@ -18,11 +18,13 @@ The examples share argument parsing, the spec-driven builder, compiler, launcher, and numerical verifier. Numerical tests use opt-in fixtures in `tests/instances/conftest.py` and are independently runnable by family. -The target-independent aliases in `core/dtypes.py` also recognize `fp4e2m1`, `fp6e2m3`, and -`fp6e3m2` (`fp4`, `fp6`, `bf6`). Recognition does not imply that a target -supports an atom; use its catalog to query support. Packed gfx1250 FP4, -FP6, mixed formats, and alternative scale formats are separate extensions. +The target-independent aliases in `core/dtypes.py` normalize logical format +names. Recognition does not imply that a target supports an atom; use its +catalog to query supported operand formats and shapes. Each example defines +its accepted matrix and scale formats. `e8m0` identifies a scale format carried in bytes and packed integer operands. -It is not a general scalar IR type or conversion API. Operation IDs retain -their instruction-format tokens. +It is not a general scalar IR type or conversion API. Native scaled WMMA uses +one `wmma_scaled` catalog family. Its operation IDs encode the matrix source +and result dtypes and each source's scale format and block size; the backend +selects the LLVM intrinsic and packed operand types from that contract. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py index 17127c707c3f..8dfdb56dd771 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py @@ -14,6 +14,7 @@ from typing import List, Tuple from ...core.dtypes import normalize_dtype +from ...core.arch import ArchTarget, MmaScaleOperand from ...core.arch.wmma_scale import gfx1250_scaled_wmma from ...core.ir import ( BF16, @@ -121,6 +122,21 @@ def resolved_matrix_path(self) -> str: return self.matrix_path +def _native_scaled_atom(spec: BlockScaledGemmSpec, target: ArchTarget): + scale = MmaScaleOperand( + "e8m0" if spec.scale_dtype == "i8" else spec.scale_dtype, spec.block_k + ) + return target.mma.op_for_shape( + family="wmma_scaled", + src_dtypes=(spec.dtype_a, spec.dtype_b, spec.dtype_acc), + dst_dtype=spec.dtype_acc, + src_scales=(scale, scale, None), + m=_BLOCK_M, + n=_BLOCK_N, + k=_WMMA_SCALE_K, + ) + + def is_valid_spec(spec: BlockScaledGemmSpec, arch: str = "gfx1250") -> Tuple[bool, str]: """Return ``(ok, reason)`` for the gfx1250 block-scaled GEMM contract.""" from ...core.arch import ArchTarget @@ -160,15 +176,7 @@ def is_valid_spec(spec: BlockScaledGemmSpec, arch: str = "gfx1250") -> Tuple[boo native_scale = matrix_path in ("wmma_scale", "wmma_scale16") family = matrix_path if native_scale else "wmma" atom_k = _WMMA_SCALE_K if native_scale else _WMMA_K - if native_scale and ( - _canon_lowbit(spec.dtype_a), - _canon_lowbit(spec.dtype_b), - ) not in (("fp8", "fp8"), ("bf8", "bf8")): - return ( - False, - "native gfx1250 SCALE/SCALE16 supports matching fp8 or bf8 operands", - ) - if not target.mma.has_shape( + if not native_scale and not target.mma.has_shape( family=family, a_dtype=_canon_lowbit(spec.dtype_a), b_dtype=_canon_lowbit(spec.dtype_b), @@ -188,21 +196,22 @@ def is_valid_spec(spec: BlockScaledGemmSpec, arch: str = "gfx1250") -> Tuple[boo if spec.layout != "RCR": return False, f"block_scaled_gemm supports RCR only (got {spec.layout!r})" if native_scale: - if spec.scale_dtype not in ("e8m0", "i8"): - return False, ( - "native gfx1250 SCALE/SCALE16 requires packed E8M0 scale bytes " - f"(got {spec.scale_dtype!r})" - ) - scale_op = gfx1250_scaled_wmma( - f"{matrix_path}_f32_16x16x128_{_canon_lowbit(spec.dtype_a)}_{_canon_lowbit(spec.dtype_b)}" - ) - assert scale_op is not None - required_block_k = scale_op.scales.block_k + required_block_k = 16 if matrix_path == "wmma_scale16" else 32 if spec.block_k != required_block_k: return False, ( f"{matrix_path} requires block_k={required_block_k} E8M0 groups " f"(got {spec.block_k})" ) + try: + atom = _native_scaled_atom(spec, target) + except ValueError as exc: + return False, str(exc) + if atom is None: + return ( + False, + "no gfx1250 scaled WMMA atom for the requested operand and scale contract", + ) + else: try: _wire_scale_dtype(spec.scale_dtype) @@ -272,17 +281,17 @@ def _as_f32(b: IRBuilder, v): def build_block_scaled_gemm( spec: BlockScaledGemmSpec, arch: str = "gfx1250" ) -> KernelDef: - """Build a gfx1250 FP8/BF8 block-scaled GEMM (RCR, ``C = A @ B^T``). + """Build a gfx1250 block-scaled GEMM (RCR, ``C = A @ B^T``). One wave (32 lanes) computes one 16x16 output tile without LDS. The legacy ``wmma`` path uses K=64 FP8/BF8 atoms, accumulates each ``block_k`` group, and applies FP16/FP32 A/B scales in software. The native ``wmma_scale`` and - ``wmma_scale16`` paths use K=128 FP8/BF8 atoms and pass packed E8M0 scale bytes + ``wmma_scale16`` paths use K=128 scaled WMMA atoms and pass packed E8M0 scale bytes directly to the instruction, with K=32 and K=16 scale groups respectively. Lane ``l`` owns output column ``l % 16`` and rows ``(l // 16) * 8 : (l // 16 + 1) * 8``. Legacy matrix fragments carry 32 - low-bit bytes per lane as ``<8 x i32>``. Native FP8 fragments carry 64 bytes as + low-bit bytes per lane as ``<8 x i32>``. For FP8/BF8, native fragments carry 64 bytes as ``<16 x i32>`` as four 16-byte K chunks, alternating chunks between lane halves. Both paths use the gfx12 column-distributed ``<8 x f32>`` accumulator layout. @@ -298,14 +307,13 @@ def build_block_scaled_gemm( matrix_path = spec.resolved_matrix_path() native_scale = matrix_path in ("wmma_scale", "wmma_scale16") scale_ty = I8 if native_scale else _scale_type(spec.scale_dtype) - op_id = ( - f"{matrix_path}_f32_16x16x128_{_canon_lowbit(spec.dtype_a)}_{_canon_lowbit(spec.dtype_b)}" - if native_scale - else _wmma_op_id(spec.dtype_a, spec.dtype_b) + atom = ( + _native_scaled_atom(spec, ArchTarget.from_gfx(arch)) if native_scale else None ) + op_id = atom.op_id if atom is not None else _wmma_op_id(spec.dtype_a, spec.dtype_b) scale_op = gfx1250_scaled_wmma(op_id) atom_k = _WMMA_SCALE_K if native_scale else _WMMA_K - frag_words = 16 if native_scale else _ACC + frag_words = atom.srcs[0].frag_len if atom is not None else _ACC a_frag_ty = VectorType(I32, frag_words) groups = spec.K // spec.block_k diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp index af39a12bcf2d..19ba29f2502e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp @@ -19,6 +19,7 @@ #include #include +#include "rocke/arch_target.h" #include "rocke/ir.h" #include "rocke/lower_hip.h" #include "rocke/lower_llvm.h" @@ -793,10 +794,15 @@ void case_gfx1250_scaled_wmma() rocke_value_t* fragment = rocke_b_vec_concat(b, lo, hi); rocke_value_t* c = rocke_b_global_load_vN(b, accum, lane, rocke_f32(), 8, 0); rocke_value_t* scale = rocke_b_global_load(b, scales, lane, v.scale_type(), 1); - rocke_value_t* d = v.scale16 ? rocke_b_wmma_scale16_f32_16x16x128_fp8_fp8( - b, fragment, fragment, c, scale, scale) - : rocke_b_wmma_scale_f32_16x16x128_fp8_fp8( - b, fragment, fragment, c, scale, scale); + const char* src_dtypes[3] = {"fp8", "fp8", "fp32"}; + const int block = v.scale16 ? 16 : 32; + const rocke_mma_scale_operand_t scales_query[3] + = {{"e8m0", block}, {"e8m0", block}, {NULL, 0}}; + const rocke_arch_target_t* target = rocke_arch_target_from_gfx("gfx1250"); + const rocke_mma_op_t* atom = rocke_mma_catalog_op_for_shape_indexed( + &target->mma, "wmma_scaled", src_dtypes, "fp32", scales_query, 16, 16, 128); + rocke_value_t* extra[2] = {scale, scale}; + rocke_value_t* d = rocke_b_mma(b, atom->op_id, fragment, fragment, c, extra, 2); rocke_b_global_store(b, accum, lane, rocke_b_vec_extract(b, d, 0), 1); }; const char* name = v.scale16 ? "wmma_scale16" : "wmma_scale"; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp index ceb797c3d5cc..3a172134c346 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp @@ -4,8 +4,8 @@ * tests/core/mma_frag_ssot.cpp -- host unit test for the IR-layer MMA * frag-length / accumulator-dtype tables consulted by rocke_b_mma. * - * rocke_b_mma sizes its tile.mma result vector as , - * where c_frag_len comes from the op_id frag-length table and acc_elem is i32 + * rocke_b_mma sizes its tile.mma result vector as , + * where dst_frag_len comes from the op_id frag-length table and acc_elem is i32 * for integer WMMA atoms (else f32). This test pins that mapping for a * representative set of atoms and checks the unknown-op_id error path, so a * table edit that changes a result width/dtype is caught here. @@ -16,8 +16,10 @@ */ #include #include +#include #include "rocke/arch_target.h" +#include "rocke/error.hpp" #include "rocke/ir.h" static int g_failures = 0; @@ -60,7 +62,9 @@ static void check_atom(rocke_ir_builder_t* b, const char* op_id, int expect_frag } } -static void check_catalog_fragments(const rocke_mma_catalog_t* catalog, const char* op_id) +static void check_catalog_fragments(const rocke_mma_catalog_t* catalog, + const char* op_id, + int scale_block_size) { const rocke_mma_op_t* op = rocke_mma_catalog_by_op_id(catalog, op_id); CHECK(op != NULL, op_id); @@ -68,13 +72,203 @@ static void check_catalog_fragments(const rocke_mma_catalog_t* catalog, const ch { return; } - CHECK(op->a_frag_len == 16, op_id); - CHECK(op->b_frag_len == 16, op_id); - CHECK(op->c_frag_len == 8, op_id); + CHECK(op->srcs[0].frag_len == 16, op_id); + CHECK(op->srcs[1].frag_len == 16, op_id); + CHECK(op->srcs[2].frag_len == 8, op_id); + CHECK(op->dst.frag_len == 8, op_id); + for(int i = 0; i < 2; ++i) + { + CHECK(strcmp(op->srcs[i].dtype, "fp8e4m3") == 0, op_id); + CHECK(op->srcs[i].scale_dtype != NULL, op_id); + if(op->srcs[i].scale_dtype) + { + CHECK(strcmp(op->srcs[i].scale_dtype, "e8m0") == 0, op_id); + } + CHECK(op->srcs[i].scale_block_size == scale_block_size, op_id); + } + CHECK(op->srcs[2].scale_dtype == NULL && op->srcs[2].scale_block_size == 0, op_id); +} + +static void check_contract_queries() +{ + rocke_mma_op_t rows[10] = {}; + rows[0].family = "wmma_scaled"; + rows[0].srcs[0] = {"fp8e4m3", 16, NULL, "e8m0", 32}; + rows[0].srcs[1] = {"bf8e5m2", 16, NULL, "e8m0", 32}; + rows[0].srcs[2] = {"fp32", 8, NULL, NULL, 0}; + rows[0].dst = {"fp32", 8, NULL}; + rows[0].m = rows[0].n = 16; + rows[0].k = 128; + for(int i = 1; i < 10; ++i) + rows[i] = rows[0]; + for(int src = 0; src < 2; ++src) + { + rows[1 + src * 3].srcs[src].scale_dtype = "e4m3"; + rows[2 + src * 3].srcs[src].scale_block_size = 16; + rows[3 + src * 3].srcs[src].scale_dtype = NULL; + rows[3 + src * 3].srcs[src].scale_block_size = 0; + } + rows[7].srcs[2].dtype = "fp16"; + rows[8].dst.dtype = "fp16"; + for(int i = 0; i < 2; ++i) + { + rows[9].srcs[i].scale_dtype = NULL; + rows[9].srcs[i].scale_block_size = 0; + } + const rocke_mma_catalog_t catalog = {rows, 10}; + char ids[10][256]; + for(int i = 0; i < 10; ++i) + { + rocke_mma_op_t* row = &rows[i]; + CHECK(rocke_mma_op_semantic_id(row, ids[i], sizeof(ids[i])) != NULL, "format contract ID"); + row->op_id = ids[i]; + for(int j = 0; j < i; ++j) + CHECK(strcmp(ids[i], ids[j]) != 0, "different contracts have different IDs"); + const char* dtypes[3] = {"fp8", "bf8", row->srcs[2].dtype}; + rocke_mma_scale_operand_t scales[3]; + for(int j = 0; j < 3; ++j) + scales[j] = {row->srcs[j].scale_dtype, row->srcs[j].scale_block_size}; + // Exercise the scale-format alias through every query operation. + if(i == 1) + scales[0].dtype = "fp8e4m3"; + const rocke_mma_op_t* out[10]; + CHECK(rocke_mma_catalog_enumerate_indexed( + &catalog, row->family, dtypes, row->dst.dtype, scales, 16, 16, out, 10) + == 1 + && out[0] == row, + "enumeration distinguishes every operand contract"); + CHECK(rocke_mma_catalog_has_shape_indexed( + &catalog, row->family, dtypes, row->dst.dtype, scales, 16, 16, 128), + "full contract exists"); + CHECK(rocke_mma_catalog_op_for_shape_indexed( + &catalog, row->family, dtypes, row->dst.dtype, scales, 16, 16, 128) + == row, + "exact contract selection"); + CHECK(rocke_mma_catalog_select_largest_k_indexed( + &catalog, row->family, dtypes, row->dst.dtype, scales, 16, 16, -1) + == row, + "largest K contract selection"); + } + const char* dtypes[3] = {"fp8", "bf8", "fp32"}; + CHECK(rocke_mma_catalog_enumerate_indexed( + &catalog, "wmma_scaled", dtypes, "fp32", NULL, 16, 16, NULL, 0) + == 8, + "omitted scales are wildcard"); + for(bool largest : {false, true}) + { + bool rejected = false; + try + { + if(largest) + rocke_mma_catalog_select_largest_k_indexed( + &catalog, "wmma_scaled", dtypes, "fp32", NULL, 16, 16, -1); + else + rocke_mma_catalog_op_for_shape_indexed( + &catalog, "wmma_scaled", dtypes, "fp32", NULL, 16, 16, 128); + } + catch(const ckc::Error&) + { + rejected = true; + } + CHECK(rejected, "underspecified selection rejects ambiguity"); + } + rocke_mma_scale_operand_t scales[3] = {{"e5m3", 32}, {"e5m3", 32}, {NULL, 0}}; + CHECK(rocke_mma_catalog_op_for_shape_indexed( + &catalog, "wmma_scaled", dtypes, "fp32", scales, 16, 16, 128) + == NULL, + "unsupported contract absent"); } int main(void) { + check_contract_queries(); + const char* cpu_targets[] = {"gfx950", "gfx1250"}; + for(const char* gfx : cpu_targets) + { + const rocke_arch_target_t* target = rocke_arch_target_from_gfx(gfx); + CHECK(target != NULL, "CPU catalog target exists"); + if(!target) + { + continue; + } + for(int i = 0; i < target->mma.num_ops; ++i) + { + const rocke_mma_op_t* op = &target->mma.ops[i]; + const char* src_dtypes[3] = {op->srcs[0].dtype, op->srcs[1].dtype, op->srcs[2].dtype}; + rocke_mma_scale_operand_t scales[3]; + for(int j = 0; j < 3; ++j) + scales[j] = {op->srcs[j].scale_dtype, op->srcs[j].scale_block_size}; + if(strcmp(op->family, "wmma_scaled") == 0) + { + char id[256]; + CHECK(rocke_mma_op_semantic_id(op, id, sizeof(id)) != NULL, "semantic ID fits"); + CHECK(strcmp(id, op->op_id) == 0, "semantic ID agrees with catalog"); + } + CHECK(rocke_mma_catalog_op_for_shape_indexed(&target->mma, + op->family, + src_dtypes, + op->dst.dtype, + scales, + op->m, + op->n, + op->k) + != NULL, + "indexed query resolves CPU catalog key"); + } + } + + { + rocke_mma_op_t distinct = {}; + const rocke_layout_map_t src2_layout = {ROCKE_MMA_ROLE_SRC2, 3, 32, NULL}; + const rocke_layout_map_t dst_layout = {ROCKE_MMA_ROLE_DST, 7, 32, NULL}; + distinct.family = "mma"; + distinct.srcs[0].dtype = "xf32"; + distinct.srcs[1].dtype = "xf32"; + distinct.srcs[0].scale_dtype = "e8m0"; + distinct.srcs[0].scale_block_size = 32; + distinct.srcs[1].scale_dtype = "e4m3"; + distinct.srcs[1].scale_block_size = 16; + distinct.srcs[2].dtype = "fp32"; + distinct.srcs[2].frag_len = 3; + distinct.srcs[2].layout = &src2_layout; + distinct.dst.dtype = "i32"; + distinct.dst.frag_len = 7; + distinct.dst.layout = &dst_layout; + distinct.m = 16; + distinct.n = 16; + distinct.k = 8; + distinct.op_id = "synthetic_distinct_dst"; + const rocke_mma_catalog_t catalog = {&distinct, 1}; + const char* src_dtypes[3] = {"xf32", "xf32", "fp32"}; + CHECK(rocke_mma_catalog_op_for_shape_indexed( + &catalog, "mma", src_dtypes, "i32", NULL, 16, 16, 8) + == &distinct, + "indexed query distinguishes src2 and dst"); + CHECK(rocke_mma_catalog_op_for_shape(&catalog, "mma", "xf32", "xf32", "fp32", 16, 16, 8) + == NULL, + "legacy query defaults dst to src2"); + CHECK(rocke_mma_op_src_layout(&distinct, 2, NULL) == &src2_layout, + "indexed src2 layout remains independent"); + CHECK(rocke_mma_op_dst_layout(&distinct, NULL) == &dst_layout, + "dst layout remains independent"); + CHECK(rocke_mma_op_c_layout(&distinct, NULL) == &dst_layout, + "historical C layout projects dst"); + const rocke_mma_op_t* selected + = rocke_mma_catalog_by_op_id(&catalog, "synthetic_distinct_dst"); + CHECK(selected != NULL, "catalog preserves scaled operand metadata"); + if(selected) + { + CHECK(strcmp(selected->srcs[0].scale_dtype, "e8m0") == 0 + && selected->srcs[0].scale_block_size == 32, + "src0 scale retains its value format and block size"); + CHECK(strcmp(selected->srcs[1].scale_dtype, "e4m3") == 0 + && selected->srcs[1].scale_block_size == 16, + "src1 scale is independent of src0"); + CHECK(selected->srcs[2].scale_dtype == NULL && selected->srcs[2].scale_block_size == 0, + "src2 remains unscaled"); + } + } + rocke_ir_builder_t b; if(rocke_ir_builder_init(&b, "rocke_mma_frag_ssot") != ROCKE_OK) { @@ -99,8 +293,14 @@ int main(void) CHECK(gfx1250 != NULL, "gfx1250 target"); if(gfx1250) { - check_catalog_fragments(&gfx1250->mma, "wmma_scale_f32_16x16x128_fp8_fp8"); - check_catalog_fragments(&gfx1250->mma, "wmma_scale16_f32_16x16x128_fp8_fp8"); + check_catalog_fragments( + &gfx1250->mma, + "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32", + 32); + check_catalog_fragments( + &gfx1250->mma, + "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32", + 16); } /* Unknown op_id must be rejected. The engine's error path either returns diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py index fe55778e8f03..edce7ca18915 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py @@ -1,12 +1,11 @@ # Copyright (c) Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT -"""SSOT guards for the bare-op_id MMA accumulator-dtype lookup. +"""SSOT guards for indexed MMA ``srcs`` and ``dst`` metadata. -``IRBuilder.mma`` uses ``target._op_id_c_dtype()`` to size a ``tile.mma`` result -vector's accumulator element without an ``ArchTarget`` in hand. These tests pin -the first-wins / raise-on-drift contract of that lookup so it stays deterministic -across the arches that list a given op_id. +``IRBuilder.mma`` uses ``dst`` metadata to size a ``tile.mma`` result. These +tests pin the ``dst`` lookup, indexed catalog query, and optional per-source +scale parsing without assuming ``src2`` and ``dst`` are always identical. """ from __future__ import annotations @@ -15,49 +14,252 @@ from unittest import mock from rocke.core.arch.target import ( + ArchTarget, + MmaCatalog, + MmaOp, + MmaDst, + MmaScaleOperand, + MmaSrc, + _FragInfo, + _FragOperand, + _MMA_FRAGMENT_INFO, + _build_mma_op, _load_specs, - _op_id_c_dtype, + _op_id_dst_dtype, normalize_dtype, ) +from rocke.core.ir import IRBuilder -class TestOpIdCDtype(unittest.TestCase): +class TestOpIdDstDtype(unittest.TestCase): def test_matches_catalog_first_hit(self): - # Every op_id in the catalog resolves to its normalized accumulator dtype, + # Every op_id in the catalog resolves to its normalized dst dtype, # taking the first arch that lists it (dict preserves catalog order). expected: dict = {} for row in _load_specs().values(): for o in row["mma"]: - expected.setdefault(o["op_id"], normalize_dtype(o["c"])) - self.assertEqual(_op_id_c_dtype(), expected) + expected.setdefault(o["op_id"], normalize_dtype(o["dst"]["dtype"])) + self.assertEqual(_op_id_dst_dtype(), expected) - def test_c_dtype_invariant_across_arches(self): - # The whole premise of the bare-op_id lookup: an op_id's accumulator dtype + def test_dst_dtype_invariant_across_arches(self): + # The whole premise of the bare-op_id lookup: an op_id's dst dtype # is invariant across the arches that list it, so building the map must not # raise on the real catalog. (The raise path is exercised below.) try: - _op_id_c_dtype() + _op_id_dst_dtype() except ValueError as exc: # pragma: no cover - only hit on real drift - self.fail(f"_op_id_c_dtype() raised on the shipped catalog: {exc}") + self.fail(f"_op_id_dst_dtype() raised on the shipped catalog: {exc}") def test_raises_on_cross_arch_disagreement(self): specs = _load_specs() - # Find an op_id and clone its row into a fake arch with a different c dtype. + # Find an op_id and clone its row into a fake arch with a different dst dtype. sample = next(o for row in specs.values() for o in row["mma"]) - original_c = normalize_dtype(sample["c"]) - other_c = "i32" if original_c != "i32" else "f32" + original_dst = normalize_dtype(sample["dst"]["dtype"]) + other_dst = "i32" if original_dst != "i32" else "f32" clash = dict(sample) - clash["c"] = other_c + clash["dst"] = {"dtype": other_dst} drifted = dict(specs) drifted["_synthetic_drift"] = {"mma": [clash]} - _op_id_c_dtype.cache_clear() + _op_id_dst_dtype.cache_clear() try: with mock.patch("rocke.core.arch.target._load_specs", return_value=drifted): with self.assertRaises(ValueError): - _op_id_c_dtype() + _op_id_dst_dtype() finally: - _op_id_c_dtype.cache_clear() + _op_id_dst_dtype.cache_clear() + + +class TestIndexedMmaOperands(unittest.TestCase): + def test_cpu_catalog_indexed_queries_cover_gfx950_and_gfx1250(self): + for gfx in ("gfx950", "gfx1250"): + catalog = ArchTarget.from_gfx(gfx).mma + for op in catalog.ops: + with self.subTest(gfx=gfx, op_id=op.op_id): + self.assertEqual(len(op.srcs), 3) + self.assertIn( + op, + catalog.enumerate( + family=op.family, + src_dtypes=tuple(src.dtype for src in op.srcs), + dst_dtype=op.dst.dtype, + m=op.m, + n=op.n, + ), + ) + + def test_indexed_query_distinguishes_src2_and_dst(self): + distinct = MmaOp( + family="mma", + srcs=(MmaSrc("xf32"), MmaSrc("xf32"), MmaSrc("fp32")), + dst=MmaDst("i32", frag_len=7), + m=16, + n=16, + k=8, + op_id="synthetic_distinct_dst", + ) + catalog = MmaCatalog([distinct]) + self.assertEqual( + catalog.enumerate(src_dtypes=("xf32", "xf32", "fp32"), dst_dtype="i32"), + [distinct], + ) + self.assertEqual( + catalog.enumerate(a_dtype="xf32", b_dtype="xf32", c_dtype="fp32"), + [], + ) + + builder = IRBuilder("distinct_dst") + value = builder.const_i32(0) + result = builder.mma(distinct, value, value, value) + self.assertEqual(result.type.count, distinct.dst.frag_len) + self.assertEqual(result.type.elem.name, "i32") + + def test_fragment_metadata_distinguishes_src2_and_dst(self): + def src2_coord(builder, lane, slot): + return builder.const_i32(slot), lane + + def dst_coord(builder, lane, slot): + return lane, builder.const_i32(slot) + + op_id = "synthetic_distinct_fragment_metadata" + info = _FragInfo( + srcs=( + _FragOperand(1), + _FragOperand(1), + _FragOperand(3, src2_coord), + ), + dst=_FragOperand(7, dst_coord), + wave_size=32, + ) + row = { + "family": "mma", + "srcs": [ + {"dtype": "xf32"}, + {"dtype": "xf32"}, + {"dtype": "fp32"}, + ], + "dst": {"dtype": "i32"}, + "m": 16, + "n": 16, + "k": 8, + "op_id": op_id, + } + + with mock.patch.dict(_MMA_FRAGMENT_INFO, {op_id: info}): + op = _build_mma_op(row) + + self.assertEqual(op.srcs[2].frag_len, 3) + self.assertEqual(op.dst.frag_len, 7) + self.assertIs(op.src_layout(2).fn, src2_coord) + self.assertIs(op.dst_layout().fn, dst_coord) + self.assertEqual(op.c_frag_len, op.dst.frag_len) + self.assertIs(op.c_layout(), op.dst_layout()) + + builder = IRBuilder("distinct_fragment_metadata") + value = builder.const_i32(0) + result = builder.mma(op, value, value, value) + self.assertEqual(result.type.count, op.dst.frag_len) + self.assertEqual(result.type.elem.name, "i32") + + def test_scaled_wmma_catalog_preserves_scale_contracts(self): + catalog = ArchTarget.from_gfx("gfx1250").mma + for family, block_size in (("wmma_scale", 32), ("wmma_scale16", 16)): + with self.subTest(family=family): + scale = MmaScaleOperand("e8m0", block_size) + op = catalog.op_for_shape( + family="wmma_scaled", + src_dtypes=("fp8", "fp8", "fp32"), + dst_dtype="fp32", + src_scales=(scale, scale, None), + m=16, + n=16, + k=128, + ) + self.assertIsNotNone(op) + self.assertEqual(op.family, "wmma_scaled") + self.assertEqual(op.shape, (16, 16, 128)) + for src in op.srcs[:2]: + self.assertEqual(src.dtype, "fp8e4m3") + self.assertEqual(src.frag_len, 16) + self.assertEqual(src.scale, MmaScaleOperand("e8m0", block_size)) + self.assertIsNone(op.srcs[2].scale) + self.assertEqual(op.srcs[2].frag_len, 8) + self.assertEqual(op.dst.frag_len, 8) + self.assertEqual(op.src_layout(2).role, "src2") + self.assertEqual(op.dst_layout().role, "dst") + + def test_scale_format_and_block_size_are_independent_per_source(self): + # Synthetic rows exercise the descriptor without admitting new hardware + # operations into the shipped catalog. + for dtype, block_size in ( + ("e8m0", 32), + ("e8m0", 16), + ("e4m3", 16), + ("fp8e4m3", 16), + ("e5m3", 16), + ): + with self.subTest(dtype=dtype, block_size=block_size): + op = _build_mma_op( + { + "family": "mma", + "srcs": [ + { + "dtype": "fp4", + "scale": {"dtype": dtype, "block_size": block_size}, + }, + { + "dtype": "fp4", + "scale": {"dtype": "e8m0", "block_size": 32}, + }, + {"dtype": "fp32"}, + ], + "dst": {"dtype": "fp32"}, + "m": 16, + "n": 16, + "k": 128, + "op_id": "synthetic_scaled_mma", + } + ) + expected_dtype = "e4m3" if dtype == "fp8e4m3" else dtype + self.assertEqual( + op.srcs[0].scale, MmaScaleOperand(expected_dtype, block_size) + ) + self.assertEqual(op.srcs[1].scale, MmaScaleOperand("e8m0", 32)) + self.assertIsNone(op.srcs[2].scale) + self.assertFalse(hasattr(op.dst, "scale")) + + def test_scale_dtype_must_be_a_supported_value_format(self): + for dtype in ("i32", "fp4", "e5m2", "", None): + with self.subTest(dtype=dtype): + with self.assertRaisesRegex(ValueError, "e8m0, e4m3, or e5m3"): + MmaScaleOperand(dtype, 16) + + def test_scale_dtype_alias_is_canonicalized(self): + for block_size in (16, 32): + with self.subTest(block_size=block_size): + alias = MmaScaleOperand("fp8e4m3", block_size) + canonical = MmaScaleOperand("e4m3", block_size) + self.assertEqual(alias.dtype, "e4m3") + self.assertEqual(alias, canonical) + self.assertEqual(hash(alias), hash(canonical)) + + def test_scale_block_size_must_be_16_or_32(self): + for block_size in (0, -16, 1, 8, 64, 16.0, 32.0, 16.5, "16", True, None): + with self.subTest(block_size=block_size): + with self.assertRaisesRegex(ValueError, "integer equal to 16 or 32"): + MmaScaleOperand("e8m0", block_size) + + def test_scale_identity_includes_format_and_block_size(self): + self.assertEqual( + len( + { + MmaScaleOperand("e8m0", 32), + MmaScaleOperand("e8m0", 16), + MmaScaleOperand("e4m3", 16), + } + ), + 3, + ) if __name__ == "__main__": # pragma: no cover diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py index 2d43a966553e..0fbf5d7b6f3f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py @@ -6,8 +6,9 @@ import unittest -from rocke.core.arch import ArchTarget +from rocke.core.arch import ArchTarget, MmaScaleOperand from rocke.core.ir import F32, I32, I64, IRBuilder, PtrType +from rocke.core.ir_serialize import parse, serialize from rocke.core.lower_hip import lower_kernel_to_hip from rocke.core.lower_llvm import lower_kernel_to_llvm from rocke.instances.gfx1250.block_scaled_gemm import ( @@ -18,6 +19,21 @@ ) +def _scaled_atom(dtype="fp8", scale16=False): + scale = MmaScaleOperand("e8m0", 16 if scale16 else 32) + atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( + family="wmma_scaled", + src_dtypes=(dtype, dtype, "fp32"), + dst_dtype="fp32", + src_scales=(scale, scale, None), + m=16, + n=16, + k=128, + ) + assert atom is not None + return atom + + def _build_scaled_atom(*, scale16: bool): b = IRBuilder("gfx1250_scaled_wmma") matrix = b.param("matrix", PtrType(I32, "global"), readonly=True) @@ -30,20 +46,73 @@ def _build_scaled_atom(*, scale16: bool): fragment = b.vec_concat(lo, hi) c = b.global_load_vN(accum, lane, F32, 8) scale = b.global_load(scales, lane, scale_ty) - if scale16: - d = b.wmma_scale16_f32_16x16x128_fp8_fp8(fragment, fragment, c, scale, scale) - else: - d = b.wmma_scale_f32_16x16x128_fp8_fp8(fragment, fragment, c, scale, scale) + d = b.mma(_scaled_atom(scale16=scale16), fragment, fragment, c, scale, scale) b.global_store(accum, lane, b.vec_extract(d, 0)) return b.kernel class TestGfx1250ScaledWmma(unittest.TestCase): + def test_matrix_formats_share_intrinsic_declarations(self): + b = IRBuilder("shared_scaled_wmma_declarations") + matrix = b.param("matrix", PtrType(I32, "global"), readonly=True) + accum = b.param("accum", PtrType(F32, "global")) + lane = b.thread_id_x() + lo = b.global_load_vN(matrix, lane, I32, 8) + hi = b.global_load_vN(matrix, b.add(lane, b.const_i32(8)), I32, 8) + fragment = b.vec_concat(lo, hi) + c = b.global_load_vN(accum, lane, F32, 8) + for mode, scale_ty in (("scale", I32), ("scale16", I64)): + scale_ptr = b.param( + f"{mode}_ptr", PtrType(scale_ty, "global"), readonly=True + ) + scale = b.global_load(scale_ptr, lane, scale_ty) + for dtype in ("fp8", "bf8"): + c = b.mma( + _scaled_atom(dtype, mode == "scale16"), + fragment, + fragment, + c, + scale, + scale, + ) + b.global_store(accum, lane, b.vec_extract(c, 0)) + llvm = lower_kernel_to_llvm(b.kernel, arch="gfx1250", llvm_flavor="llvm23") + for mode, scale_ty in (("scale", "i32"), ("scale16", "i64")): + intrinsic = ( + f"llvm.amdgcn.wmma.{mode}.f32.16x16x128.f8f6f4.v8f32.v16i32.v16i32" + ) + with self.subTest(mode=mode): + self.assertEqual(llvm.count(f"declare <8 x float> @{intrinsic}("), 1) + calls = [ + line + for line in llvm.splitlines() + if f"call <8 x float> @{intrinsic}(" in line + ] + self.assertEqual(len(calls), 2) + for selector, call in enumerate(calls): + self.assertEqual(call.count(f"i32 {selector}, <16 x i32>"), 2) + self.assertEqual(call.count(f", {scale_ty} %"), 2) + + def test_semantic_ids_survive_serialization(self): + for scale16 in (False, True): + kernel = _build_scaled_atom(scale16=scale16) + text = serialize(kernel) + self.assertIn(_scaled_atom(scale16=scale16).op_id, text) + restored = parse(text) + self.assertEqual( + lower_kernel_to_llvm(kernel, arch="gfx1250", llvm_flavor="llvm23"), + lower_kernel_to_llvm(restored, arch="gfx1250", llvm_flavor="llvm23"), + ) + self.assertEqual( + lower_kernel_to_hip(kernel, arch="gfx1250"), + lower_kernel_to_hip(restored, arch="gfx1250"), + ) + def test_catalog_fragment_lengths(self): target = ArchTarget.from_gfx("gfx1250") for op_id in ( - "wmma_scale_f32_16x16x128_fp8_fp8", - "wmma_scale16_f32_16x16x128_fp8_fp8", + "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32", + "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32", ): with self.subTest(op_id=op_id): op = target.mma.by_op_id(op_id) @@ -108,6 +177,27 @@ def test_scale_and_scale16_hip_builtin_lowering(self): ) self.assertIn(builtin, hip) + def test_scaled_hip_dispatch_preserves_neutral_and_concrete_ops(self): + for scale16 in (False, True): + mode = "wmma_scale16" if scale16 else "wmma_scale" + for dtype, selector in (("fp8", 0), ("bf8", 1)): + with self.subTest(mode=mode, dtype=dtype): + kernel = _build_scaled_atom(scale16=scale16) + call = next(op for op in kernel.body.ops if op.name == "tile.mma") + op_id = _scaled_atom(dtype, scale16).op_id + call.attrs["op_id"] = op_id + neutral = lower_kernel_to_hip(kernel, arch="gfx1250") + self.assertIn( + f"__builtin_amdgcn_{mode}_f32_16x16x128_f8f6f4({selector},", + neutral, + ) + call.name = f"tile.{call.attrs.pop('op_id')}" + self.assertEqual( + neutral, lower_kernel_to_hip(kernel, arch="gfx1250") + ) + with self.assertRaisesRegex(NotImplementedError, "not available"): + lower_kernel_to_hip(kernel, arch="gfx942") + def test_native_block_scaled_gemm_uses_packed_e8m0_in_instruction(self): for matrix_path, block_k, scale_ty, fragment_load, load_count in ( ("wmma_scale", 32, "i32", "load <16 x i8>", 8), @@ -181,7 +271,7 @@ def test_native_slice_rejects_unsupported_contracts(self): ) ok, why = is_valid_spec(bad_dtype) self.assertFalse(ok) - self.assertIn("matching fp8 or bf8 operands", why) + self.assertIn("requested operand and scale contract", why) bad_block = BlockScaledGemmSpec( name="bad_block", @@ -207,7 +297,9 @@ def test_unknown_scaled_atom_reports_supported_operations(self): NotImplementedError, "not yet wired for gfx1250" ) as error: lower_kernel_to_llvm(kernel, arch="gfx1250", llvm_flavor="llvm23") - self.assertIn(f"{mode}_f32_16x16x128_bf8_bf8", str(error.exception)) + self.assertIn(_scaled_atom("bf8", scale16).op_id, str(error.exception)) + with self.assertRaisesRegex(NotImplementedError, "no HIP lowering"): + lower_kernel_to_hip(kernel, arch="gfx1250") if __name__ == "__main__": diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py index ce77824d5806..3936546309b4 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py @@ -32,6 +32,7 @@ def test_alias_catalog_lookup(alias, canonical): found = True assert catalog.has_shape( family=op.family, + src_scales=tuple(src.scale for src in op.srcs), a_dtype=alias, b_dtype=op.b_dtype, c_dtype=op.c_dtype, @@ -39,7 +40,7 @@ def test_alias_catalog_lookup(alias, canonical): n=op.n, k=op.k, ) - # BF6 is only present on the branch that supplies its packed atom. + # BF6 name recognition does not require a corresponding catalog atom. if alias != "bf6": assert found diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp index 927b4b2a912d..0b9cb310a1f5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp @@ -34,7 +34,15 @@ int main() for(int i = 0; i < arch->mma.num_ops; ++i) { const rocke_mma_op_t* op = &arch->mma.ops[i]; - for(const char* dtype : {op->a_dtype, op->b_dtype, op->c_dtype}) + // K=64 low-bit WMMA packs 32 bytes into eight i32 registers per source. + if(strcmp(gfx, "gfx1250") == 0 && op->k == 64 && strcmp(op->family, "wmma") == 0 + && (op->srcs[0].frag_len != 8 || op->srcs[1].frag_len != 8)) + { + fprintf(stderr, "%s: fragment length must count packed i32 elements\n", op->op_id); + return 1; + } + for(const char* dtype : + {op->srcs[0].dtype, op->srcs[1].dtype, op->srcs[2].dtype, op->dst.dtype}) { char scratch[64]; if(strcmp(dtype, rocke_normalize_dtype(dtype, scratch, sizeof(scratch))) != 0) @@ -43,20 +51,49 @@ int main() return 1; } } - for(const char* a : {op->a_dtype, short_dtype(op->a_dtype)}) + for(const char* a : {op->srcs[0].dtype, short_dtype(op->srcs[0].dtype)}) { - for(const char* b : {op->b_dtype, short_dtype(op->b_dtype)}) + for(const char* b : {op->srcs[1].dtype, short_dtype(op->srcs[1].dtype)}) { - if(!rocke_mma_catalog_has_shape( - &arch->mma, op->family, a, b, op->c_dtype, op->m, op->n, op->k) - || rocke_mma_catalog_op_for_shape( - &arch->mma, op->family, a, b, op->c_dtype, op->m, op->n, op->k) + const char* src_dtypes[3] = {a, b, op->srcs[2].dtype}; + rocke_mma_scale_operand_t src_scales[3]; + for(int j = 0; j < 3; ++j) + src_scales[j] = {op->srcs[j].scale_dtype, op->srcs[j].scale_block_size}; + if(!rocke_mma_catalog_has_shape_indexed(&arch->mma, + op->family, + src_dtypes, + op->dst.dtype, + src_scales, + op->m, + op->n, + op->k) + || rocke_mma_catalog_op_for_shape_indexed(&arch->mma, + op->family, + src_dtypes, + op->dst.dtype, + src_scales, + op->m, + op->n, + op->k) != op - || rocke_mma_catalog_select_largest_k( - &arch->mma, op->family, a, b, op->c_dtype, op->m, op->n, op->k) + || rocke_mma_catalog_select_largest_k_indexed(&arch->mma, + op->family, + src_dtypes, + op->dst.dtype, + src_scales, + op->m, + op->n, + op->k) != op - || rocke_mma_catalog_enumerate( - &arch->mma, op->family, a, b, op->c_dtype, op->m, op->n, NULL, 0) + || rocke_mma_catalog_enumerate_indexed(&arch->mma, + op->family, + src_dtypes, + op->dst.dtype, + src_scales, + op->m, + op->n, + NULL, + 0) < 1) { fprintf(stderr, "%s: catalog query missed %s/%s\n", op->op_id, a, b); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_contracts.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_contracts.py new file mode 100644 index 000000000000..daf5b0f487a8 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_contracts.py @@ -0,0 +1,121 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Selection must distinguish logical operands even when carriers agree.""" + +from dataclasses import replace + +import pytest + +from rocke.core.arch import ( + ArchTarget, + MmaCatalog, + MmaDst, + MmaOp, + MmaScaleOperand, + MmaSrc, +) +from rocke.core.arch.wmma_scale import gfx1250_scaled_wmma + + +def _contracts(): + base = MmaOp( + family="wmma_scaled", + srcs=( + MmaSrc("fp8", scale=MmaScaleOperand("e8m0", 32)), + MmaSrc("bf8", scale=MmaScaleOperand("e8m0", 32)), + MmaSrc("fp32"), + ), + dst=MmaDst("fp32"), + m=16, + n=16, + k=128, + op_id="fixture", + ) + rows = [base] + for index in (0, 1): + for scale in (MmaScaleOperand("e4m3", 32), MmaScaleOperand("e8m0", 16), None): + srcs = list(base.srcs) + srcs[index] = replace(srcs[index], scale=scale) + rows.append(replace(base, srcs=tuple(srcs))) + rows.append(replace(base, srcs=(*base.srcs[:2], MmaSrc("fp16")))) + rows.append(replace(base, dst=MmaDst("fp16"))) + rows.append( + replace(base, srcs=tuple(replace(src, scale=None) for src in base.srcs)) + ) + return [replace(row, op_id=row.semantic_id()) for row in rows] + + +def test_full_contract_queries_distinguish_each_source_scale_and_result(): + rows = _contracts() + assert len({row.op_id for row in rows}) == len(rows) + catalog = MmaCatalog(rows) + for row in rows: + query = dict( + family=row.family, + src_dtypes=tuple(src.dtype for src in row.srcs), + dst_dtype=row.dst.dtype, + src_scales=tuple(src.scale for src in row.srcs), + m=row.m, + n=row.n, + ) + assert catalog.enumerate(**query) == [row] + assert catalog.has_shape(**query, k=row.k) + assert catalog.op_for_shape(**query, k=row.k) is row + assert catalog.select_largest_k(**query) is row + assert catalog.select_largest_k(**query, k_max=64) is None + + +def test_partial_query_rejects_ambiguity_but_enumeration_and_existence_are_valid(): + catalog = MmaCatalog(_contracts()) + query = dict( + family="wmma_scaled", + src_dtypes=("fp8", "bf8", "fp32"), + dst_dtype="fp32", + m=16, + n=16, + ) + assert len(catalog.enumerate(**query)) == 8 + assert catalog.has_shape(**query, k=128) + with pytest.raises(ValueError, match="ambiguous MMA query"): + catalog.op_for_shape(**query, k=128) + with pytest.raises(ValueError, match="ambiguous MMA query"): + catalog.select_largest_k(**query) + assert len(catalog.enumerate(**query, src_scales=(None, None, None))) == 1 + scale = MmaScaleOperand("e5m3", 32) + assert catalog.op_for_shape(**query, src_scales=(scale, scale, None), k=128) is None + with pytest.raises(ValueError, match="exactly 3"): + catalog.enumerate(**query, src_scales=(None, None)) + + +def test_scale_alias_selects_the_same_contract(): + catalog = MmaCatalog(_contracts()) + query = dict( + family="wmma_scaled", + src_dtypes=("fp8", "bf8", "f32"), + dst_dtype="fp32", + m=16, + n=16, + k=128, + ) + scales = (MmaScaleOperand("fp8e4m3", 32), MmaScaleOperand("e8m0", 32), None) + row = catalog.op_for_shape(**query, src_scales=scales) + assert row is not None and row.srcs[0].scale.dtype == "e4m3" + + +def test_scaled_catalog_identity_and_backend_contract(): + catalog = ArchTarget.from_gfx("gfx1250").mma + rows = [row for row in catalog.ops if row.family == "wmma_scaled"] + assert len(rows) == 4 + for row in rows: + assert row.op_id == row.semantic_id() + packing = gfx1250_scaled_wmma(row.op_id) + assert packing.atom is row + assert packing.matrix_formats == ( + (0, 0) if row.srcs[0].dtype == "fp8e4m3" else (1, 1) + ) + assert packing.scales.count * packing.scales.block_k == row.k + assert packing.matrix_llvm_types == ("<16 x i32>", "<16 x i32>") + for family in ("wmma_scale", "wmma_scale16"): + old_id = f"{family}_f32_16x16x128_fp8_fp8" + assert catalog.by_op_id(old_id) is None + assert gfx1250_scaled_wmma(old_id) is None diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py index eeb6fa6b1bf4..3758d86ebf01 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py @@ -7,10 +7,10 @@ int-accumulator tables. ``IRBuilder.mma`` sizes a ``tile.mma`` result vector from a bare ``op_id`` string via three pieces: -* ``_mma_c_frag_len`` — accumulator fragment length, resolved from the arch SSOT +* ``_mma_dst_frag_len`` — ``dst`` fragment length, resolved from the arch SSOT ``core.arch.target._MMA_FRAGMENT_INFO`` (no ir-side copy); -* ``_mma_c_is_int`` — whether the atom accumulates in i32, resolved from the JSON - catalog accumulator dtype (``target._op_id_c_dtype``); +* ``_mma_dst_is_int`` — whether the atom produces i32, resolved from the JSON + catalog ``dst`` dtype (``target._op_id_dst_dtype``); * ``_MMA_RESULT_HINT`` — the ir-side SSA result-name hints (naming only, kept in ir.py to preserve byte-identical value numbering). @@ -26,20 +26,20 @@ import unittest -from rocke.core.arch.target import _MMA_FRAGMENT_INFO, _op_id_c_dtype +from rocke.core.arch.target import _MMA_FRAGMENT_INFO, _op_id_dst_dtype from rocke.core.ir import ( _MMA_RESULT_HINT, - _mma_c_frag_len, - _mma_c_is_int, + _mma_dst_frag_len, + _mma_dst_is_int, ) def _sizable_op_ids(): - """Op_ids the SSOT can actually size (positive accumulator frag length).""" + """Op_ids the SSOT can actually size (positive ``dst`` frag length).""" return { - op_id: info.c_frag_len + op_id: info.dst.frag_len for op_id, info in _MMA_FRAGMENT_INFO.items() - if info.c_frag_len > 0 + if info.dst.frag_len > 0 } @@ -50,27 +50,27 @@ def test_frag_lengths_are_positive(self): for op_id, frag in sizable.items(): self.assertIsInstance(frag, int) self.assertGreater( - frag, 0, msg=f"c_frag_len for {op_id!r} must be positive" + frag, 0, msg=f"dst.frag_len for {op_id!r} must be positive" ) def test_accessor_matches_ssot_and_raises_on_unknown(self): for op_id, frag in _sizable_op_ids().items(): - self.assertEqual(_mma_c_frag_len(op_id), frag) + self.assertEqual(_mma_dst_frag_len(op_id), frag) with self.assertRaises(ValueError): - _mma_c_frag_len("not_a_real_op_id") + _mma_dst_frag_len("not_a_real_op_id") def test_int_acc_op_ids_have_frag_lengths(self): - int_op_ids = [op for op, dtype in _op_id_c_dtype().items() if dtype == "i32"] + int_op_ids = [op for op, dtype in _op_id_dst_dtype().items() if dtype == "i32"] self.assertTrue( int_op_ids, "expected at least one i32-accumulator atom in the catalog" ) for op_id in int_op_ids: self.assertTrue( - _mma_c_is_int(op_id), - msg=f"{op_id!r} is i32 in the catalog but _mma_c_is_int disagrees", + _mma_dst_is_int(op_id), + msg=f"{op_id!r} is i32 in the catalog but _mma_dst_is_int disagrees", ) self.assertGreater( - _mma_c_frag_len(op_id), + _mma_dst_frag_len(op_id), 0, msg=f"int-accumulator op_id {op_id!r} has no SSOT frag length, so " f"IRBuilder.mma would raise for it", @@ -79,16 +79,16 @@ def test_int_acc_op_ids_have_frag_lengths(self): def test_result_hint_op_ids_have_frag_lengths(self): for op_id in _MMA_RESULT_HINT: self.assertGreater( - _mma_c_frag_len(op_id), + _mma_dst_frag_len(op_id), 0, msg=f"result-hint op_id {op_id!r} has no SSOT frag length", ) def test_known_frag_lengths(self): # Spot-check representative 16x16 vs 32x32 accumulator widths. - self.assertEqual(_mma_c_frag_len("mfma_f32_16x16x16_f16"), 4) - self.assertEqual(_mma_c_frag_len("mfma_f32_32x32x8_f16"), 16) - self.assertEqual(_mma_c_frag_len("wmma_f32_16x16x16_f16"), 8) + self.assertEqual(_mma_dst_frag_len("mfma_f32_16x16x16_f16"), 4) + self.assertEqual(_mma_dst_frag_len("mfma_f32_32x32x8_f16"), 16) + self.assertEqual(_mma_dst_frag_len("wmma_f32_16x16x16_f16"), 8) if __name__ == "__main__": # pragma: no cover diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_wmma_gfx12_acc_layout.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_wmma_gfx12_acc_layout.py index 32283cd99c9f..4390380282fe 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_wmma_gfx12_acc_layout.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_wmma_gfx12_acc_layout.py @@ -1,12 +1,13 @@ # Copyright (c) Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT -"""Validation tests: gfx1201 and gfx1250 share the same accumulator fragment layout. +"""Validation tests: gfx1201 and gfx1250 share the same ``dst`` fragment layout. Three layers of validation: -1. **Metadata identity** — every gfx1250 WMMA op carries the same ``c_frag_len``, - ``wave_size``, and ``c_fn`` object as its gfx1201 counterparts. +1. **Metadata identity** — every gfx1250 WMMA op carries the same ``dst`` + fragment length, ``wave_size``, and coordinate function as its gfx1201 + counterparts. 2. **Coordinate exhaustion** — the coordinate function produces identical (row, col) pairs for all 256 (lane, slot) inputs on both sets of op_ids, using the same @@ -23,7 +24,7 @@ import unittest from typing import Callable, Tuple -from rocke.core.arch.target import _MMA_FRAGMENT_INFO, _wmma_gfx12_acc_16x16 +from rocke.core.arch.target import _MMA_FRAGMENT_INFO, _wmma_gfx12_row_col_16x16 # --------------------------------------------------------------------------- @@ -51,7 +52,7 @@ def mul(self, a: int, b: int) -> int: return a * b -def _eval_acc(fn: Callable, lane: int, slot: int) -> Tuple[int, int]: +def _eval_row_col(fn: Callable, lane: int, slot: int) -> Tuple[int, int]: b = _ConstBuilder() return fn(b, lane, slot) @@ -59,7 +60,7 @@ def _eval_acc(fn: Callable, lane: int, slot: int) -> Tuple[int, int]: def _full_layout(fn: Callable, frag_len: int, wave_size: int): """Return the complete (lane, slot) -> (row, col) mapping as a dict.""" return { - (lane, slot): _eval_acc(fn, lane, slot) + (lane, slot): _eval_row_col(fn, lane, slot) for lane in range(wave_size) for slot in range(frag_len) } @@ -84,13 +85,13 @@ def _full_layout(fn: Callable, frag_len: int, wave_size: int): "wmma_gfx1250_f32_16x16x64_bf8_bf8", } -# Accumulator params shared by both arches: 16x16 output tile, wave32, +# ``dst`` params shared by both arches: 16x16 output tile, wave32, # <8 x float> per lane. -_EXPECTED_C_FRAG_LEN = 8 +_EXPECTED_DST_FRAG_LEN = 8 _EXPECTED_WAVE_SIZE = 32 -class TestWmmaGfx12AccLayoutCoverage(unittest.TestCase): +class TestWmmaGfx12DstLayoutCoverage(unittest.TestCase): """Guard that the op-id sets are non-empty and fully registered.""" def test_gfx1201_ops_in_ssot(self): @@ -110,15 +111,15 @@ def test_gfx1250_ops_in_ssot(self): ) -class TestWmmaGfx12AccFragMetadata(unittest.TestCase): - """gfx1201 and gfx1250 WMMA ops must carry identical accumulator metadata.""" +class TestWmmaGfx12DstFragMetadata(unittest.TestCase): + """gfx1201 and gfx1250 WMMA ops must carry identical ``dst`` metadata.""" - def _check_acc_metadata(self, op_id: str): + def _check_dst_metadata(self, op_id: str): info = _MMA_FRAGMENT_INFO[op_id] self.assertEqual( - info.c_frag_len, - _EXPECTED_C_FRAG_LEN, - msg=f"{op_id!r}: c_frag_len {info.c_frag_len} != {_EXPECTED_C_FRAG_LEN}", + info.dst.frag_len, + _EXPECTED_DST_FRAG_LEN, + msg=f"{op_id!r}: dst.frag_len {info.dst.frag_len} != {_EXPECTED_DST_FRAG_LEN}", ) self.assertEqual( info.wave_size, @@ -126,24 +127,24 @@ def _check_acc_metadata(self, op_id: str): msg=f"{op_id!r}: wave_size {info.wave_size} != {_EXPECTED_WAVE_SIZE}", ) self.assertIs( - info.c_fn, - _wmma_gfx12_acc_16x16, - msg=f"{op_id!r}: c_fn is not _wmma_gfx12_acc_16x16 — " - f"accumulator layout has been forked from the shared gfx12 map", + info.dst.fn, + _wmma_gfx12_row_col_16x16, + msg=f"{op_id!r}: dst coordinate function has been forked " + "from the shared gfx12 map", ) - def test_gfx1201_acc_metadata(self): + def test_gfx1201_dst_metadata(self): for op_id in _GFX1201_WMMA_OPS: with self.subTest(op_id=op_id): - self._check_acc_metadata(op_id) + self._check_dst_metadata(op_id) - def test_gfx1250_acc_metadata(self): + def test_gfx1250_dst_metadata(self): for op_id in _GFX1250_WMMA_OPS: with self.subTest(op_id=op_id): - self._check_acc_metadata(op_id) + self._check_dst_metadata(op_id) -class TestWmmaGfx12AccCoordinateFormula(unittest.TestCase): +class TestWmmaGfx12DstCoordinateFormula(unittest.TestCase): """The shared coordinate function must encode the expected formula for all inputs. Expected: slot ``i`` of lane ``l`` (0..31) maps to @@ -159,13 +160,13 @@ def _expected_coord(self, lane: int, slot: int) -> Tuple[int, int]: def test_coordinate_formula(self): builder = _ConstBuilder() for lane in range(_EXPECTED_WAVE_SIZE): - for slot in range(_EXPECTED_C_FRAG_LEN): - got = _wmma_gfx12_acc_16x16(builder, lane, slot) + for slot in range(_EXPECTED_DST_FRAG_LEN): + got = _wmma_gfx12_row_col_16x16(builder, lane, slot) want = self._expected_coord(lane, slot) self.assertEqual( got, want, - msg=f"_wmma_gfx12_acc_16x16(lane={lane}, slot={slot}): " + msg=f"_wmma_gfx12_row_col_16x16(lane={lane}, slot={slot}): " f"got {got}, expected {want}", ) @@ -174,20 +175,20 @@ def test_output_tile_is_fully_covered(self): coords = { self._expected_coord(lane, slot) for lane in range(_EXPECTED_WAVE_SIZE) - for slot in range(_EXPECTED_C_FRAG_LEN) + for slot in range(_EXPECTED_DST_FRAG_LEN) } expected = {(r, c) for r in range(16) for c in range(16)} self.assertEqual( coords, expected, - msg="Accumulator layout does not cover the full 16x16 output tile", + msg="dst layout does not cover the full 16x16 output tile", ) def test_no_coordinate_collisions(self): """Each (lane, slot) pair must map to a unique matrix element.""" seen: dict = {} for lane in range(_EXPECTED_WAVE_SIZE): - for slot in range(_EXPECTED_C_FRAG_LEN): + for slot in range(_EXPECTED_DST_FRAG_LEN): coord = self._expected_coord(lane, slot) if coord in seen: prev_lane, prev_slot = seen[coord] @@ -199,15 +200,15 @@ def test_no_coordinate_collisions(self): seen[coord] = (lane, slot) -class TestWmmaGfx12AccLayoutIdentical(unittest.TestCase): +class TestWmmaGfx12DstLayoutIdentical(unittest.TestCase): """The concrete (lane,slot)->(row,col) mapping must be bit-for-bit identical - across every gfx1201 op and every gfx1250 op that shares ``lm_wmma_gfx12_c``.""" + across every gfx1201 op and every gfx1250 op that shares the gfx12 map.""" def _layout_for(self, op_id: str) -> dict: info = _MMA_FRAGMENT_INFO[op_id] - if info.c_fn is None: - self.skipTest(f"{op_id!r} has no verified accumulator lane map (c_fn=None)") - return _full_layout(info.c_fn, info.c_frag_len, info.wave_size) + if info.dst.fn is None: + self.skipTest(f"{op_id!r} has no verified dst lane map") + return _full_layout(info.dst.fn, info.dst.frag_len, info.wave_size) def test_gfx1250_layout_matches_gfx1201_reference(self): # Use wmma_gfx12_f32_16x16x16_f16 as the canonical gfx1201 reference. @@ -217,15 +218,15 @@ def test_gfx1250_layout_matches_gfx1201_reference(self): for op_id in _GFX1250_WMMA_OPS: with self.subTest(op_id=op_id): info = _MMA_FRAGMENT_INFO[op_id] - if info.c_fn is None: + if info.dst.fn is None: # op has no verified map; identity is enforced via the - # metadata test (same c_frag_len / wave_size / c_fn object). + # metadata test (same dst metadata object values). continue - candidate = _full_layout(info.c_fn, info.c_frag_len, info.wave_size) + candidate = _full_layout(info.dst.fn, info.dst.frag_len, info.wave_size) self.assertEqual( candidate, ref_layout, - msg=f"{op_id!r} accumulator layout differs from {reference_op!r}", + msg=f"{op_id!r} dst layout differs from {reference_op!r}", ) def test_gfx1201_ops_have_identical_layouts(self): @@ -235,7 +236,7 @@ def test_gfx1201_ops_have_identical_layouts(self): self.assertEqual( layouts[ops[i]], layouts[ops[0]], - msg=f"gfx1201 accumulator layout mismatch: {ops[i]!r} vs {ops[0]!r}", + msg=f"gfx1201 dst layout mismatch: {ops[i]!r} vs {ops[0]!r}", ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json index d9936c915f2f..ca8f4b2799bf 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json @@ -1117,19 +1117,19 @@ }, "expected_failures": { "gemm/gfx1250/mxfp8/wmma_scale/bf8e5m2": { - "message": "tile.wmma_scale_f32_16x16x128_bf8_bf8 requires llvm23 (ROCm 7.13+), got llvm20", + "message": "tile.wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm20", "type": "NotImplementedError" }, "gemm/gfx1250/mxfp8/wmma_scale/fp8e4m3": { - "message": "tile.wmma_scale_f32_16x16x128_fp8_fp8 requires llvm23 (ROCm 7.13+), got llvm20", + "message": "tile.wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm20", "type": "NotImplementedError" }, "gemm/gfx1250/mxfp8/wmma_scale16/bf8e5m2": { - "message": "tile.wmma_scale16_f32_16x16x128_bf8_bf8 requires llvm23 (ROCm 7.13+), got llvm20", + "message": "tile.wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm20", "type": "NotImplementedError" }, "gemm/gfx1250/mxfp8/wmma_scale16/fp8e4m3": { - "message": "tile.wmma_scale16_f32_16x16x128_fp8_fp8 requires llvm23 (ROCm 7.13+), got llvm20", + "message": "tile.wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm20", "type": "NotImplementedError" } }, @@ -2281,19 +2281,19 @@ }, "expected_failures": { "gemm/gfx1250/mxfp8/wmma_scale/bf8e5m2": { - "message": "tile.wmma_scale_f32_16x16x128_bf8_bf8 requires llvm23 (ROCm 7.13+), got llvm22", + "message": "tile.wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm22", "type": "NotImplementedError" }, "gemm/gfx1250/mxfp8/wmma_scale/fp8e4m3": { - "message": "tile.wmma_scale_f32_16x16x128_fp8_fp8 requires llvm23 (ROCm 7.13+), got llvm22", + "message": "tile.wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm22", "type": "NotImplementedError" }, "gemm/gfx1250/mxfp8/wmma_scale16/bf8e5m2": { - "message": "tile.wmma_scale16_f32_16x16x128_bf8_bf8 requires llvm23 (ROCm 7.13+), got llvm22", + "message": "tile.wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm22", "type": "NotImplementedError" }, "gemm/gfx1250/mxfp8/wmma_scale16/fp8e4m3": { - "message": "tile.wmma_scale16_f32_16x16x128_fp8_fp8 requires llvm23 (ROCm 7.13+), got llvm22", + "message": "tile.wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm22", "type": "NotImplementedError" } }, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c index 1fb8d10ac466..e86befe8ca25 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c @@ -145,22 +145,34 @@ static void wmma_scaled(rocke_ir_builder_t* b, bool scale16, const char* op_id) static void build_wmma_scale(rocke_ir_builder_t* b) { - wmma_scaled(b, false, "wmma_scale_f32_16x16x128_fp8_fp8"); + wmma_scaled( + b, + false, + "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32"); } static void build_wmma_scale_bf8(rocke_ir_builder_t* b) { - wmma_scaled(b, false, "wmma_scale_f32_16x16x128_bf8_bf8"); + wmma_scaled( + b, + false, + "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32"); } static void build_wmma_scale16(rocke_ir_builder_t* b) { - wmma_scaled(b, true, "wmma_scale16_f32_16x16x128_fp8_fp8"); + wmma_scaled( + b, + true, + "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32"); } static void build_wmma_scale16_bf8(rocke_ir_builder_t* b) { - wmma_scaled(b, true, "wmma_scale16_f32_16x16x128_bf8_bf8"); + wmma_scaled( + b, + true, + "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32"); } /* ds_read_b128_tr_b16. gfx950 has one type-agnostic opcode returning diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py index bf23e17a7b1b..0e02e99e270b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py @@ -19,6 +19,7 @@ # each with its gfx950 counterpart pins both sides of the choice. # # arch is per-config (see _spec), llvm_flavor = AUTO, matching the C side. +from rocke.core.arch import ArchTarget, MmaScaleOperand from rocke.core.ir import BF16, F16, F32, I16, I32, I64, IRBuilder, KernelDef, PtrType from _emit_common import run_emit @@ -77,7 +78,18 @@ def build(b: IRBuilder) -> None: def _wmma_scaled(a_kind, b_kind, scale_mode): """K=128 scaled WMMA, parameterized by operand dtypes and scale mode.""" scale_ty = {"scale": I32, "scale16": I64}[scale_mode] - op_id = f"wmma_{scale_mode}_f32_16x16x128_{a_kind}_{b_kind}" + scale = MmaScaleOperand("e8m0", 16 if scale_mode == "scale16" else 32) + atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( + family="wmma_scaled", + src_dtypes=(a_kind, b_kind, "fp32"), + dst_dtype="fp32", + src_scales=(scale, scale, None), + m=16, + n=16, + k=128, + ) + assert atom is not None + op_id = atom.op_id def build(b: IRBuilder) -> None: a_ptr = b.param( From 4b70563dc1a5aadaa55c535b35e3e4424fea4249 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 18 Sep 2026 18:43:41 -0400 Subject: [PATCH 06/15] refactor(rocke): keep scaled WMMA metadata compact --- .../rocke/platform/cpp/core/arch/data.cpp | 1034 ++++++++++------- .../rocke/platform/cpp/core/arch/helpers.cpp | 9 +- .../rocke/platform/cpp/core/arch/query.cpp | 396 +++---- .../rocke/platform/cpp/core/ir/ir_tile.cpp | 45 +- .../rocke/platform/cpp/helpers/epilogues.cpp | 6 +- .../platform/cpp/helpers/mfma_attention.cpp | 68 +- .../platform/cpp/include/rocke/arch_target.h | 227 ++-- .../include/rocke/helper_rocke.core.arch.h | 13 +- .../rocke/helper_rocke.helpers.epilogues.h | 2 +- .../include/rocke/instance_gemm_internal.h | 2 +- .../rocke/platform/cpp/include/rocke/ir.h | 2 +- .../cpp/include/rocke/wmma_scale_internal.h | 21 +- .../conv_implicit_gemm_conv_build_glue.cpp | 6 +- .../conv_implicit_gemm_conv_epilogue.cpp | 2 +- .../common/conv_implicit_gemm_dgrad.cpp | 10 +- .../common/conv_implicit_gemm_wgrad.cpp | 8 +- ...d_conv_pool_conv1_gemm_and_lds_staging.cpp | 12 +- .../instances/common/gemm_build_compute.cpp | 2 +- .../cpp/instances/common/gemm_mma.cpp | 8 +- ...ol_scatter_maxpool_finalization_phases.cpp | 8 +- ...gemm_fragment_loaders_register_handoff.cpp | 32 +- .../dsl_docs/architecture/authoring_model.md | 25 +- .../python/rocke/core/arch/__init__.py | 4 - .../rocke/core/arch/data/arch_specs.json | 108 +- .../platform/python/rocke/core/arch/target.py | 567 ++++----- .../python/rocke/core/arch/wmma_scale.py | 27 +- .../rocke/platform/python/rocke/core/ir.py | 49 +- .../platform/python/rocke/core/lower_hip.py | 1 - .../platform/python/rocke/core/lower_llvm.py | 1 - .../instances/gfx1250/block_scaled_gemm.py | 9 +- .../tests/core/future_intrinsic_lowering.cpp | 8 +- .../platform/tests/core/mma_frag_ssot.cpp | 212 +--- .../platform/tests/core/test_arch_mma_ssot.py | 370 +++--- .../tests/core/test_gfx1250_scaled_wmma.py | 21 +- .../tests/core/test_lowbit_dtype_aliases.py | 2 +- .../tests/core/test_mma_catalog_lookup.cpp | 227 +++- .../platform/tests/core/test_mma_contracts.py | 121 -- .../tests/core/test_mma_frag_tables.py | 40 +- .../tests/core/test_wmma_gfx12_acc_layout.py | 81 +- .../instances/parity/gfx1250_lowering_emit.c | 29 +- .../instances/parity/gfx1250_lowering_emit.py | 7 +- 41 files changed, 1771 insertions(+), 2051 deletions(-) delete mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_contracts.py diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp index f93637c95353..81e1465cbe6f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp @@ -49,12 +49,12 @@ return; \ } while(0) -/* MFMA 16x16 src2/dst: slot i -> (m_blk * 4 + i, lane % 16). */ -static void _mfma_row_col_16x16(rocke_ir_builder_t* b, - rocke_value_t* lane, - int slot, - rocke_value_t** out0, - rocke_value_t** out1) +/* MFMA 16x16 accumulator: slot i -> (m_blk*4 + i, lane % 16). */ +static void _mfma_acc_16x16(rocke_ir_builder_t* b, + rocke_value_t* lane, + int slot, + rocke_value_t** out0, + rocke_value_t** out1) { rocke_value_t *c16, *n_in_atom, *m_blk, *row; ROCKE_ATI_COORD_GUARD(b, out0, out1); @@ -74,13 +74,13 @@ static void _mfma_row_col_16x16(rocke_ir_builder_t* b, *out1 = n_in_atom; } -/* MFMA 32x32 src2/dst row-column map: +/* MFMA 32x32 accumulator: * row = (slot//4)*8 + (lane//32)*4 + (slot%4), col = lane % 32. */ -static void _mfma_row_col_32x32(rocke_ir_builder_t* b, - rocke_value_t* lane, - int slot, - rocke_value_t** out0, - rocke_value_t** out1) +static void _mfma_acc_32x32(rocke_ir_builder_t* b, + rocke_value_t* lane, + int slot, + rocke_value_t** out0, + rocke_value_t** out1) { rocke_value_t *c32, *n_in_atom, *m_blk, *row; int rb, ri; @@ -384,12 +384,12 @@ static void _mfma_b_32x32x2_f32(rocke_ir_builder_t* b, *out1 = n_in_atom; } -/* WMMA 16x16x16 src2/dst row-column map (wave32). */ -static void _wmma_row_col_16x16(rocke_ir_builder_t* b, - rocke_value_t* lane, - int slot, - rocke_value_t** out0, - rocke_value_t** out1) +/* WMMA 16x16x16 accumulator (wave32): (2*slot + lane//16, lane % 16). */ +static void _wmma_acc_16x16(rocke_ir_builder_t* b, + rocke_value_t* lane, + int slot, + rocke_value_t** out0, + rocke_value_t** out1) { rocke_value_t *c16, *col, *half, *row; ROCKE_ATI_COORD_GUARD(b, out0, out1); @@ -505,12 +505,12 @@ static void _wmma_b_16x16_iu4(rocke_ir_builder_t* b, *out1 = col; } -/* RDNA4 WMMA 16x16x16 src2/dst row-column map (wave32). */ -static void _wmma_gfx12_row_col_16x16(rocke_ir_builder_t* b, - rocke_value_t* lane, - int slot, - rocke_value_t** out0, - rocke_value_t** out1) +/* RDNA4 WMMA 16x16x16 accumulator (wave32): ((lane//16)*8 + slot, lane % 16). */ +static void _wmma_gfx12_acc_16x16(rocke_ir_builder_t* b, + rocke_value_t* lane, + int slot, + rocke_value_t** out0, + rocke_value_t** out1) { rocke_value_t *c16, *col, *half, *row; ROCKE_ATI_COORD_GUARD(b, out0, out1); @@ -606,10 +606,9 @@ bool rocke_layout_map_coord(const rocke_layout_map_t* m, { if(b != NULL) { - const char* role_txt = (m->role == ROCKE_MMA_ROLE_SRC0) ? "src0" - : (m->role == ROCKE_MMA_ROLE_SRC1) ? "src1" - : (m->role == ROCKE_MMA_ROLE_SRC2) ? "src2" - : "dst"; + const char* role_txt = (m->role == ROCKE_MMA_ROLE_ACC) ? "acc" + : (m->role == ROCKE_MMA_ROLE_A) ? "a" + : "b"; b->status = ROCKE_ERR_VALUE; snprintf(b->err, ROCKE_ERR_MSG_CAP, @@ -632,112 +631,75 @@ bool rocke_layout_map_coord(const rocke_layout_map_t* m, * Per-op_id LayoutMap statics (the precomputed _build_mma_op output) * ========================================================================= * - * In Python _build_mma_op constructs, for each catalog row, up to four + * In Python _build_mma_op constructs, for each catalog row, up to three * LayoutMap instances from the op_id's _FragInfo (role + frag_len + wave_size + * fn). Because the maps depend ONLY on op_id (not on the per-arch row), we define - * one static set of LayoutMaps per op_id here and point every arch's catalog rows + * one static LayoutMap triplet per op_id here and point every arch's catalog rows * at them. A map is omitted (left NULL) when the _FragInfo has no fn for that * role or a zero frag_len (the `_mk(...) is None` case). * - * Private names, declaration order, stored roles, and catalog positions all use - * the canonical src0/src1/src2/dst machine operand order. Only op_ids with at - * least one verified map need descriptors; the rest carry NULL layout pointers - * in the catalog directly. + * Naming: lm__{a,b,c}. Only op_ids with at least one verified map need a + * struct; the rest carry NULL layout pointers in the catalog directly. */ -/* --- mfma_f32_16x16x16_f16 / _bf16: src0/src1/src2/dst (frag 4/4/4/4, wave64) --- */ -static const rocke_layout_map_t lm_mfma_16x16x16_src0 = {ROCKE_MMA_ROLE_SRC0, 4, 64, _mfma_a_16x16}; -static const rocke_layout_map_t lm_mfma_16x16x16_src1 = {ROCKE_MMA_ROLE_SRC1, 4, 64, _mfma_b_16x16}; -static const rocke_layout_map_t lm_mfma_16x16x16_src2 - = {ROCKE_MMA_ROLE_SRC2, 4, 64, _mfma_row_col_16x16}; -static const rocke_layout_map_t lm_mfma_16x16x16_dst - = {ROCKE_MMA_ROLE_DST, 4, 64, _mfma_row_col_16x16}; +/* --- mfma_f32_16x16x16_f16 / _bf16: a/b/c all present (frag 4/4/4, wave64) --- */ +static const rocke_layout_map_t lm_mfma_16x16x16_a = {ROCKE_MMA_ROLE_A, 4, 64, _mfma_a_16x16}; +static const rocke_layout_map_t lm_mfma_16x16x16_b = {ROCKE_MMA_ROLE_B, 4, 64, _mfma_b_16x16}; +static const rocke_layout_map_t lm_mfma_16x16x16_c = {ROCKE_MMA_ROLE_ACC, 4, 64, _mfma_acc_16x16}; -/* --- mfma_f32_16x16x32_{f16,bf16,fp8,bf8}: src0/src1/src2/dst (frag 8/8/4/4, wave64). +/* --- mfma_f32_16x16x32_{f16,bf16,fp8,bf8}: a/b/c present (frag 8/8/4, wave64). * fp8/bf8 share the f16 operand lane layout (same a_per_lane=8, K-packing). --- */ -static const rocke_layout_map_t lm_mfma_16x16x32_src0 - = {ROCKE_MMA_ROLE_SRC0, 8, 64, _mfma_a_16x16x32}; -static const rocke_layout_map_t lm_mfma_16x16x32_src1 - = {ROCKE_MMA_ROLE_SRC1, 8, 64, _mfma_b_16x16x32}; -static const rocke_layout_map_t lm_mfma_16x16x32_src2 - = {ROCKE_MMA_ROLE_SRC2, 4, 64, _mfma_row_col_16x16}; -static const rocke_layout_map_t lm_mfma_16x16x32_dst - = {ROCKE_MMA_ROLE_DST, 4, 64, _mfma_row_col_16x16}; - -/* --- mfma_f32_32x32x8_f16: src0/src1/src2/dst (frag 4/4/16/16, wave64) --- */ -static const rocke_layout_map_t lm_mfma_32x32x8_src0 - = {ROCKE_MMA_ROLE_SRC0, 4, 64, _mfma_a_32x32x8}; -static const rocke_layout_map_t lm_mfma_32x32x8_src1 - = {ROCKE_MMA_ROLE_SRC1, 4, 64, _mfma_b_32x32x8}; -static const rocke_layout_map_t lm_mfma_32x32x8_src2 - = {ROCKE_MMA_ROLE_SRC2, 16, 64, _mfma_row_col_32x32}; -static const rocke_layout_map_t lm_mfma_32x32x8_dst - = {ROCKE_MMA_ROLE_DST, 16, 64, _mfma_row_col_32x32}; - -/* --- mfma_f32_32x32x16_{f16,bf16,fp8,bf8}: src0/src1/src2/dst (frag 8/8/16/16, wave64). +static const rocke_layout_map_t lm_mfma_16x16x32_a = {ROCKE_MMA_ROLE_A, 8, 64, _mfma_a_16x16x32}; +static const rocke_layout_map_t lm_mfma_16x16x32_b = {ROCKE_MMA_ROLE_B, 8, 64, _mfma_b_16x16x32}; +static const rocke_layout_map_t lm_mfma_16x16x32_c = {ROCKE_MMA_ROLE_ACC, 4, 64, _mfma_acc_16x16}; + +/* --- mfma_f32_32x32x8_f16: a/b/c present (frag 4/4/16, wave64) --- */ +static const rocke_layout_map_t lm_mfma_32x32x8_a = {ROCKE_MMA_ROLE_A, 4, 64, _mfma_a_32x32x8}; +static const rocke_layout_map_t lm_mfma_32x32x8_b = {ROCKE_MMA_ROLE_B, 4, 64, _mfma_b_32x32x8}; +static const rocke_layout_map_t lm_mfma_32x32x8_c = {ROCKE_MMA_ROLE_ACC, 16, 64, _mfma_acc_32x32}; + +/* --- mfma_f32_32x32x16_{f16,bf16,fp8,bf8}: a/b/c present (frag 8/8/16, wave64). * fp8/bf8 share the f16 operand lane layout (same a_per_lane=8, K-packing). --- */ -static const rocke_layout_map_t lm_mfma_32x32x16_src0 - = {ROCKE_MMA_ROLE_SRC0, 8, 64, _mfma_a_32x32x16}; -static const rocke_layout_map_t lm_mfma_32x32x16_src1 - = {ROCKE_MMA_ROLE_SRC1, 8, 64, _mfma_b_32x32x16}; -static const rocke_layout_map_t lm_mfma_32x32x16_src2 - = {ROCKE_MMA_ROLE_SRC2, 16, 64, _mfma_row_col_32x32}; -static const rocke_layout_map_t lm_mfma_32x32x16_dst - = {ROCKE_MMA_ROLE_DST, 16, 64, _mfma_row_col_32x32}; - -/* --- mfma_f32_16x16x4_f32: src0/src1/src2/dst (frag 1/1/4/4, wave64) (#8348) --- */ -static const rocke_layout_map_t lm_mfma_16x16x4_f32_src0 - = {ROCKE_MMA_ROLE_SRC0, 1, 64, _mfma_a_16x16x4_f32}; -static const rocke_layout_map_t lm_mfma_16x16x4_f32_src1 - = {ROCKE_MMA_ROLE_SRC1, 1, 64, _mfma_b_16x16x4_f32}; -static const rocke_layout_map_t lm_mfma_16x16x4_f32_src2 - = {ROCKE_MMA_ROLE_SRC2, 4, 64, _mfma_row_col_16x16}; -static const rocke_layout_map_t lm_mfma_16x16x4_f32_dst - = {ROCKE_MMA_ROLE_DST, 4, 64, _mfma_row_col_16x16}; - -/* --- mfma_f32_32x32x2_f32: src0/src1/src2/dst (frag 1/1/16/16, wave64) (#8348) --- */ -static const rocke_layout_map_t lm_mfma_32x32x2_f32_src0 - = {ROCKE_MMA_ROLE_SRC0, 1, 64, _mfma_a_32x32x2_f32}; -static const rocke_layout_map_t lm_mfma_32x32x2_f32_src1 - = {ROCKE_MMA_ROLE_SRC1, 1, 64, _mfma_b_32x32x2_f32}; -static const rocke_layout_map_t lm_mfma_32x32x2_f32_src2 - = {ROCKE_MMA_ROLE_SRC2, 16, 64, _mfma_row_col_32x32}; -static const rocke_layout_map_t lm_mfma_32x32x2_f32_dst - = {ROCKE_MMA_ROLE_DST, 16, 64, _mfma_row_col_32x32}; - -/* --- wmma_f32_16x16x16_{f16,bf16}: src0/src1/src2/dst (frag 16/16/8/8, wave32) --- */ -static const rocke_layout_map_t lm_wmma_16x16x16_src0 - = {ROCKE_MMA_ROLE_SRC0, 16, 32, _wmma_a_16x16}; -static const rocke_layout_map_t lm_wmma_16x16x16_src1 - = {ROCKE_MMA_ROLE_SRC1, 16, 32, _wmma_b_16x16}; -static const rocke_layout_map_t lm_wmma_16x16x16_src2 - = {ROCKE_MMA_ROLE_SRC2, 8, 32, _wmma_row_col_16x16}; -static const rocke_layout_map_t lm_wmma_16x16x16_dst - = {ROCKE_MMA_ROLE_DST, 8, 32, _wmma_row_col_16x16}; - -/* --- wmma_i32_16x16x16_iu8: src0/src1/src2/dst (frag 4/4/8/8, wave32) --- */ -static const rocke_layout_map_t lm_wmma_iu8_src0 = {ROCKE_MMA_ROLE_SRC0, 4, 32, _wmma_a_16x16_iu8}; -static const rocke_layout_map_t lm_wmma_iu8_src1 = {ROCKE_MMA_ROLE_SRC1, 4, 32, _wmma_b_16x16_iu8}; -static const rocke_layout_map_t lm_wmma_iu8_src2 - = {ROCKE_MMA_ROLE_SRC2, 8, 32, _wmma_row_col_16x16}; -static const rocke_layout_map_t lm_wmma_iu8_dst = {ROCKE_MMA_ROLE_DST, 8, 32, _wmma_row_col_16x16}; - -/* --- wmma_i32_16x16x16_iu4: src0/src1/src2/dst (frag 2/2/8/8, wave32) --- */ -static const rocke_layout_map_t lm_wmma_iu4_src0 = {ROCKE_MMA_ROLE_SRC0, 2, 32, _wmma_a_16x16_iu4}; -static const rocke_layout_map_t lm_wmma_iu4_src1 = {ROCKE_MMA_ROLE_SRC1, 2, 32, _wmma_b_16x16_iu4}; -static const rocke_layout_map_t lm_wmma_iu4_src2 - = {ROCKE_MMA_ROLE_SRC2, 8, 32, _wmma_row_col_16x16}; -static const rocke_layout_map_t lm_wmma_iu4_dst = {ROCKE_MMA_ROLE_DST, 8, 32, _wmma_row_col_16x16}; - -/* --- wmma_gfx12_f32_16x16x16_{f16,bf16}: src0/src1/src2/dst (frag 8/8/8/8, wave32) --- */ -static const rocke_layout_map_t lm_wmma_gfx12_src0 - = {ROCKE_MMA_ROLE_SRC0, 8, 32, _wmma_gfx12_a_16x16}; -static const rocke_layout_map_t lm_wmma_gfx12_src1 - = {ROCKE_MMA_ROLE_SRC1, 8, 32, _wmma_gfx12_b_16x16}; -static const rocke_layout_map_t lm_wmma_gfx12_src2 - = {ROCKE_MMA_ROLE_SRC2, 8, 32, _wmma_gfx12_row_col_16x16}; -static const rocke_layout_map_t lm_wmma_gfx12_dst - = {ROCKE_MMA_ROLE_DST, 8, 32, _wmma_gfx12_row_col_16x16}; +static const rocke_layout_map_t lm_mfma_32x32x16_a = {ROCKE_MMA_ROLE_A, 8, 64, _mfma_a_32x32x16}; +static const rocke_layout_map_t lm_mfma_32x32x16_b = {ROCKE_MMA_ROLE_B, 8, 64, _mfma_b_32x32x16}; +static const rocke_layout_map_t lm_mfma_32x32x16_c = {ROCKE_MMA_ROLE_ACC, 16, 64, _mfma_acc_32x32}; + +/* --- mfma_f32_16x16x4_f32: a/b/c present (frag 1/1/4, wave64) (#8348) --- */ +static const rocke_layout_map_t lm_mfma_16x16x4_f32_a + = {ROCKE_MMA_ROLE_A, 1, 64, _mfma_a_16x16x4_f32}; +static const rocke_layout_map_t lm_mfma_16x16x4_f32_b + = {ROCKE_MMA_ROLE_B, 1, 64, _mfma_b_16x16x4_f32}; +static const rocke_layout_map_t lm_mfma_16x16x4_f32_c + = {ROCKE_MMA_ROLE_ACC, 4, 64, _mfma_acc_16x16}; + +/* --- mfma_f32_32x32x2_f32: a/b/c present (frag 1/1/16, wave64) (#8348) --- */ +static const rocke_layout_map_t lm_mfma_32x32x2_f32_a + = {ROCKE_MMA_ROLE_A, 1, 64, _mfma_a_32x32x2_f32}; +static const rocke_layout_map_t lm_mfma_32x32x2_f32_b + = {ROCKE_MMA_ROLE_B, 1, 64, _mfma_b_32x32x2_f32}; +static const rocke_layout_map_t lm_mfma_32x32x2_f32_c + = {ROCKE_MMA_ROLE_ACC, 16, 64, _mfma_acc_32x32}; + +/* --- wmma_f32_16x16x16_{f16,bf16}: a/b/c present (frag 16/16/8, wave32) --- */ +static const rocke_layout_map_t lm_wmma_16x16x16_a = {ROCKE_MMA_ROLE_A, 16, 32, _wmma_a_16x16}; +static const rocke_layout_map_t lm_wmma_16x16x16_b = {ROCKE_MMA_ROLE_B, 16, 32, _wmma_b_16x16}; +static const rocke_layout_map_t lm_wmma_16x16x16_c = {ROCKE_MMA_ROLE_ACC, 8, 32, _wmma_acc_16x16}; + +/* --- wmma_i32_16x16x16_iu8: a/b/c present (frag 4/4/8, wave32) --- */ +static const rocke_layout_map_t lm_wmma_iu8_a = {ROCKE_MMA_ROLE_A, 4, 32, _wmma_a_16x16_iu8}; +static const rocke_layout_map_t lm_wmma_iu8_b = {ROCKE_MMA_ROLE_B, 4, 32, _wmma_b_16x16_iu8}; +static const rocke_layout_map_t lm_wmma_iu8_c = {ROCKE_MMA_ROLE_ACC, 8, 32, _wmma_acc_16x16}; + +/* --- wmma_i32_16x16x16_iu4: a/b/c present (frag 2/2/8, wave32) --- */ +static const rocke_layout_map_t lm_wmma_iu4_a = {ROCKE_MMA_ROLE_A, 2, 32, _wmma_a_16x16_iu4}; +static const rocke_layout_map_t lm_wmma_iu4_b = {ROCKE_MMA_ROLE_B, 2, 32, _wmma_b_16x16_iu4}; +static const rocke_layout_map_t lm_wmma_iu4_c = {ROCKE_MMA_ROLE_ACC, 8, 32, _wmma_acc_16x16}; + +/* --- wmma_gfx12_f32_16x16x16_{f16,bf16}: a/b/c present (frag 8/8/8, wave32) --- */ +static const rocke_layout_map_t lm_wmma_gfx12_a = {ROCKE_MMA_ROLE_A, 8, 32, _wmma_gfx12_a_16x16}; +static const rocke_layout_map_t lm_wmma_gfx12_b = {ROCKE_MMA_ROLE_B, 8, 32, _wmma_gfx12_b_16x16}; +static const rocke_layout_map_t lm_wmma_gfx12_c + = {ROCKE_MMA_ROLE_ACC, 8, 32, _wmma_gfx12_acc_16x16}; /* gfx1250 WMMA 16x16x4 fp32 A operand (wave32): kABKLane=2, kAK1PerLane=2. * row = lane / 2 (M index 0..15); K = (lane % 2) * 2 + slot (slot in {0,1}). @@ -779,11 +741,11 @@ static void _wmma_gfx1250_b_16x16x4_f32(rocke_ir_builder_t* b, *out1 = col; } -/* --- wmma_gfx1250_f32_16x16x4_f32: src0/src1 plus shared gfx12 src2/dst maps --- */ -static const rocke_layout_map_t lm_wmma_gfx1250_f32_src0 - = {ROCKE_MMA_ROLE_SRC0, 2, 32, _wmma_gfx1250_a_16x16x4_f32}; -static const rocke_layout_map_t lm_wmma_gfx1250_f32_src1 - = {ROCKE_MMA_ROLE_SRC1, 2, 32, _wmma_gfx1250_b_16x16x4_f32}; +/* --- wmma_gfx1250_f32_16x16x4_f32: a/b/c present (frag 2/2/8, wave32) --- */ +static const rocke_layout_map_t lm_wmma_gfx1250_f32_a + = {ROCKE_MMA_ROLE_A, 2, 32, _wmma_gfx1250_a_16x16x4_f32}; +static const rocke_layout_map_t lm_wmma_gfx1250_f32_b + = {ROCKE_MMA_ROLE_B, 2, 32, _wmma_gfx1250_b_16x16x4_f32}; /* gfx1250 WMMA 16x16x32 A operand (wave32): row = lane % 16, * k = (lane / 16) * 16 + slot (slot in 0..15, a_frag_len=16). @@ -833,55 +795,55 @@ static void _wmma_gfx1250_b_16x16x32(rocke_ir_builder_t* b, *out1 = col; } -/* --- wmma_gfx1250_f32_16x16x32_{f16,bf16}: src0/src1 plus shared gfx12 src2/dst maps --- */ -static const rocke_layout_map_t lm_wmma_gfx1250_32_src0 - = {ROCKE_MMA_ROLE_SRC0, 16, 32, _wmma_gfx1250_a_16x16x32}; -static const rocke_layout_map_t lm_wmma_gfx1250_32_src1 - = {ROCKE_MMA_ROLE_SRC1, 16, 32, _wmma_gfx1250_b_16x16x32}; +/* --- wmma_gfx1250_f32_16x16x32_{f16,bf16}: a/b/c present (frag 16/16/8, wave32) --- */ +static const rocke_layout_map_t lm_wmma_gfx1250_32_a + = {ROCKE_MMA_ROLE_A, 16, 32, _wmma_gfx1250_a_16x16x32}; +static const rocke_layout_map_t lm_wmma_gfx1250_32_b + = {ROCKE_MMA_ROLE_B, 16, 32, _wmma_gfx1250_b_16x16x32}; /* ========================================================================= - * op_id -> dst fragment length (the dst.frag_len projection of + * op_id -> accumulator fragment length (the c_frag_len projection of * target.py::_MMA_FRAGMENT_INFO). * ========================================================================= * * This is the arch SSOT that rocke.core.ir.IRBuilder.mma (rocke_b_mma) consults * to size a tile.mma result vector from a bare op_id string; the ir bucket keeps - * no private copy. It mirrors _MMA_FRAGMENT_INFO's dst.frag_len exactly. + * no private copy. It mirrors _MMA_FRAGMENT_INFO's c_frag_len column exactly. * Only atoms this engine can emit are listed: this table mirrors the Python SSOT * including the gfx1250 WMMA atoms. * * `ati` = arch-target internal (see arch_target_internal.h): the naming prefix * reserved for storage private to this arch-target port, kept out of the public * rocke_ / rocke_arch_ namespace. This table is file-local static (only - * rocke_arch_mma_dst_frag_len below reads it), so it is not extern'd in the + * rocke_arch_mma_c_frag_len below reads it), so it is not extern'd in the * internal header like the cross-bucket rocke_ati_ symbols are. * * NOTE (drift): this is still a hand-maintained mirror of the Python - * _MMA_FRAGMENT_INFO dst.frag_len values, not generated from them. + * _MMA_FRAGMENT_INFO c_frag_len column, not generated from it. * TODO: codegen this table from the Python SSOT (_MMA_FRAGMENT_INFO) at build * time so the two engines share a single source of truth. Until then there is * no direct mirror-agreement test; each side's frag lengths are pinned * independently -- C via tests/core/mma_frag_ssot.cpp (rocke_b_mma) and Python * via tests/core/test_mma_frag_tables.py (_MMA_FRAGMENT_INFO) -- and the * byte-identity gate (tools/check_byte_identity.py) catches any cross-engine - * dst-fragment drift, since a wrong length changes the emitted tile.mma result + * c_frag_len drift, since a wrong length changes the emitted tile.mma result * vector width. */ typedef struct rocke_ati_mma_frag_row { const char* op_id; - int dst_frag_len; + int c_frag_len; } rocke_ati_mma_frag_row_t; -/* Layout guard for the rocke_arch_mma_dst_frag_len lookup below, which derives its +/* Layout guard for the rocke_arch_mma_c_frag_len lookup below, which derives its * row count from sizeof(table)/sizeof(row). That division is exact for any * element size, so padding of rocke_ati_mma_frag_row_t (12 bytes of data padded * out to 16 for op_id's 8-byte alignment) cannot desync the count. These asserts * pin the field order so a future reorder/resize is caught at compile time. */ static_assert(offsetof(rocke_ati_mma_frag_row_t, op_id) == 0, "op_id must be the first field of rocke_ati_mma_frag_row_t"); -static_assert(offsetof(rocke_ati_mma_frag_row_t, dst_frag_len) == sizeof(const char*), - "dst_frag_len must immediately follow op_id (no leading padding)"); +static_assert(offsetof(rocke_ati_mma_frag_row_t, c_frag_len) == sizeof(const char*), + "c_frag_len must immediately follow op_id (no leading padding)"); static const rocke_ati_mma_frag_row_t rocke_ati_mma_frag[] = { /* --- MFMA fp32 (wave64) --- */ @@ -931,7 +893,7 @@ static const rocke_ati_mma_frag_row_t rocke_ati_mma_frag[] = { {"wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32", 8}, }; -int rocke_arch_mma_dst_frag_len(const char* op_id) +int rocke_arch_mma_c_frag_len(const char* op_id) { size_t i; if(!op_id) @@ -942,7 +904,7 @@ int rocke_arch_mma_dst_frag_len(const char* op_id) { if(strcmp(rocke_ati_mma_frag[i].op_id, op_id) == 0) { - return rocke_ati_mma_frag[i].dst_frag_len; + return rocke_ati_mma_frag[i].c_frag_len; } } return 0; /* unknown atom: the zero-length _frag_info fallback */ @@ -954,15 +916,13 @@ int rocke_arch_mma_dst_frag_len(const char* op_id) * ========================================================================= * * Frag lengths come from _MMA_FRAGMENT_INFO[op_id]; op_ids absent from that - * table carry zero fragment lengths and NULL maps. dtype fields hold the normalised - * (canonical) catalog keys (interned strings from rocke_normalize_dtype), exactly as - * normalize_dtype() would produce for each indexed src and dst. + * table carry (0,0,0,64) and NULL maps. dtype fields hold the normalised + * (canonical) catalog keys from rocke_normalize_dtype, exactly as + * normalize_dtype(o["a"/"b"/"c"]) would produce. * * Field order of rocke_mma_op_t: - * family, srcs[0], srcs[1], srcs[2], dst, m, n, k, op_id, wave_size. - * Each source stores dtype, frag_len, layout, optional scale value dtype, and - * scale block size along K (zero when unscaled); dst stores dtype, frag_len, - * and layout. Scale value dtype is independent of the packed register carrier. + * family, a_dtype, b_dtype, c_dtype, m, n, k, op_id, + * a_frag_len, b_frag_len, c_frag_len, wave_size, a_layout, b_layout, c_layout * * Low-bit spellings normalize to explicit exponent/mantissa catalog keys. */ @@ -973,529 +933,813 @@ int rocke_arch_mma_dst_frag_len(const char* op_id) * bf16 arrived in CDNA2 with the same layout maps as fp16 for those shapes). */ static const rocke_mma_op_t k_mma_gfx90a[] = { {"mma", - {{"fp16", 4, &lm_mfma_16x16x16_src0}, - {"fp16", 4, &lm_mfma_16x16x16_src1}, - {"fp32", 4, &lm_mfma_16x16x16_src2}}, - {"fp32", 4, &lm_mfma_16x16x16_dst}, + "fp16", + "fp16", + "fp32", 16, 16, 16, "mfma_f32_16x16x16_f16", - 64}, + 4, + 4, + 4, + 64, + &lm_mfma_16x16x16_a, + &lm_mfma_16x16x16_b, + &lm_mfma_16x16x16_c}, {"mma", - {{"fp16", 4, &lm_mfma_32x32x8_src0}, - {"fp16", 4, &lm_mfma_32x32x8_src1}, - {"fp32", 16, &lm_mfma_32x32x8_src2}}, - {"fp32", 16, &lm_mfma_32x32x8_dst}, + "fp16", + "fp16", + "fp32", 32, 32, 8, "mfma_f32_32x32x8_f16", - 64}, + 4, + 4, + 16, + 64, + &lm_mfma_32x32x8_a, + &lm_mfma_32x32x8_b, + &lm_mfma_32x32x8_c}, {"mma", - {{"bf16", 4, &lm_mfma_16x16x16_src0}, - {"bf16", 4, &lm_mfma_16x16x16_src1}, - {"fp32", 4, &lm_mfma_16x16x16_src2}}, - {"fp32", 4, &lm_mfma_16x16x16_dst}, + "bf16", + "bf16", + "fp32", 16, 16, 16, "mfma_f32_16x16x16_bf16", - 64}, + 4, + 4, + 4, + 64, + &lm_mfma_16x16x16_a, + &lm_mfma_16x16x16_b, + &lm_mfma_16x16x16_c}, {"mma", - {{"bf16", 4, &lm_mfma_32x32x8_src0}, - {"bf16", 4, &lm_mfma_32x32x8_src1}, - {"fp32", 16, &lm_mfma_32x32x8_src2}}, - {"fp32", 16, &lm_mfma_32x32x8_dst}, + "bf16", + "bf16", + "fp32", 32, 32, 8, "mfma_f32_32x32x8_bf16", - 64}, + 4, + 4, + 16, + 64, + &lm_mfma_32x32x8_a, + &lm_mfma_32x32x8_b, + &lm_mfma_32x32x8_c}, }; /* ----------------------------- gfx942 (CDNA) ----------------------------- */ static const rocke_mma_op_t k_mma_gfx942[] = { {"mma", - {{"fp32", 1, &lm_mfma_16x16x4_f32_src0}, - {"fp32", 1, &lm_mfma_16x16x4_f32_src1}, - {"fp32", 4, &lm_mfma_16x16x4_f32_src2}}, - {"fp32", 4, &lm_mfma_16x16x4_f32_dst}, + "fp32", + "fp32", + "fp32", 16, 16, 4, "mfma_f32_16x16x4_f32", - 64}, + 1, + 1, + 4, + 64, + &lm_mfma_16x16x4_f32_a, + &lm_mfma_16x16x4_f32_b, + &lm_mfma_16x16x4_f32_c}, {"mma", - {{"fp32", 1, &lm_mfma_32x32x2_f32_src0}, - {"fp32", 1, &lm_mfma_32x32x2_f32_src1}, - {"fp32", 16, &lm_mfma_32x32x2_f32_src2}}, - {"fp32", 16, &lm_mfma_32x32x2_f32_dst}, + "fp32", + "fp32", + "fp32", 32, 32, 2, "mfma_f32_32x32x2_f32", - 64}, + 1, + 1, + 16, + 64, + &lm_mfma_32x32x2_f32_a, + &lm_mfma_32x32x2_f32_b, + &lm_mfma_32x32x2_f32_c}, {"mma", - {{"fp16", 4, &lm_mfma_16x16x16_src0}, - {"fp16", 4, &lm_mfma_16x16x16_src1}, - {"fp32", 4, &lm_mfma_16x16x16_src2}}, - {"fp32", 4, &lm_mfma_16x16x16_dst}, + "fp16", + "fp16", + "fp32", 16, 16, 16, "mfma_f32_16x16x16_f16", - 64}, + 4, + 4, + 4, + 64, + &lm_mfma_16x16x16_a, + &lm_mfma_16x16x16_b, + &lm_mfma_16x16x16_c}, {"mma", - {{"fp16", 4, &lm_mfma_32x32x8_src0}, - {"fp16", 4, &lm_mfma_32x32x8_src1}, - {"fp32", 16, &lm_mfma_32x32x8_src2}}, - {"fp32", 16, &lm_mfma_32x32x8_dst}, + "fp16", + "fp16", + "fp32", 32, 32, 8, "mfma_f32_32x32x8_f16", - 64}, + 4, + 4, + 16, + 64, + &lm_mfma_32x32x8_a, + &lm_mfma_32x32x8_b, + &lm_mfma_32x32x8_c}, {"mma", - {{"bf16", 4, &lm_mfma_16x16x16_src0}, - {"bf16", 4, &lm_mfma_16x16x16_src1}, - {"fp32", 4, &lm_mfma_16x16x16_src2}}, - {"fp32", 4, &lm_mfma_16x16x16_dst}, + "bf16", + "bf16", + "fp32", 16, 16, 16, "mfma_f32_16x16x16_bf16", - 64}, + 4, + 4, + 4, + 64, + &lm_mfma_16x16x16_a, + &lm_mfma_16x16x16_b, + &lm_mfma_16x16x16_c}, {"mma", - {{"bf16", 4, &lm_mfma_32x32x8_src0}, - {"bf16", 4, &lm_mfma_32x32x8_src1}, - {"fp32", 16, &lm_mfma_32x32x8_src2}}, - {"fp32", 16, &lm_mfma_32x32x8_dst}, + "bf16", + "bf16", + "fp32", 32, 32, 8, "mfma_f32_32x32x8_bf16", - 64}, + 4, + 4, + 16, + 64, + &lm_mfma_32x32x8_a, + &lm_mfma_32x32x8_b, + &lm_mfma_32x32x8_c}, {"mma", - {{"fp8e4m3", 8, &lm_mfma_16x16x32_src0}, - {"fp8e4m3", 8, &lm_mfma_16x16x32_src1}, - {"fp32", 4, &lm_mfma_16x16x32_src2}}, - {"fp32", 4, &lm_mfma_16x16x32_dst}, + "fp8e4m3", + "fp8e4m3", + "fp32", 16, 16, 32, "mfma_f32_16x16x32_fp8", - 64}, + 8, + 8, + 4, + 64, + &lm_mfma_16x16x32_a, + &lm_mfma_16x16x32_b, + &lm_mfma_16x16x32_c}, {"mma", - {{"fp8e4m3", 8, &lm_mfma_32x32x16_src0}, - {"fp8e4m3", 8, &lm_mfma_32x32x16_src1}, - {"fp32", 16, &lm_mfma_32x32x16_src2}}, - {"fp32", 16, &lm_mfma_32x32x16_dst}, + "fp8e4m3", + "fp8e4m3", + "fp32", 32, 32, 16, "mfma_f32_32x32x16_fp8", - 64}, + 8, + 8, + 16, + 64, + &lm_mfma_32x32x16_a, + &lm_mfma_32x32x16_b, + &lm_mfma_32x32x16_c}, {"mma", - {{"bf8e5m2", 8, &lm_mfma_16x16x32_src0}, - {"bf8e5m2", 8, &lm_mfma_16x16x32_src1}, - {"fp32", 4, &lm_mfma_16x16x32_src2}}, - {"fp32", 4, &lm_mfma_16x16x32_dst}, + "bf8e5m2", + "bf8e5m2", + "fp32", 16, 16, 32, "mfma_f32_16x16x32_bf8", - 64}, + 8, + 8, + 4, + 64, + &lm_mfma_16x16x32_a, + &lm_mfma_16x16x32_b, + &lm_mfma_16x16x32_c}, {"mma", - {{"bf8e5m2", 8, &lm_mfma_32x32x16_src0}, - {"bf8e5m2", 8, &lm_mfma_32x32x16_src1}, - {"fp32", 16, &lm_mfma_32x32x16_src2}}, - {"fp32", 16, &lm_mfma_32x32x16_dst}, + "bf8e5m2", + "bf8e5m2", + "fp32", 32, 32, 16, "mfma_f32_32x32x16_bf8", - 64}, + 8, + 8, + 16, + 64, + &lm_mfma_32x32x16_a, + &lm_mfma_32x32x16_b, + &lm_mfma_32x32x16_c}, }; /* ----------------------------- gfx950 (CDNA) ----------------------------- */ static const rocke_mma_op_t k_mma_gfx950[] = { {"mma", - {{"fp32", 1, &lm_mfma_16x16x4_f32_src0}, - {"fp32", 1, &lm_mfma_16x16x4_f32_src1}, - {"fp32", 4, &lm_mfma_16x16x4_f32_src2}}, - {"fp32", 4, &lm_mfma_16x16x4_f32_dst}, + "fp32", + "fp32", + "fp32", 16, 16, 4, "mfma_f32_16x16x4_f32", - 64}, + 1, + 1, + 4, + 64, + &lm_mfma_16x16x4_f32_a, + &lm_mfma_16x16x4_f32_b, + &lm_mfma_16x16x4_f32_c}, {"mma", - {{"fp32", 1, &lm_mfma_32x32x2_f32_src0}, - {"fp32", 1, &lm_mfma_32x32x2_f32_src1}, - {"fp32", 16, &lm_mfma_32x32x2_f32_src2}}, - {"fp32", 16, &lm_mfma_32x32x2_f32_dst}, + "fp32", + "fp32", + "fp32", 32, 32, 2, "mfma_f32_32x32x2_f32", - 64}, + 1, + 1, + 16, + 64, + &lm_mfma_32x32x2_f32_a, + &lm_mfma_32x32x2_f32_b, + &lm_mfma_32x32x2_f32_c}, {"mma", - {{"fp16", 4, &lm_mfma_16x16x16_src0}, - {"fp16", 4, &lm_mfma_16x16x16_src1}, - {"fp32", 4, &lm_mfma_16x16x16_src2}}, - {"fp32", 4, &lm_mfma_16x16x16_dst}, + "fp16", + "fp16", + "fp32", 16, 16, 16, "mfma_f32_16x16x16_f16", - 64}, + 4, + 4, + 4, + 64, + &lm_mfma_16x16x16_a, + &lm_mfma_16x16x16_b, + &lm_mfma_16x16x16_c}, {"mma", - {{"fp16", 8, &lm_mfma_16x16x32_src0}, - {"fp16", 8, &lm_mfma_16x16x32_src1}, - {"fp32", 4, &lm_mfma_16x16x32_src2}}, - {"fp32", 4, &lm_mfma_16x16x32_dst}, + "fp16", + "fp16", + "fp32", 16, 16, 32, "mfma_f32_16x16x32_f16", - 64}, + 8, + 8, + 4, + 64, + &lm_mfma_16x16x32_a, + &lm_mfma_16x16x32_b, + &lm_mfma_16x16x32_c}, {"mma", - {{"fp16", 4, &lm_mfma_32x32x8_src0}, - {"fp16", 4, &lm_mfma_32x32x8_src1}, - {"fp32", 16, &lm_mfma_32x32x8_src2}}, - {"fp32", 16, &lm_mfma_32x32x8_dst}, + "fp16", + "fp16", + "fp32", 32, 32, 8, "mfma_f32_32x32x8_f16", - 64}, + 4, + 4, + 16, + 64, + &lm_mfma_32x32x8_a, + &lm_mfma_32x32x8_b, + &lm_mfma_32x32x8_c}, {"mma", - {{"fp16", 8, &lm_mfma_32x32x16_src0}, - {"fp16", 8, &lm_mfma_32x32x16_src1}, - {"fp32", 16, &lm_mfma_32x32x16_src2}}, - {"fp32", 16, &lm_mfma_32x32x16_dst}, + "fp16", + "fp16", + "fp32", 32, 32, 16, "mfma_f32_32x32x16_f16", - 64}, + 8, + 8, + 16, + 64, + &lm_mfma_32x32x16_a, + &lm_mfma_32x32x16_b, + &lm_mfma_32x32x16_c}, {"mma", - {{"bf16", 4, &lm_mfma_16x16x16_src0}, - {"bf16", 4, &lm_mfma_16x16x16_src1}, - {"fp32", 4, &lm_mfma_16x16x16_src2}}, - {"fp32", 4, &lm_mfma_16x16x16_dst}, + "bf16", + "bf16", + "fp32", 16, 16, 16, "mfma_f32_16x16x16_bf16", - 64}, + 4, + 4, + 4, + 64, + &lm_mfma_16x16x16_a, + &lm_mfma_16x16x16_b, + &lm_mfma_16x16x16_c}, {"mma", - {{"bf16", 8, &lm_mfma_16x16x32_src0}, - {"bf16", 8, &lm_mfma_16x16x32_src1}, - {"fp32", 4, &lm_mfma_16x16x32_src2}}, - {"fp32", 4, &lm_mfma_16x16x32_dst}, + "bf16", + "bf16", + "fp32", 16, 16, 32, "mfma_f32_16x16x32_bf16", - 64}, + 8, + 8, + 4, + 64, + &lm_mfma_16x16x32_a, + &lm_mfma_16x16x32_b, + &lm_mfma_16x16x32_c}, {"mma", - {{"bf16", 4, &lm_mfma_32x32x8_src0}, - {"bf16", 4, &lm_mfma_32x32x8_src1}, - {"fp32", 16, &lm_mfma_32x32x8_src2}}, - {"fp32", 16, &lm_mfma_32x32x8_dst}, + "bf16", + "bf16", + "fp32", 32, 32, 8, "mfma_f32_32x32x8_bf16", - 64}, + 4, + 4, + 16, + 64, + &lm_mfma_32x32x8_a, + &lm_mfma_32x32x8_b, + &lm_mfma_32x32x8_c}, {"mma", - {{"bf16", 8, &lm_mfma_32x32x16_src0}, - {"bf16", 8, &lm_mfma_32x32x16_src1}, - {"fp32", 16, &lm_mfma_32x32x16_src2}}, - {"fp32", 16, &lm_mfma_32x32x16_dst}, + "bf16", + "bf16", + "fp32", 32, 32, 16, "mfma_f32_32x32x16_bf16", - 64}, + 8, + 8, + 16, + 64, + &lm_mfma_32x32x16_a, + &lm_mfma_32x32x16_b, + &lm_mfma_32x32x16_c}, {"mma", - {{"fp8e4m3", 8, &lm_mfma_16x16x32_src0}, - {"fp8e4m3", 8, &lm_mfma_16x16x32_src1}, - {"fp32", 4, &lm_mfma_16x16x32_src2}}, - {"fp32", 4, &lm_mfma_16x16x32_dst}, + "fp8e4m3", + "fp8e4m3", + "fp32", 16, 16, 32, "mfma_f32_16x16x32_fp8", - 64}, + 8, + 8, + 4, + 64, + &lm_mfma_16x16x32_a, + &lm_mfma_16x16x32_b, + &lm_mfma_16x16x32_c}, {"mma", - {{"fp8e4m3", 8, &lm_mfma_32x32x16_src0}, - {"fp8e4m3", 8, &lm_mfma_32x32x16_src1}, - {"fp32", 16, &lm_mfma_32x32x16_src2}}, - {"fp32", 16, &lm_mfma_32x32x16_dst}, + "fp8e4m3", + "fp8e4m3", + "fp32", 32, 32, 16, "mfma_f32_32x32x16_fp8", - 64}, + 8, + 8, + 16, + 64, + &lm_mfma_32x32x16_a, + &lm_mfma_32x32x16_b, + &lm_mfma_32x32x16_c}, {"mma", - {{"bf8e5m2", 8, &lm_mfma_16x16x32_src0}, - {"bf8e5m2", 8, &lm_mfma_16x16x32_src1}, - {"fp32", 4, &lm_mfma_16x16x32_src2}}, - {"fp32", 4, &lm_mfma_16x16x32_dst}, + "bf8e5m2", + "bf8e5m2", + "fp32", 16, 16, 32, "mfma_f32_16x16x32_bf8", - 64}, + 8, + 8, + 4, + 64, + &lm_mfma_16x16x32_a, + &lm_mfma_16x16x32_b, + &lm_mfma_16x16x32_c}, {"mma", - {{"bf8e5m2", 8, &lm_mfma_32x32x16_src0}, - {"bf8e5m2", 8, &lm_mfma_32x32x16_src1}, - {"fp32", 16, &lm_mfma_32x32x16_src2}}, - {"fp32", 16, &lm_mfma_32x32x16_dst}, + "bf8e5m2", + "bf8e5m2", + "fp32", 32, 32, 16, "mfma_f32_32x32x16_bf8", - 64}, + 8, + 8, + 16, + 64, + &lm_mfma_32x32x16_a, + &lm_mfma_32x32x16_b, + &lm_mfma_32x32x16_c}, /* Low-bit catalog keys use exponent/mantissa names; op_ids carry frag lengths * only (no verified maps), per _MMA_FRAGMENT_INFO. */ {"mma", - {{"fp4e2m1", 16, NULL}, {"fp4e2m1", 16, NULL}, {"fp32", 4, NULL}}, - {"fp32", 4, NULL}, + "fp4e2m1", + "fp4e2m1", + "fp32", 16, 16, 128, "mfma_f32_16x16x128_fp4", - 64}, + 16, + 16, + 4, + 64, + NULL, + NULL, + NULL}, {"mma", - {{"fp6e2m3", 12, NULL}, {"fp6e2m3", 12, NULL}, {"fp32", 4, NULL}}, - {"fp32", 4, NULL}, + "fp6e2m3", + "fp6e2m3", + "fp32", 16, 16, 96, "mfma_f32_16x16x96_fp6", - 64}, + 12, + 12, + 4, + 64, + NULL, + NULL, + NULL}, }; /* ----------------------------- gfx1151 (RDNA3.5) ------------------------- */ static const rocke_mma_op_t k_mma_gfx1151[] = { {"wmma", - {{"fp16", 16, &lm_wmma_16x16x16_src0}, - {"fp16", 16, &lm_wmma_16x16x16_src1}, - {"fp32", 8, &lm_wmma_16x16x16_src2}}, - {"fp32", 8, &lm_wmma_16x16x16_dst}, + "fp16", + "fp16", + "fp32", 16, 16, 16, "wmma_f32_16x16x16_f16", - 32}, + 16, + 16, + 8, + 32, + &lm_wmma_16x16x16_a, + &lm_wmma_16x16x16_b, + &lm_wmma_16x16x16_c}, {"wmma", - {{"bf16", 16, &lm_wmma_16x16x16_src0}, - {"bf16", 16, &lm_wmma_16x16x16_src1}, - {"fp32", 8, &lm_wmma_16x16x16_src2}}, - {"fp32", 8, &lm_wmma_16x16x16_dst}, + "bf16", + "bf16", + "fp32", 16, 16, 16, "wmma_f32_16x16x16_bf16", - 32}, + 16, + 16, + 8, + 32, + &lm_wmma_16x16x16_a, + &lm_wmma_16x16x16_b, + &lm_wmma_16x16x16_c}, {"wmma", - {{"iu8", 4, &lm_wmma_iu8_src0}, {"iu8", 4, &lm_wmma_iu8_src1}, {"i32", 8, &lm_wmma_iu8_src2}}, - {"i32", 8, &lm_wmma_iu8_dst}, + "iu8", + "iu8", + "i32", 16, 16, 16, "wmma_i32_16x16x16_iu8", - 32}, + 4, + 4, + 8, + 32, + &lm_wmma_iu8_a, + &lm_wmma_iu8_b, + &lm_wmma_iu8_c}, {"wmma", - {{"iu4", 2, &lm_wmma_iu4_src0}, {"iu4", 2, &lm_wmma_iu4_src1}, {"i32", 8, &lm_wmma_iu4_src2}}, - {"i32", 8, &lm_wmma_iu4_dst}, + "iu4", + "iu4", + "i32", 16, 16, 16, "wmma_i32_16x16x16_iu4", - 32}, + 2, + 2, + 8, + 32, + &lm_wmma_iu4_a, + &lm_wmma_iu4_b, + &lm_wmma_iu4_c}, }; /* ----------------------------- gfx1201 (RDNA4) -------------------------- */ static const rocke_mma_op_t k_mma_gfx1201[] = { {"wmma", - {{"fp16", 8, &lm_wmma_gfx12_src0}, - {"fp16", 8, &lm_wmma_gfx12_src1}, - {"fp32", 8, &lm_wmma_gfx12_src2}}, - {"fp32", 8, &lm_wmma_gfx12_dst}, + "fp16", + "fp16", + "fp32", 16, 16, 16, "wmma_gfx12_f32_16x16x16_f16", - 32}, + 8, + 8, + 8, + 32, + &lm_wmma_gfx12_a, + &lm_wmma_gfx12_b, + &lm_wmma_gfx12_c}, {"wmma", - {{"bf16", 8, &lm_wmma_gfx12_src0}, - {"bf16", 8, &lm_wmma_gfx12_src1}, - {"fp32", 8, &lm_wmma_gfx12_src2}}, - {"fp32", 8, &lm_wmma_gfx12_dst}, + "bf16", + "bf16", + "fp32", 16, 16, 16, "wmma_gfx12_f32_16x16x16_bf16", - 32}, + 8, + 8, + 8, + 32, + &lm_wmma_gfx12_a, + &lm_wmma_gfx12_b, + &lm_wmma_gfx12_c}, }; /* --------------------------- gfx11-generic (RDNA3) ---------------------- */ static const rocke_mma_op_t k_mma_gfx11_generic[] = { {"wmma", - {{"fp16", 16, &lm_wmma_16x16x16_src0}, - {"fp16", 16, &lm_wmma_16x16x16_src1}, - {"fp32", 8, &lm_wmma_16x16x16_src2}}, - {"fp32", 8, &lm_wmma_16x16x16_dst}, + "fp16", + "fp16", + "fp32", 16, 16, 16, "wmma_f32_16x16x16_f16", - 32}, + 16, + 16, + 8, + 32, + &lm_wmma_16x16x16_a, + &lm_wmma_16x16x16_b, + &lm_wmma_16x16x16_c}, {"wmma", - {{"bf16", 16, &lm_wmma_16x16x16_src0}, - {"bf16", 16, &lm_wmma_16x16x16_src1}, - {"fp32", 8, &lm_wmma_16x16x16_src2}}, - {"fp32", 8, &lm_wmma_16x16x16_dst}, + "bf16", + "bf16", + "fp32", 16, 16, 16, "wmma_f32_16x16x16_bf16", - 32}, + 16, + 16, + 8, + 32, + &lm_wmma_16x16x16_a, + &lm_wmma_16x16x16_b, + &lm_wmma_16x16x16_c}, {"wmma", - {{"iu8", 4, &lm_wmma_iu8_src0}, {"iu8", 4, &lm_wmma_iu8_src1}, {"i32", 8, &lm_wmma_iu8_src2}}, - {"i32", 8, &lm_wmma_iu8_dst}, + "iu8", + "iu8", + "i32", 16, 16, 16, "wmma_i32_16x16x16_iu8", - 32}, + 4, + 4, + 8, + 32, + &lm_wmma_iu8_a, + &lm_wmma_iu8_b, + &lm_wmma_iu8_c}, {"wmma", - {{"iu4", 2, &lm_wmma_iu4_src0}, {"iu4", 2, &lm_wmma_iu4_src1}, {"i32", 8, &lm_wmma_iu4_src2}}, - {"i32", 8, &lm_wmma_iu4_dst}, + "iu4", + "iu4", + "i32", 16, 16, 16, "wmma_i32_16x16x16_iu4", - 32}, + 2, + 2, + 8, + 32, + &lm_wmma_iu4_a, + &lm_wmma_iu4_b, + &lm_wmma_iu4_c}, }; /* ----------------------------- gfx1250 (CDNA/GFX12) ---------------------- */ -/* gfx1250: wave32, WMMA only (no MFMA). The fp32 K=4 atom has two fp32 values - * per lane in src0 and src1; src2 and dst use separate - * descriptors with the same gfx12 column-distributed row/column function. - * The primary fp16/bf16 atom is 16x16x32 (K=32, not the gfx12 RDNA - * 16x16x16). Its src0 and src1 have 16 fp16/bf16 values per lane. FP8/BF8 - * K=64 atoms carry eight packed i32 values per lane in src0 and src1; those - * source layout maps remain NULL pending the gfx1250 ISA backend port - * (rocke_ll_backend_for). */ +/* gfx1250: wave32, WMMA only (no MFMA). fp32 K=4 atom: A/B are scalar float + * per lane (same lane layout as mfma_f32_16x16x4_f32 on CDNA — reuses + * lm_mfma_16x16x4_f32_{a,b}); accumulator is <8 x float> in the gfx12 + * column-distributed layout (lm_wmma_gfx12_c, c_frag_len=8). + * Primary fp16/bf16 atom is 16x16x32 (K=32, not the gfx12 RDNA 16x16x16). + * A/B frag_len=16 (16 fp16/bf16 elements per lane); A/B layout maps are NULL + * pending the gfx1250 ISA backend port (rocke_ll_backend_for); the accumulator + * reuses lm_wmma_gfx12_c (same 16x16 tile layout). FP8/BF8 K=64 atoms carry + * frag_len=8 (8 i32 per lane) with NULL A/B maps. */ static const rocke_mma_op_t k_mma_gfx1250[] = { {"wmma", - {{"fp32", 2, &lm_wmma_gfx1250_f32_src0}, - {"fp32", 2, &lm_wmma_gfx1250_f32_src1}, - {"fp32", 8, &lm_wmma_gfx12_src2}}, - {"fp32", 8, &lm_wmma_gfx12_dst}, + "fp32", + "fp32", + "fp32", 16, 16, 4, "wmma_gfx1250_f32_16x16x4_f32", - 32}, + 2, + 2, + 8, + 32, + &lm_wmma_gfx1250_f32_a, + &lm_wmma_gfx1250_f32_b, + &lm_wmma_gfx12_c}, {"wmma", - {{"fp16", 16, &lm_wmma_gfx1250_32_src0}, - {"fp16", 16, &lm_wmma_gfx1250_32_src1}, - {"fp32", 8, &lm_wmma_gfx12_src2}}, - {"fp32", 8, &lm_wmma_gfx12_dst}, + "fp16", + "fp16", + "fp32", 16, 16, 32, "wmma_gfx1250_f32_16x16x32_f16", - 32}, + 16, + 16, + 8, + 32, + &lm_wmma_gfx1250_32_a, + &lm_wmma_gfx1250_32_b, + &lm_wmma_gfx12_c}, {"wmma", - {{"bf16", 16, &lm_wmma_gfx1250_32_src0}, - {"bf16", 16, &lm_wmma_gfx1250_32_src1}, - {"fp32", 8, &lm_wmma_gfx12_src2}}, - {"fp32", 8, &lm_wmma_gfx12_dst}, + "bf16", + "bf16", + "fp32", 16, 16, 32, "wmma_gfx1250_f32_16x16x32_bf16", - 32}, + 16, + 16, + 8, + 32, + &lm_wmma_gfx1250_32_a, + &lm_wmma_gfx1250_32_b, + &lm_wmma_gfx12_c}, {"wmma", - {{"fp8e4m3", 8, NULL}, {"fp8e4m3", 8, NULL}, {"fp32", 8, &lm_wmma_gfx12_src2}}, - {"fp32", 8, &lm_wmma_gfx12_dst}, + "fp8e4m3", + "fp8e4m3", + "fp32", 16, 16, 64, "wmma_gfx1250_f32_16x16x64_fp8_fp8", - 32}, + 8, + 8, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c}, {"wmma", - {{"fp8e4m3", 8, NULL}, {"bf8e5m2", 8, NULL}, {"fp32", 8, &lm_wmma_gfx12_src2}}, - {"fp32", 8, &lm_wmma_gfx12_dst}, + "fp8e4m3", + "bf8e5m2", + "fp32", 16, 16, 64, "wmma_gfx1250_f32_16x16x64_fp8_bf8", - 32}, + 8, + 8, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c}, {"wmma", - {{"bf8e5m2", 8, NULL}, {"fp8e4m3", 8, NULL}, {"fp32", 8, &lm_wmma_gfx12_src2}}, - {"fp32", 8, &lm_wmma_gfx12_dst}, + "bf8e5m2", + "fp8e4m3", + "fp32", 16, 16, 64, "wmma_gfx1250_f32_16x16x64_bf8_fp8", - 32}, + 8, + 8, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c}, {"wmma", - {{"bf8e5m2", 8, NULL}, {"bf8e5m2", 8, NULL}, {"fp32", 8, &lm_wmma_gfx12_src2}}, - {"fp32", 8, &lm_wmma_gfx12_dst}, + "bf8e5m2", + "bf8e5m2", + "fp32", 16, 16, 64, "wmma_gfx1250_f32_16x16x64_bf8_bf8", - 32}, + 8, + 8, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c}, {"wmma_scaled", - {{"fp8e4m3", 16, NULL, "e8m0", 32}, - {"fp8e4m3", 16, NULL, "e8m0", 32}, - {"fp32", 8, &lm_wmma_gfx12_src2}}, - {"fp32", 8, &lm_wmma_gfx12_dst}, + "fp8e4m3", + "fp8e4m3", + "fp32", 16, 16, 128, "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32", - 32}, + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + {"e8m0", 32}, + {"e8m0", 32}}, {"wmma_scaled", - {{"bf8e5m2", 16, NULL, "e8m0", 32}, - {"bf8e5m2", 16, NULL, "e8m0", 32}, - {"fp32", 8, &lm_wmma_gfx12_src2}}, - {"fp32", 8, &lm_wmma_gfx12_dst}, + "bf8e5m2", + "bf8e5m2", + "fp32", 16, 16, 128, "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32", - 32}, + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + {"e8m0", 32}, + {"e8m0", 32}}, {"wmma_scaled", - {{"fp8e4m3", 16, NULL, "e8m0", 16}, - {"fp8e4m3", 16, NULL, "e8m0", 16}, - {"fp32", 8, &lm_wmma_gfx12_src2}}, - {"fp32", 8, &lm_wmma_gfx12_dst}, + "fp8e4m3", + "fp8e4m3", + "fp32", 16, 16, 128, "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32", - 32}, + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + {"e8m0", 16}, + {"e8m0", 16}}, {"wmma_scaled", - {{"bf8e5m2", 16, NULL, "e8m0", 16}, - {"bf8e5m2", 16, NULL, "e8m0", 16}, - {"fp32", 8, &lm_wmma_gfx12_src2}}, - {"fp32", 8, &lm_wmma_gfx12_dst}, + "bf8e5m2", + "bf8e5m2", + "fp32", 16, 16, 128, "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32", - 32}, + 16, + 16, + 8, + 32, + NULL, + NULL, + &lm_wmma_gfx12_c, + {"e8m0", 16}, + {"e8m0", 16}}, }; /* ========================================================================= diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/helpers.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/helpers.cpp index 9eadf48cc081..5f74ed7ddc7d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/helpers.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/helpers.cpp @@ -40,10 +40,17 @@ const rocke_arch_layout_map_t* rocke_mmaop_b_layout(const rocke_mmaop_t* op, roc const rocke_arch_layout_map_t* rocke_mmaop_c_layout(const rocke_mmaop_t* op, rocke_ir_builder_t* b) { - /* Historical MmaOp.c_layout(): result (row, col) lane/slot map. */ + /* MmaOp.c_layout(): accumulator (row, col) lane/slot map. */ return rocke_mma_op_c_layout(op, b); } +const rocke_arch_layout_map_t* rocke_mmaop_acc_layout(const rocke_mmaop_t* op, + rocke_ir_builder_t* b) +{ + /* MmaOp.acc_layout(): alias for the accumulator (C) map. */ + return rocke_mma_op_acc_layout(op, b); +} + bool rocke_arch_layout_map_coord(const rocke_arch_layout_map_t* m, rocke_ir_builder_t* b, rocke_value_t* lane, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp index 5ef09ff437d8..3d7fda13359c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include "rocke/arch_target.h" #include "rocke/arch_target_internal.h" @@ -49,36 +48,89 @@ static const char* rocke_ati_family_or_default(const char* family) return family ? family : "mma"; } -/* ============================== MMA atom getters ====================== */ +static const char* rocke_ati_scale_dtype(const char* dtype) +{ + return dtype && strcmp(dtype, "fp8e4m3") == 0 ? "e4m3" : dtype; +} + +static void rocke_ati_validate_scales(const rocke_mma_scale_operand_t* scales) +{ + if(!scales) + return; + for(int i = 0; i < 2; ++i) + { + const char* dtype = rocke_ati_scale_dtype(scales[i].dtype); + if(dtype && strcmp(dtype, "e8m0") != 0 && strcmp(dtype, "e4m3") != 0 + && strcmp(dtype, "e5m3") != 0) + ckc::raise_status(ROCKE_ERR_VALUE, "MMA scale dtype must be e8m0, e4m3, or e5m3"); + if((dtype && scales[i].block_size != 16 && scales[i].block_size != 32) + || (!dtype && scales[i].block_size != 0)) + ckc::raise_status(ROCKE_ERR_VALUE, + "MMA scale block_size must be an integer equal to 16 or 32"); + } +} + +static bool rocke_ati_scales_match(const rocke_mma_op_t* op, + const rocke_mma_scale_operand_t* scales) +{ + if(!scales) + return true; + const rocke_mma_scale_operand_t actual[2] = {op->a_scale, op->b_scale}; + for(int i = 0; i < 2; ++i) + { + const char* expected = rocke_ati_scale_dtype(scales[i].dtype); + const char* dtype = rocke_ati_scale_dtype(actual[i].dtype); + if(bool(expected) != bool(dtype) || (dtype && strcmp(expected, dtype) != 0) + || scales[i].block_size != actual[i].block_size) + return false; + } + return true; +} const char* rocke_mma_op_semantic_id(const rocke_mma_op_t* op, char* out, size_t cap) { if(!op || !out || !cap) return NULL; - std::string id = op->family; - for(char& c : id) - if(c == '_') - c = '.'; - id += "." + std::to_string(op->m) + "x" + std::to_string(op->n) + "x" + std::to_string(op->k); - char dtype_buf[64]; - for(int i = 0; i < 3; ++i) + const rocke_mma_scale_operand_t scales[2] = {op->a_scale, op->b_scale}; + rocke_ati_validate_scales(scales); + int n = snprintf(out, cap, "%s.%dx%dx%d", op->family, op->m, op->n, op->k); + if(n < 0 || (size_t)n >= cap) + return NULL; + for(size_t i = 0; i < strlen(op->family); ++i) + if(out[i] == '_') + out[i] = '.'; + size_t used = (size_t)n; + const char* dtypes[2] = {op->a_dtype, op->b_dtype}; + char buf[64]; + for(int i = 0; i < 2; ++i) { - const rocke_mma_src_t& src = op->srcs[i]; - id += ".src" + std::to_string(i) + "_" - + rocke_normalize_dtype(src.dtype, dtype_buf, sizeof(dtype_buf)); - if(src.scale_dtype) + n = snprintf(out + used, + cap - used, + ".src%d_%s", + i, + rocke_normalize_dtype(dtypes[i], buf, sizeof(buf))); + if(n < 0 || (size_t)n >= cap - used) + return NULL; + used += (size_t)n; + if(scales[i].dtype) { - const char* scale = strcmp(src.scale_dtype, "fp8e4m3") == 0 ? "e4m3" : src.scale_dtype; - id += std::string("_") + scale + "_b" + std::to_string(src.scale_block_size); + n = snprintf(out + used, + cap - used, + "_%s_b%d", + rocke_ati_scale_dtype(scales[i].dtype), + scales[i].block_size); + if(n < 0 || (size_t)n >= cap - used) + return NULL; + used += (size_t)n; } } - id += std::string(".dst_") + rocke_normalize_dtype(op->dst.dtype, dtype_buf, sizeof(dtype_buf)); - if(id.size() >= cap) - return NULL; - memcpy(out, id.c_str(), id.size() + 1); - return out; + const char* acc = rocke_normalize_dtype(op->c_dtype, buf, sizeof(buf)); + n = snprintf(out + used, cap - used, ".src2_%s.dst_%s", acc, acc); + return n < 0 || (size_t)n >= cap - used ? NULL : out; } +/* ============================== MMA atom getters ====================== */ + void rocke_mma_op_shape(const rocke_mma_op_t* op, int* m, int* n, int* k) { if(!op) @@ -125,34 +177,23 @@ static const rocke_layout_map_t* rocke_ati_require_layout(const rocke_mma_op_t* const rocke_layout_map_t* rocke_mma_op_a_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b) { - return rocke_mma_op_src_layout(op, 0, b); + return rocke_ati_require_layout(op, op ? op->a_layout : NULL, "a", b); } const rocke_layout_map_t* rocke_mma_op_b_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b) { - return rocke_mma_op_src_layout(op, 1, b); + return rocke_ati_require_layout(op, op ? op->b_layout : NULL, "b", b); } const rocke_layout_map_t* rocke_mma_op_c_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b) { - return rocke_mma_op_dst_layout(op, b); -} - -const rocke_layout_map_t* - rocke_mma_op_src_layout(const rocke_mma_op_t* op, int index, rocke_ir_builder_t* b) -{ - if(index < 0 || index >= 3) - { - rocke_ati_q_set_err(b, ROCKE_ERR_VALUE, "MMA src index %d is outside [0, 3)", index); - } - char role[8]; - (void)snprintf(role, sizeof(role), "src%d", index); - return rocke_ati_require_layout(op, op ? op->srcs[index].layout : NULL, role, b); + return rocke_ati_require_layout(op, op ? op->c_layout : NULL, "c", b); } -const rocke_layout_map_t* rocke_mma_op_dst_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b) +const rocke_layout_map_t* rocke_mma_op_acc_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b) { - return rocke_ati_require_layout(op, op ? op->dst.layout : NULL, "dst", b); + /* Python alias: acc_layout() == c_layout(). */ + return rocke_mma_op_c_layout(op, b); } /* ============================== MMA catalog =========================== */ @@ -173,96 +214,57 @@ const rocke_mma_op_t* rocke_mma_catalog_ops(const rocke_mma_catalog_t* cat, int* /* Shared predicate matching MmaCatalog.enumerate's per-op filter. dtypes here * are already-normalised canonical keys (a/b/c). family is non-NULL. m/n < 0 * mean "any" (Python None). */ -static void rocke_ati_validate_scales(const rocke_mma_scale_operand_t* scales) -{ - if(!scales) - return; - for(int i = 0; i < 3; ++i) - { - const char* dtype = scales[i].dtype; - if(dtype && strcmp(dtype, "fp8e4m3") == 0) - dtype = "e4m3"; - if(dtype && strcmp(dtype, "e8m0") != 0 && strcmp(dtype, "e4m3") != 0 - && strcmp(dtype, "e5m3") != 0) - ckc::raise_status(ROCKE_ERR_VALUE, "MMA scale dtype must be e8m0, e4m3, or e5m3"); - if((dtype && scales[i].block_size != 16 && scales[i].block_size != 32) - || (!dtype && scales[i].block_size != 0)) - ckc::raise_status(ROCKE_ERR_VALUE, "invalid MMA scale block size"); - } -} - -static bool rocke_ati_scales_match(const rocke_mma_op_t* op, - const rocke_mma_scale_operand_t* scales) -{ - if(!scales) - return true; - for(int i = 0; i < 3; ++i) - { - const char* dtype = scales[i].dtype; - if(dtype && strcmp(dtype, "fp8e4m3") == 0) - dtype = "e4m3"; - const char* actual = op->srcs[i].scale_dtype; - if(bool(dtype) != bool(actual) || (dtype && strcmp(dtype, actual) != 0) - || scales[i].block_size != op->srcs[i].scale_block_size) - return false; - } - return true; -} - static bool rocke_ati_op_matches(const rocke_mma_op_t* op, const char* family, const char* a, const char* b, const char* c, - const char* d, int m, - int n) + int n, + const rocke_mma_scale_operand_t* scales) { if(strcmp(op->family, family) != 0) return false; - if(strcmp(op->srcs[0].dtype, a) != 0) - return false; - if(strcmp(op->srcs[1].dtype, b) != 0) + if(strcmp(op->a_dtype, a) != 0) return false; - if(strcmp(op->srcs[2].dtype, c) != 0) + if(strcmp(op->b_dtype, b) != 0) return false; - if(strcmp(op->dst.dtype, d) != 0) + if(strcmp(op->c_dtype, c) != 0) return false; if(m >= 0 && op->m != m) return false; if(n >= 0 && op->n != n) return false; - return true; + return rocke_ati_scales_match(op, scales); } -int rocke_mma_catalog_enumerate_indexed(const rocke_mma_catalog_t* cat, - const char* family, - const char* const src_dtypes[3], - const char* dst_dtype, - const rocke_mma_scale_operand_t src_scales[3], - int m, - int n, - const rocke_mma_op_t** out, - int cap) +int rocke_mma_catalog_enumerate(const rocke_mma_catalog_t* cat, + const char* family, + const char* a_dtype, + const char* b_dtype, + const char* c_dtype, + int m, + int n, + const rocke_mma_op_t** out, + int cap, + const rocke_mma_scale_operand_t scales[2]) { - char abuf[64], bbuf[64], cbuf[64], dbuf[64]; - const char *a, *bd, *c, *d, *fam; + char abuf[64], bbuf[64], cbuf[64]; + const char *a, *bd, *c, *fam; int i, total = 0; - if(!cat || !src_dtypes) + if(!cat) return 0; - rocke_ati_validate_scales(src_scales); + rocke_ati_validate_scales(scales); fam = rocke_ati_family_or_default(family); - a = rocke_normalize_dtype(src_dtypes[0], abuf, sizeof abuf); - bd = rocke_normalize_dtype(src_dtypes[1], bbuf, sizeof bbuf); - c = rocke_normalize_dtype(src_dtypes[2], cbuf, sizeof cbuf); - d = rocke_normalize_dtype(dst_dtype, dbuf, sizeof dbuf); + a = rocke_normalize_dtype(a_dtype, abuf, sizeof abuf); + bd = rocke_normalize_dtype(b_dtype, bbuf, sizeof bbuf); + c = rocke_normalize_dtype(c_dtype, cbuf, sizeof cbuf); for(i = 0; i < cat->num_ops; ++i) { const rocke_mma_op_t* op = &cat->ops[i]; - if(!rocke_ati_op_matches(op, fam, a, bd, c, d, m, n) - || !rocke_ati_scales_match(op, src_scales)) + if(!rocke_ati_op_matches(op, fam, a, bd, c, m, n, scales)) continue; if(out && total < cap) out[total] = op; @@ -271,49 +273,33 @@ int rocke_mma_catalog_enumerate_indexed(const rocke_mma_catalog_t* cat, return total; } -int rocke_mma_catalog_enumerate(const rocke_mma_catalog_t* cat, - const char* family, - const char* a_dtype, - const char* b_dtype, - const char* c_dtype, - int m, - int n, - const rocke_mma_op_t** out, - int cap) -{ - const char* src_dtypes[3] = {a_dtype, b_dtype, c_dtype}; - return rocke_mma_catalog_enumerate_indexed( - cat, family, src_dtypes, c_dtype, NULL, m, n, out, cap); -} - -bool rocke_mma_catalog_has_shape_indexed(const rocke_mma_catalog_t* cat, - const char* family, - const char* const src_dtypes[3], - const char* dst_dtype, - const rocke_mma_scale_operand_t src_scales[3], - int m, - int n, - int k) +bool rocke_mma_catalog_has_shape(const rocke_mma_catalog_t* cat, + const char* family, + const char* a_dtype, + const char* b_dtype, + const char* c_dtype, + int m, + int n, + int k, + const rocke_mma_scale_operand_t scales[2]) { - char abuf[64], bbuf[64], cbuf[64], dbuf[64]; - const char *a, *bd, *c, *d, *fam; + char abuf[64], bbuf[64], cbuf[64]; + const char *a, *bd, *c, *fam; int i; - if(!cat || !src_dtypes) + if(!cat) return false; - rocke_ati_validate_scales(src_scales); + rocke_ati_validate_scales(scales); fam = rocke_ati_family_or_default(family); - a = rocke_normalize_dtype(src_dtypes[0], abuf, sizeof abuf); - bd = rocke_normalize_dtype(src_dtypes[1], bbuf, sizeof bbuf); - c = rocke_normalize_dtype(src_dtypes[2], cbuf, sizeof cbuf); - d = rocke_normalize_dtype(dst_dtype, dbuf, sizeof dbuf); + a = rocke_normalize_dtype(a_dtype, abuf, sizeof abuf); + bd = rocke_normalize_dtype(b_dtype, bbuf, sizeof bbuf); + c = rocke_normalize_dtype(c_dtype, cbuf, sizeof cbuf); /* Python enumerates with m=m, n=n then checks op.shape == (m, n, k). */ for(i = 0; i < cat->num_ops; ++i) { const rocke_mma_op_t* op = &cat->ops[i]; - if(!rocke_ati_op_matches(op, fam, a, bd, c, d, m, n) - || !rocke_ati_scales_match(op, src_scales)) + if(!rocke_ati_op_matches(op, fam, a, bd, c, m, n, scales)) continue; if(op->m == m && op->n == n && op->k == k) return true; @@ -321,49 +307,34 @@ bool rocke_mma_catalog_has_shape_indexed(const rocke_mma_catalog_t* cat, return false; } -bool rocke_mma_catalog_has_shape(const rocke_mma_catalog_t* cat, - const char* family, - const char* a_dtype, - const char* b_dtype, - const char* c_dtype, - int m, - int n, - int k) -{ - const char* src_dtypes[3] = {a_dtype, b_dtype, c_dtype}; - return rocke_mma_catalog_has_shape_indexed(cat, family, src_dtypes, c_dtype, NULL, m, n, k); -} - -const rocke_mma_op_t* - rocke_mma_catalog_select_largest_k_indexed(const rocke_mma_catalog_t* cat, - const char* family, - const char* const src_dtypes[3], - const char* dst_dtype, - const rocke_mma_scale_operand_t src_scales[3], - int m, - int n, - int k_max) +const rocke_mma_op_t* rocke_mma_catalog_select_largest_k(const rocke_mma_catalog_t* cat, + const char* family, + const char* a_dtype, + const char* b_dtype, + const char* c_dtype, + int m, + int n, + int k_max, + const rocke_mma_scale_operand_t scales[2]) { - char abuf[64], bbuf[64], cbuf[64], dbuf[64]; - const char *a, *bd, *c, *d, *fam; + char abuf[64], bbuf[64], cbuf[64]; + const char *a, *bd, *c, *fam; const rocke_mma_op_t* best = NULL; - bool ambiguous = false; int i; + bool ambiguous = false; - if(!cat || !src_dtypes) + if(!cat) return NULL; - rocke_ati_validate_scales(src_scales); + rocke_ati_validate_scales(scales); fam = rocke_ati_family_or_default(family); - a = rocke_normalize_dtype(src_dtypes[0], abuf, sizeof abuf); - bd = rocke_normalize_dtype(src_dtypes[1], bbuf, sizeof bbuf); - c = rocke_normalize_dtype(src_dtypes[2], cbuf, sizeof cbuf); - d = rocke_normalize_dtype(dst_dtype, dbuf, sizeof dbuf); + a = rocke_normalize_dtype(a_dtype, abuf, sizeof abuf); + bd = rocke_normalize_dtype(b_dtype, bbuf, sizeof bbuf); + c = rocke_normalize_dtype(c_dtype, cbuf, sizeof cbuf); for(i = 0; i < cat->num_ops; ++i) { const rocke_mma_op_t* op = &cat->ops[i]; - if(!rocke_ati_op_matches(op, fam, a, bd, c, d, m, n) - || !rocke_ati_scales_match(op, src_scales)) + if(!rocke_ati_op_matches(op, fam, a, bd, c, m, n, scales)) continue; if(k_max >= 0 && op->k > k_max) continue; /* Python: k_max is None || op.k <= k_max */ @@ -381,20 +352,6 @@ const rocke_mma_op_t* return best; } -const rocke_mma_op_t* rocke_mma_catalog_select_largest_k(const rocke_mma_catalog_t* cat, - const char* family, - const char* a_dtype, - const char* b_dtype, - const char* c_dtype, - int m, - int n, - int k_max) -{ - const char* src_dtypes[3] = {a_dtype, b_dtype, c_dtype}; - return rocke_mma_catalog_select_largest_k_indexed( - cat, family, src_dtypes, c_dtype, NULL, m, n, k_max); -} - const rocke_mma_op_t* rocke_mma_catalog_by_op_id(const rocke_mma_catalog_t* cat, const char* op_id) { int i; @@ -409,36 +366,34 @@ const rocke_mma_op_t* rocke_mma_catalog_by_op_id(const rocke_mma_catalog_t* cat, return NULL; } -const rocke_mma_op_t* - rocke_mma_catalog_op_for_shape_indexed(const rocke_mma_catalog_t* cat, - const char* family, - const char* const src_dtypes[3], - const char* dst_dtype, - const rocke_mma_scale_operand_t src_scales[3], - int m, - int n, - int k) +const rocke_mma_op_t* rocke_mma_catalog_op_for_shape(const rocke_mma_catalog_t* cat, + const char* family, + const char* a_dtype, + const char* b_dtype, + const char* c_dtype, + int m, + int n, + int k, + const rocke_mma_scale_operand_t scales[2]) { - char abuf[64], bbuf[64], cbuf[64], dbuf[64]; - const char *a, *bd, *c, *d, *fam; + char abuf[64], bbuf[64], cbuf[64]; + const char *a, *bd, *c, *fam; int i; const rocke_mma_op_t* match = NULL; - if(!cat || !src_dtypes) + if(!cat) return NULL; - rocke_ati_validate_scales(src_scales); + rocke_ati_validate_scales(scales); fam = rocke_ati_family_or_default(family); - a = rocke_normalize_dtype(src_dtypes[0], abuf, sizeof abuf); - bd = rocke_normalize_dtype(src_dtypes[1], bbuf, sizeof bbuf); - c = rocke_normalize_dtype(src_dtypes[2], cbuf, sizeof cbuf); - d = rocke_normalize_dtype(dst_dtype, dbuf, sizeof dbuf); + a = rocke_normalize_dtype(a_dtype, abuf, sizeof abuf); + bd = rocke_normalize_dtype(b_dtype, bbuf, sizeof bbuf); + c = rocke_normalize_dtype(c_dtype, cbuf, sizeof cbuf); - /* Exact selection rejects an underspecified contract. */ + /* Exact selection requires a unique contract. */ for(i = 0; i < cat->num_ops; ++i) { const rocke_mma_op_t* op = &cat->ops[i]; - if(!rocke_ati_op_matches(op, fam, a, bd, c, d, m, n) - || !rocke_ati_scales_match(op, src_scales)) + if(!rocke_ati_op_matches(op, fam, a, bd, c, m, n, scales)) continue; if(op->k == k) { @@ -451,32 +406,19 @@ const rocke_mma_op_t* return match; } -const rocke_mma_op_t* rocke_mma_catalog_op_for_shape(const rocke_mma_catalog_t* cat, - const char* family, - const char* a_dtype, - const char* b_dtype, - const char* c_dtype, - int m, - int n, - int k) -{ - const char* src_dtypes[3] = {a_dtype, b_dtype, c_dtype}; - return rocke_mma_catalog_op_for_shape_indexed(cat, family, src_dtypes, c_dtype, NULL, m, n, k); -} - /* ===================== bare-op_id SSOT lookups ======================== */ -const char* rocke_arch_mma_op_id_dst_dtype(const char* op_id) +const char* rocke_arch_mma_op_id_c_dtype(const char* op_id) { int i; if(!op_id) { return NULL; } - /* Mirrors target._op_id_dst_dtype()[op_id]: the dst dtype names a + /* Mirrors target._op_id_c_dtype()[op_id]: the accumulator dtype names a * specific atom, so it is invariant across the arches that list op_id -- - * the first catalog hit wins. The catalog dst.dtype is already the normalised - * (canonical) key, matching normalize_dtype(o["dst"]["dtype"]). Op_ids absent from + * the first catalog hit wins. The catalog c_dtype is already the normalised + * (canonical) key, matching normalize_dtype(o["c"]). Op_ids absent from * every catalog return NULL. */ for(i = 0; i < rocke_ati_arch_registry_len; ++i) { @@ -484,7 +426,7 @@ const char* rocke_arch_mma_op_id_dst_dtype(const char* op_id) const rocke_mma_op_t* op = t ? rocke_mma_catalog_by_op_id(&t->mma, op_id) : NULL; if(op) { - return op->dst.dtype; + return op->c_dtype; } } return NULL; @@ -526,25 +468,13 @@ bool rocke_arch_fits_lds(const rocke_arch_target_t* t, long bytes_in_use) return bytes_in_use <= (long)t->lds_capacity_bytes; } -bool rocke_arch_supports_dtype_combo_indexed(const rocke_arch_target_t* t, - const char* const src_dtypes[3], - const char* dst_dtype, - const rocke_mma_scale_operand_t src_scales[3], - const char* family) +bool rocke_arch_supports_dtype_combo( + const rocke_arch_target_t* t, const char* a, const char* b, const char* c, const char* family) { if(!t) return false; /* Python: len(enumerate(...)) > 0 (m/n omitted => None => "any"). */ - return rocke_mma_catalog_enumerate_indexed( - &t->mma, family, src_dtypes, dst_dtype, src_scales, -1, -1, NULL, 0) - > 0; -} - -bool rocke_arch_supports_dtype_combo( - const rocke_arch_target_t* t, const char* a, const char* b, const char* c, const char* family) -{ - const char* src_dtypes[3] = {a, b, c}; - return rocke_arch_supports_dtype_combo_indexed(t, src_dtypes, c, NULL, family); + return rocke_mma_catalog_enumerate(&t->mma, family, a, b, c, -1, -1, NULL, 0) > 0; } int rocke_arch_max_vector_load_dwords(const rocke_arch_target_t* t, const char* dtype) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp index 48aeb992b0c3..a77f7656c774 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp @@ -25,10 +25,10 @@ /* * rocke_b_mma emits a single tile.mma op keyed by op_id; the ISA backend lowers * that op_id to the matching MFMA/WMMA call. To size the result vector it needs - * the dst fragment length and dtype for the atom. Both come from the - * arch SSOT (core/arch/target): frag lengths from rocke_arch_mma_dst_frag_len - * (the _MMA_FRAGMENT_INFO projection) and the dst dtype from - * rocke_arch_mma_op_id_dst_dtype (the JSON catalog aggregation). This bucket keeps + * the accumulator fragment length and dtype for the atom. Both come from the + * arch SSOT (core/arch/target): frag lengths from rocke_arch_mma_c_frag_len + * (the _MMA_FRAGMENT_INFO projection) and the accumulator dtype from + * rocke_arch_mma_op_id_c_dtype (the JSON catalog aggregation). This bucket keeps * NO private copy of that data, mirroring ir.py after it dropped _MMA_C_FRAG_LEN * / _MMA_C_INT_OP_IDS. * @@ -51,19 +51,20 @@ static const rocke_mma_hint_row_t ROCKE_MMA_RESULT_HINT[] = { {"mfma_scale_f32_16x16x128_f8f6f4", "mxacc"}, }; -/* dst fragment length for op_id, from the arch SSOT - * (target._frag_info(op_id).dst.frag_len). Returns <= 0 for an unknown atom (the +/* accumulator fragment length for op_id, from the arch SSOT + * (target._frag_info(op_id).c_frag_len). Returns <= 0 for an unknown atom (the * zero-length _frag_info fallback), which rocke_b_mma reports as an error. */ -static int rocke_mma_dst_frag_len(const char* op_id) +static int rocke_mma_c_frag_len(const char* op_id) { - return rocke_arch_mma_dst_frag_len(op_id); + return rocke_arch_mma_c_frag_len(op_id); } -/* True when op_id produces i32, from the arch catalog SSOT. */ -static bool rocke_mma_dst_is_int(const char* op_id) +/* True when op_id accumulates in i32 (integer WMMA), from the arch catalog SSOT + * (target._op_id_c_dtype()[op_id] == "i32"). */ +static bool rocke_mma_is_int_acc(const char* op_id) { - const char* dst_dtype = rocke_arch_mma_op_id_dst_dtype(op_id); - return dst_dtype != NULL && strcmp(dst_dtype, ROCKE_DTYPE_I32) == 0; + const char* c_dtype = rocke_arch_mma_op_id_c_dtype(op_id); + return c_dtype != NULL && strcmp(c_dtype, ROCKE_DTYPE_I32) == 0; } static const char* rocke_mma_result_hint(const char* op_id) @@ -550,9 +551,9 @@ rocke_value_t* rocke_b_mma(rocke_ir_builder_t* b, rocke_value_t* const* extra, int num_extra) { - int dst_frag_len; - bool is_int_dst; - const rocke_type_t* dst_elem; + int c_frag_len; + bool is_int_acc; + const rocke_type_t* c_elem; const rocke_type_t* vt; const char* hint; rocke_attr_map_t attrs; @@ -567,19 +568,19 @@ rocke_value_t* rocke_b_mma(rocke_ir_builder_t* b, { return (rocke_value_t*)rocke_i_set_err(b, ROCKE_ERR_VALUE, "mma op_id is NULL"); } - /* The C API has no MmaOp object; op_id is always a bare string, so the dst - * frag length and element type are resolved from the arch SSOT (the Python + /* C has no MmaOp object; op_id is always a bare string, so the frag length + * and accumulator element are resolved from the arch SSOT (the Python * bare-string code path). A <= 0 frag length means the op_id is unknown to * the SSOT (the zero-length _frag_info fallback). */ - dst_frag_len = rocke_mma_dst_frag_len(op_id); - if(dst_frag_len <= 0) + c_frag_len = rocke_mma_c_frag_len(op_id); + if(c_frag_len <= 0) { return (rocke_value_t*)rocke_i_set_err( b, ROCKE_ERR_VALUE, "unknown MMA op_id '%s'; pass a known mfma_*/wmma_* op_id", op_id); } - is_int_dst = rocke_mma_dst_is_int(op_id); - dst_elem = is_int_dst ? rocke_i32() : rocke_f32(); - vt = rocke_vector_type(b, dst_elem, dst_frag_len); + is_int_acc = rocke_mma_is_int_acc(op_id); + c_elem = is_int_acc ? rocke_i32() : rocke_f32(); + vt = rocke_vector_type(b, c_elem, c_frag_len); if(!vt) { return NULL; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp index ed05b5e5b330..152641849784 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp @@ -567,7 +567,7 @@ void rocke_cshuffle_epilogue_store(rocke_ir_builder_t* b, } /* ---- step 1: publish accs to LDS. ---- - * WMMA path: use rocke_mma_op_dst_layout(op).coord() for the scatter. + * WMMA path: use op->c_layout().coord() for the scatter. * MFMA path: use atom->lane_to_output(). */ for(mi = 0; mi < mfmas_m; ++mi) { @@ -589,7 +589,7 @@ void rocke_cshuffle_epilogue_store(rocke_ir_builder_t* b, if(epi->mma_op != NULL) { - c_per_lane = epi->mma_op->dst.frag_len; + c_per_lane = epi->mma_op->c_frag_len; atom_m = epi->mma_op->m; atom_n = epi->mma_op->n; } @@ -858,7 +858,7 @@ void rocke_cshuffle_epilogue_atomic_store(rocke_ir_builder_t* b, if(epi->mma_op != NULL) { - c_per_lane = epi->mma_op->dst.frag_len; + c_per_lane = epi->mma_op->c_frag_len; atom_m = epi->mma_op->m; atom_n = epi->mma_op->n; } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mfma_attention.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mfma_attention.cpp index d925455b3b78..b9102aa0cb03 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mfma_attention.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/mfma_attention.cpp @@ -722,10 +722,10 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, return rocke_ir_builder_status(b); } - const rocke_layout_map_t* src0_map = op->srcs[0].layout; - const rocke_layout_map_t* dst_map = op->dst.layout; - int src0_frag = op->srcs[0].frag_len; - int dst_frag = op->dst.frag_len; + const rocke_layout_map_t* a_map = op->a_layout; + const rocke_layout_map_t* c_map = op->c_layout; + int a_frag = op->a_frag_len; + int c_frag = op->c_frag_len; int n_dk = head_size / 16; /* Python evaluates b.mod(b.thread_id_x(), b.const_i32(wave)) left-to-right: @@ -738,22 +738,22 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, rocke_value_t* a_row = NULL; rocke_value_t* dummy = NULL; - rocke_layout_map_coord(src0_map, b, lane, 0, &a_row, &dummy); + rocke_layout_map_coord(a_map, b, lane, 0, &a_row, &dummy); /* gfx12 (RDNA4) split-K: the 16 K-elements of one WMMA step are split across - * the two lane-halves, so each lane loads src0_frag (=8) elements from K base - * (lane // 16) * src0_frag. gfx11 (RDNA3/3.5) duplicates the full K row in every - * lane (src0_frag=16, base 0). k_half_off==NULL keeps the gfx11 emission + * the two lane-halves, so each lane loads a_frag (=8) elements from K base + * (lane // 16) * a_frag. gfx11 (RDNA3/3.5) duplicates the full K row in every + * lane (a_frag=16, base 0). k_half_off==NULL keeps the gfx11 emission * byte-identical (no half-offset add); mirrors Python's split_k handling. */ - bool split_k = (src0_frag * 2 == 16); + bool split_k = (a_frag * 2 == 16); rocke_value_t* k_half_off = NULL; if(split_k) { - /* Python: b.mul(b.div(lane, c16), b.const_i32(src0_frag)) -- div created + /* Python: b.mul(b.div(lane, c16), b.const_i32(a_frag)) -- div created * before the const (left-to-right). Hoist so the C arg-eval order (gcc is * right-to-left) matches the Python value-creation order. */ rocke_value_t* half = rocke_b_div(b, lane, c16); - rocke_value_t* src0_frag_off = rocke_b_const_i32(b, src0_frag); - k_half_off = rocke_b_mul(b, half, src0_frag_off); + rocke_value_t* c_frag_off = rocke_b_const_i32(b, a_frag); + k_half_off = rocke_b_mul(b, half, c_frag_off); } rocke_value_t* col = rocke_b_mod(b, lane, c16); @@ -779,7 +779,7 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, { q_addr = rocke_b_add(b, q_addr, k_half_off); } - q_frags[d] = rocke_b_global_load_vN(b, p->Q, q_addr, dtype_ir, src0_frag, src0_frag * 2); + q_frags[d] = rocke_b_global_load_vN(b, p->Q, q_addr, dtype_ir, a_frag, a_frag * 2); } /* ---- LDS staging tiles ---- */ @@ -796,7 +796,7 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, rocke_iter_arg_t iter_args[ROCKE_ATTN_MAX_ITER_ARGS]; char name_buf[ROCKE_ATTN_MAX_ITER_ARGS][16]; int n_ia = 0; - for(int r = 0; r < dst_frag; ++r) + for(int r = 0; r < c_frag; ++r) { snprintf(name_buf[n_ia], sizeof(name_buf[0]), "m%d", r); iter_args[n_ia].name = name_buf[n_ia]; @@ -811,7 +811,7 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, { snprintf(name_buf[n_ia], sizeof(name_buf[0]), "acc%d", d); iter_args[n_ia].name = name_buf[n_ia]; - iter_args[n_ia].init = rocke_b_zero_vec_f32(b, dst_frag); + iter_args[n_ia].init = rocke_b_zero_vec_f32(b, c_frag); ++n_ia; } @@ -829,14 +829,14 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, rocke_value_t* ms[ROCKE_ATTN_MAX_LANE]; rocke_value_t* ls[ROCKE_ATTN_MAX_LANE]; rocke_value_t* accs[ROCKE_ATTN_MAX_ATOMS]; - for(int r = 0; r < dst_frag; ++r) + for(int r = 0; r < c_frag; ++r) { ms[r] = kloop.iter_vars[2 * r]; ls[r] = kloop.iter_vars[2 * r + 1]; } for(int d = 0; d < n_dk; ++d) { - accs[d] = kloop.iter_vars[2 * dst_frag + d]; + accs[d] = kloop.iter_vars[2 * c_frag + d]; } rocke_value_t* effective_kt @@ -870,7 +870,7 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, } /* ---- QK^T WMMA chain ---- */ - rocke_value_t* score = rocke_b_zero_vec_f32(b, dst_frag); + rocke_value_t* score = rocke_b_zero_vec_f32(b, c_frag); for(int d = 0; d < n_dk; ++d) { rocke_value_t* k_addr = rocke_b_add(b, k_addr_row_base, rocke_b_const_i32(b, d * 16)); @@ -879,7 +879,7 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, k_addr = rocke_b_add(b, k_addr, k_half_off); } rocke_value_t* k_frag - = rocke_b_global_load_vN(b, p->K, k_addr, dtype_ir, src0_frag, src0_frag * 2); + = rocke_b_global_load_vN(b, p->K, k_addr, dtype_ir, a_frag, a_frag * 2); score = rocke_b_mma(b, op->op_id, q_frags[d], k_frag, score, NULL, 0); } @@ -893,11 +893,11 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, new_accs[d] = accs[d]; } rocke_value_t* q_pos_for_mask = (p->q_pos_base != NULL) ? p->q_pos_base : p->q_tile_base; - for(int r = 0; r < dst_frag; ++r) + for(int r = 0; r < c_frag; ++r) { rocke_value_t* row_rel = NULL; rocke_value_t* col_k = NULL; - rocke_layout_map_coord(dst_map, b, lane, r, &row_rel, &col_k); + rocke_layout_map_coord(c_map, b, lane, r, &row_rel, &col_k); rocke_value_t* s_r = rocke_b_fmul(b, rocke_b_vec_extract(b, score, r), p->scale_log2); rocke_value_t* row_q_pos = rocke_b_add(b, q_pos_for_mask, row_rel); rocke_value_t* k_col_pos = rocke_b_add(b, k_tile_base, col_k); @@ -964,23 +964,23 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, } /* ---- P staging through LDS ---- */ - for(int r = 0; r < dst_frag; ++r) + for(int r = 0; r < c_frag; ++r) { rocke_value_t* row_rel = NULL; rocke_value_t* col_k = NULL; - rocke_layout_map_coord(dst_map, b, lane, r, &row_rel, &col_k); + rocke_layout_map_coord(c_map, b, lane, r, &row_rel, &col_k); rocke_value_t* idx[2] = {row_rel, col_k}; rocke_b_smem_store_vN(b, P_lds, idx, 2, rocke_b_cast_f32_to(b, ps_arr[r], dtype_ir), 1); } rocke_b_sync(b); /* ---- V load + PV WMMA chain ---- */ - rocke_value_t* p_a = rocke_b_zero_vec(b, dtype_ir, src0_frag); - for(int j = 0; j < src0_frag; ++j) + rocke_value_t* p_a = rocke_b_zero_vec(b, dtype_ir, a_frag); + for(int j = 0; j < a_frag; ++j) { rocke_value_t* a_k = NULL; rocke_value_t* a_dummy = NULL; - rocke_layout_map_coord(src0_map, b, lane, j, &a_dummy, &a_k); + rocke_layout_map_coord(a_map, b, lane, j, &a_dummy, &a_k); rocke_value_t* idx[2] = {a_row, a_k}; rocke_value_t* p_v = rocke_b_vec_extract(b, rocke_b_smem_load_vN(b, P_lds, idx, 2, dtype_ir, 1), 0); @@ -990,12 +990,12 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, for(int d = 0; d < n_dk; ++d) { rocke_value_t* d_col = rocke_b_add(b, rocke_b_const_i32(b, d * 16), col); - rocke_value_t* v_b = rocke_b_zero_vec(b, dtype_ir, src0_frag); - for(int j = 0; j < src0_frag; ++j) + rocke_value_t* v_b = rocke_b_zero_vec(b, dtype_ir, a_frag); + for(int j = 0; j < a_frag; ++j) { /* B-operand K row this lane's slot j feeds: j on gfx11 (full K * per lane, byte-identical to the historical literal), or - * (lane // 16) * src0_frag + j on gfx12 (split-K halves). Python: + * (lane // 16) * a_frag + j on gfx12 (split-K halves). Python: * b.add(k_half_off, b.const_i32(j)) -- k_half_off created before * the const, so it is the first add operand. */ rocke_value_t* b_k = (k_half_off != NULL) @@ -1033,7 +1033,7 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, rocke_value_t* yields[ROCKE_ATTN_MAX_ITER_ARGS]; int ny = 0; - for(int r = 0; r < dst_frag; ++r) + for(int r = 0; r < c_frag; ++r) { yields[ny++] = new_ms[r]; yields[ny++] = new_ls[r]; @@ -1048,23 +1048,23 @@ rocke_status_t rocke_wmma_attention_fwd_inner_body(rocke_ir_builder_t* b, rocke_value_t* ls_final[ROCKE_ATTN_MAX_LANE]; rocke_value_t* accs_final[ROCKE_ATTN_MAX_ATOMS]; - for(int r = 0; r < dst_frag; ++r) + for(int r = 0; r < c_frag; ++r) { ls_final[r] = (kloop.op != NULL) ? kloop.op->results[2 * r + 1] : NULL; } for(int d = 0; d < n_dk; ++d) { - accs_final[d] = (kloop.op != NULL) ? kloop.op->results[2 * dst_frag + d] : NULL; + accs_final[d] = (kloop.op != NULL) ? kloop.op->results[2 * c_frag + d] : NULL; } /* ---- Epilogue ---- */ for(int d = 0; d < n_dk; ++d) { - for(int r = 0; r < dst_frag; ++r) + for(int r = 0; r < c_frag; ++r) { rocke_value_t* row_rel = NULL; rocke_value_t* col_n = NULL; - rocke_layout_map_coord(dst_map, b, lane, r, &row_rel, &col_n); + rocke_layout_map_coord(c_map, b, lane, r, &row_rel, &col_n); rocke_value_t* l_safe = ls_final[r]; rocke_value_t* zero_mask = rocke_b_fcmp(b, "oeq", l_safe, zero_f); rocke_value_t* inv_l = rocke_b_select(b, zero_mask, zero_f, rocke_b_rcp(b, l_safe)); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h index e6aff19d876b..b8fd90966070 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h @@ -17,7 +17,7 @@ * .fn closure rocke_lane_coord_fn (fn ptr + builder) * .coord(builder, lane, slot) rocke_layout_map_coord() * class MmaOp (frozen) rocke_mma_op_t - * .shape / .src_layout()/... rocke_mma_op_shape() / *_layout() getters + * .shape / .a_layout()/... rocke_mma_op_shape() / *_layout() getters * class MemoryCapabilities rocke_memory_caps_t * class ResourceLimits rocke_resource_limits_t * class MmaCatalog rocke_mma_catalog_t + rocke_mma_catalog_* () @@ -58,14 +58,14 @@ extern "C" { /* ============================== layout map ============================== */ -/* Machine-level MMA operand position. The stored contract is - * src0/src1/src2 -> dst. */ +/* MMA fragment role. Mirrors LayoutMap.role ("acc"/"a"/"b"). The accumulator is + * spelled "c" in the C op_id maps (MmaOp.c_layout) but role text is "acc"/"a"/ + * "b" exactly as Python LayoutMap.role. */ typedef enum rocke_mma_role { - ROCKE_MMA_ROLE_SRC0 = 0, /* src0: coords (row, k) */ - ROCKE_MMA_ROLE_SRC1, /* src1: coords (k, col) */ - ROCKE_MMA_ROLE_SRC2, /* src2: coords (row, col) */ - ROCKE_MMA_ROLE_DST /* dst: coords (row, col) */ + ROCKE_MMA_ROLE_ACC = 0, /* accumulator C/D: coords (row, col) */ + ROCKE_MMA_ROLE_A, /* A operand: coords (row, k) */ + ROCKE_MMA_ROLE_B /* B operand: coords (k, col) */ } rocke_mma_role_t; /* The lane/slot -> tile-coordinate emitter. Given the builder, a runtime i32 @@ -103,70 +103,56 @@ bool rocke_layout_map_coord(const rocke_layout_map_t* m, /* ============================== MMA atom =============================== */ -/* Metadata shared by one matrix src or dst. */ -typedef struct rocke_mma_dst -{ - const char* dtype; - int frag_len; - const rocke_layout_map_t* layout; /* may be NULL */ -} rocke_mma_dst_t; - -/* A matrix source may have a distinct associated scale operand. NULL means - * that the source is unscaled. Scales are not matrix fragments and therefore - * are not flattened into the srcs[] array. */ -/* Query scale descriptor. A NULL query array leaves scales unconstrained; - * an array entry {NULL, 0} explicitly requests an unscaled source. */ +/* Logical scale format and source elements along K per scale. + * {NULL, 0} means unscaled. */ typedef struct rocke_mma_scale_operand { const char* dtype; int block_size; } rocke_mma_scale_operand_t; -typedef struct rocke_mma_src -{ - const char* dtype; - int frag_len; - const rocke_layout_map_t* layout; /* may be NULL */ - const char* scale_dtype; /* scale value format: e8m0/e4m3/e5m3; NULL if unscaled */ - int scale_block_size; /* source elements along K per scale: 16 or 32; 0 if unscaled */ -} rocke_mma_src_t; - +/* A supported matrix-multiply-accumulate atom. Frozen, in static storage. + * Mirrors MmaOp: family is "mma", "wmma", or "wmma_scaled"; matrix dtypes + * are canonical catalog keys. C describes both accumulator input and result. + * op_id is an opaque backend handle; scaled WMMA uses semantic contract IDs. + * Layout pointers are NULL when no verified lane map is registered. */ typedef struct rocke_mma_op { const char* family; - rocke_mma_src_t srcs[3]; - rocke_mma_dst_t dst; + const char* a_dtype; + const char* b_dtype; + const char* c_dtype; int m; int n; int k; const char* op_id; + int a_frag_len; + int b_frag_len; + int c_frag_len; int wave_size; + const rocke_layout_map_t* a_layout; /* may be NULL */ + const rocke_layout_map_t* b_layout; /* may be NULL */ + const rocke_layout_map_t* c_layout; /* may be NULL */ + rocke_mma_scale_operand_t a_scale; + rocke_mma_scale_operand_t b_scale; } rocke_mma_op_t; -/* Format semantic identity; NULL if the output buffer is too small. */ +/* Write the semantic contract ID into caller-owned out; NULL on truncation. */ const char* rocke_mma_op_semantic_id(const rocke_mma_op_t* op, char* out, size_t cap); /* MmaOp.shape -> (m, n, k) via out params. */ void rocke_mma_op_shape(const rocke_mma_op_t* op, int* m, int* n, int* k); -/* Canonical physical-layout accessors. Each returns the verified map for the - * machine operand position, or NULL when none is registered. Source indices - * outside [0, 3) are rejected. */ -const rocke_layout_map_t* - rocke_mma_op_src_layout(const rocke_mma_op_t* op, int index, rocke_ir_builder_t* b); -const rocke_layout_map_t* rocke_mma_op_dst_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b); - -/* Historical mathematical-role compatibility accessors. A and B project src0 - * and src1; C projects the result/dst, matching the original operator-level - * surface. src2 is available only through the indexed - * accessor above. Each returns the verified map, or NULL when none is - * registered. Unlike Python (which raises NotImplementedError), the C getters - * return NULL and -- when `b` is non-NULL -- set the builder's sticky error - * (ROCKE_ERR_NOTIMPL) with the same message text, so callers can either check - * NULL or rely on the sticky-fail builder. Pass b=NULL for a pure lookup. */ +/* Physical-layout accessors. Each returns the verified map for the role, or NULL + * when none is registered. Unlike Python (which raises NotImplementedError), the + * C getters return NULL and -- when `b` is non-NULL -- set the builder's sticky + * error (ROCKE_ERR_NOTIMPL) with the same message text, so callers can either + * check NULL or rely on the sticky-fail builder. Pass b=NULL for a pure lookup. + * Mirrors MmaOp.a_layout / b_layout / c_layout / acc_layout. */ const rocke_layout_map_t* rocke_mma_op_a_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b); const rocke_layout_map_t* rocke_mma_op_b_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b); const rocke_layout_map_t* rocke_mma_op_c_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b); +const rocke_layout_map_t* rocke_mma_op_acc_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b); /* ====================== memory caps / resource limits ================== */ @@ -198,27 +184,21 @@ typedef struct rocke_mma_catalog /* MmaCatalog.ops accessor (count + pointer). */ const rocke_mma_op_t* rocke_mma_catalog_ops(const rocke_mma_catalog_t* cat, int* num_out); -/* Indexed MmaCatalog.enumerate: filter atoms by family + the exact three src - * dtypes and dst dtype. `src_dtypes` must contain three entries. - * `src_scales` is NULL for an unconstrained query, or three exact descriptors. - * Filter by optional m/n. Writes up to `cap` matching op pointers into `out` and returns - * the total number of matches (which may exceed `cap`). */ -int rocke_mma_catalog_enumerate_indexed(const rocke_mma_catalog_t* cat, - const char* family, - const char* const src_dtypes[3], - const char* dst_dtype, - const rocke_mma_scale_operand_t src_scales[3], - int m, - int n, - const rocke_mma_op_t** out, - int cap); - -/* Compatibility MmaCatalog.enumerate: filter atoms by family + A/B/C dtypes and - * `src_scales` is NULL for an unconstrained query, or three exact descriptors. - * Filter by optional m/n. Writes up to `cap` matching op pointers into `out` and returns +/* Optional A/B scale pair: NULL leaves scales unconstrained; {NULL, 0} + * selects an unscaled input. C callers supply the pair explicitly. + * Exact selection and largest-K ties raise a query error on ambiguity. + * As with layout queries, C++ callers catch ckc::Error at their entry boundary. */ +#ifdef __cplusplus +#define ROCKE_MMA_DEFAULT_SCALES = nullptr +#else +#define ROCKE_MMA_DEFAULT_SCALES +#endif + +/* MmaCatalog.enumerate: filter atoms by family + (normalised) dtype combo and + * optional m/n. Writes up to `cap` matching op pointers into `out` and returns * the total number of matches (which may exceed `cap` -- the caller can size a * buffer with cap=0 to count first). Pass m<0 / n<0 to mean "any" (Python None). - * `family` may be NULL => "mma". The dst dtype defaults to C. */ + * `family` may be NULL => "mma". */ int rocke_mma_catalog_enumerate(const rocke_mma_catalog_t* cat, const char* family, const char* a_dtype, @@ -227,68 +207,49 @@ int rocke_mma_catalog_enumerate(const rocke_mma_catalog_t* cat, int m, int n, const rocke_mma_op_t** out, - int cap); - -/* Indexed and compatibility MmaCatalog.has_shape variants. */ -bool rocke_mma_catalog_has_shape_indexed(const rocke_mma_catalog_t* cat, - const char* family, - const char* const src_dtypes[3], - const char* dst_dtype, - const rocke_mma_scale_operand_t src_scales[3], - int m, - int n, - int k); -bool rocke_mma_catalog_has_shape(const rocke_mma_catalog_t* cat, - const char* family, - const char* a_dtype, - const char* b_dtype, - const char* c_dtype, - int m, - int n, - int k); - -/* Indexed and compatibility MmaCatalog.select_largest_k variants. */ -const rocke_mma_op_t* - rocke_mma_catalog_select_largest_k_indexed(const rocke_mma_catalog_t* cat, - const char* family, - const char* const src_dtypes[3], - const char* dst_dtype, - const rocke_mma_scale_operand_t src_scales[3], - int m, - int n, - int k_max); -const rocke_mma_op_t* rocke_mma_catalog_select_largest_k(const rocke_mma_catalog_t* cat, - const char* family, - const char* a_dtype, - const char* b_dtype, - const char* c_dtype, - int m, - int n, - int k_max); + int cap, + const rocke_mma_scale_operand_t scales[2] ROCKE_MMA_DEFAULT_SCALES); + +/* MmaCatalog.has_shape. family NULL => "mma". */ +bool rocke_mma_catalog_has_shape( + const rocke_mma_catalog_t* cat, + const char* family, + const char* a_dtype, + const char* b_dtype, + const char* c_dtype, + int m, + int n, + int k, + const rocke_mma_scale_operand_t scales[2] ROCKE_MMA_DEFAULT_SCALES); + +/* MmaCatalog.select_largest_k: the matching atom with the largest k (k <= k_max + * if k_max >= 0; pass k_max<0 for Python None). Returns NULL if none match. */ +const rocke_mma_op_t* rocke_mma_catalog_select_largest_k( + const rocke_mma_catalog_t* cat, + const char* family, + const char* a_dtype, + const char* b_dtype, + const char* c_dtype, + int m, + int n, + int k_max, + const rocke_mma_scale_operand_t scales[2] ROCKE_MMA_DEFAULT_SCALES); /* MmaCatalog.by_op_id. Returns NULL if absent. */ const rocke_mma_op_t* rocke_mma_catalog_by_op_id(const rocke_mma_catalog_t* cat, const char* op_id); -/* Indexed and compatibility MmaCatalog.op_for_shape variants. An ambiguous - * query raises ROCKE_ERR_VALUE, as does a tie at the largest eligible K in - * select_largest_k. Omitted scales never silently pick one contract. */ -const rocke_mma_op_t* - rocke_mma_catalog_op_for_shape_indexed(const rocke_mma_catalog_t* cat, - const char* family, - const char* const src_dtypes[3], - const char* dst_dtype, - const rocke_mma_scale_operand_t src_scales[3], - int m, - int n, - int k); -const rocke_mma_op_t* rocke_mma_catalog_op_for_shape(const rocke_mma_catalog_t* cat, - const char* family, - const char* a_dtype, - const char* b_dtype, - const char* c_dtype, - int m, - int n, - int k); +/* MmaCatalog.op_for_shape: the atom with exactly (m, n, k). NULL if absent. + * family NULL => "mma". */ +const rocke_mma_op_t* rocke_mma_catalog_op_for_shape( + const rocke_mma_catalog_t* cat, + const char* family, + const char* a_dtype, + const char* b_dtype, + const char* c_dtype, + int m, + int n, + int k, + const rocke_mma_scale_operand_t scales[2] ROCKE_MMA_DEFAULT_SCALES); /* ===================== bare-op_id SSOT lookups ======================== */ /* These resolve a bare op_id string (no MmaOp / ArchTarget in hand) against the @@ -296,17 +257,20 @@ const rocke_mma_op_t* rocke_mma_catalog_op_for_shape(const rocke_mma_catalog_t* * any atom metadata. They mirror the target.py module-level helpers that * rocke.core.ir.IRBuilder.mma consults. */ -/* dst fragment length for op_id -- the dst.frag_len projection of - * target.py::_MMA_FRAGMENT_INFO. Returns 0 for an unknown atom (the +#undef ROCKE_MMA_DEFAULT_SCALES + +/* Accumulator fragment length for op_id -- the c_frag_len projection of + * target.py::_MMA_FRAGMENT_INFO (rocke_arch_mma_c_frag_len(op_id) == + * target._frag_info(op_id).c_frag_len). Returns 0 for an unknown atom (the * zero-length _FragInfo fallback). */ -int rocke_arch_mma_dst_frag_len(const char* op_id); +int rocke_arch_mma_c_frag_len(const char* op_id); /* Normalised accumulator dtype for op_id, aggregated across every arch's * catalog (mirrors target._op_id_c_dtype()[op_id]). The dtype is invariant * across the arches that list an op_id, so the first catalog hit wins. Returns * NULL for an op_id absent from every catalog. The returned string is an * interned canonical key (e.g. ROCKE_DTYPE_I32 / "fp32"); do not free. */ -const char* rocke_arch_mma_op_id_dst_dtype(const char* op_id); +const char* rocke_arch_mma_op_id_c_dtype(const char* op_id); /* ============================== arch target =========================== */ @@ -339,12 +303,7 @@ const char* rocke_arch_isa_triple(const rocke_arch_target_t* t, char* out, size_ /* ArchTarget.fits_lds. */ bool rocke_arch_fits_lds(const rocke_arch_target_t* t, long bytes_in_use); -/* Indexed and compatibility ArchTarget.supports_dtype_combo variants. */ -bool rocke_arch_supports_dtype_combo_indexed(const rocke_arch_target_t* t, - const char* const src_dtypes[3], - const char* dst_dtype, - const rocke_mma_scale_operand_t src_scales[3], - const char* family); +/* ArchTarget.supports_dtype_combo. family NULL => "mma". */ bool rocke_arch_supports_dtype_combo( const rocke_arch_target_t* t, const char* a, const char* b, const char* c, const char* family); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h index 77f74d564742..1fc0f9462403 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h @@ -63,15 +63,16 @@ typedef rocke_mma_role_t rocke_arch_mma_role_t; /* MmaOp.shape -> (m, n, k). Forwards to rocke_mma_op_shape. */ void rocke_mmaop_shape(const rocke_mmaop_t* op, int* m, int* n, int* k); -/* Historical MmaOp.a_layout() / b_layout() / c_layout() projections. C is the - * result/dst map; src2 is available through the canonical indexed - * API. Each returns the verified lane/slot -> coord map for the role, or NULL - * when none is registered for the op_id (Python raises NotImplementedError; - * here, when `b` is non-NULL, the builder's sticky error is set -- pass b=NULL - * for a pure lookup). */ +/* MmaOp.a_layout() / b_layout() / c_layout() / acc_layout(). Each returns the + * verified lane/slot -> coord map for the role, or NULL when none is registered + * for the op_id (Python raises NotImplementedError; here, when `b` is non-NULL, + * the builder's sticky error is set -- pass b=NULL for a pure lookup). Forwards + * to the canonical rocke_mma_op_*_layout accessors. */ const rocke_arch_layout_map_t* rocke_mmaop_a_layout(const rocke_mmaop_t* op, rocke_ir_builder_t* b); const rocke_arch_layout_map_t* rocke_mmaop_b_layout(const rocke_mmaop_t* op, rocke_ir_builder_t* b); const rocke_arch_layout_map_t* rocke_mmaop_c_layout(const rocke_mmaop_t* op, rocke_ir_builder_t* b); +const rocke_arch_layout_map_t* rocke_mmaop_acc_layout(const rocke_mmaop_t* op, + rocke_ir_builder_t* b); /* LayoutMap.coord(builder, lane, slot) -> (coord0, coord1). Forwards to * rocke_layout_map_coord (validates slot in [0, frag_len); emits arith via b). */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.epilogues.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.epilogues.h index 2df8a195df13..7f2e7f55f73a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.epilogues.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.epilogues.h @@ -195,7 +195,7 @@ rocke_cshuffle_epilogue_t rocke_cshuffle_epilogue_from_grid(const rocke_mfma_ato int max_store_vec); /* CShuffleEpilogue.from_grid_op(op, grid, max_store_vec=8): WMMA variant. - * Uses op->dst.frag_len / rocke_mma_op_dst_layout(op) for the LDS scatter in place of the + * Uses op->c_frag_len / op->c_layout() for the LDS scatter in place of the * MFMA-specific atom.lane_to_output. */ rocke_cshuffle_epilogue_t rocke_cshuffle_epilogue_from_grid_op(const rocke_mmaop_t* op, const rocke_warp_grid_t* grid, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gemm_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gemm_internal.h index 362a57c2cd57..57c09cff9b96 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gemm_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gemm_internal.h @@ -256,7 +256,7 @@ typedef struct rocke_gemm_build_ctx /* _atom_frag_lengths(op) -> (a,b,c)_frag_len. Pure; fills out-params. */ void rocke_gemm_atom_frag_lengths(const rocke_mmaop_t* op, int* a_frag, int* b_frag, int* c_frag); -/* _emit_zero_acc_op(b, op): zero_vec_f32(op->dst.frag_len). Used to build +/* _emit_zero_acc_op(b, op): zero_vec_f32(op->c_frag_len). Used to build * acc_init. (The MFMA-only _emit_zero_acc / _mfma_atom_widths variants are not * reached by the contract-driven body; kept here for parity with the Python * module surface.) */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h index be5c3e83b22e..59ae31feced3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h @@ -985,7 +985,7 @@ rocke_value_t* rocke_b_vec_concat(rocke_ir_builder_t* b, rocke_value_t* a, rocke /* ----- ISA-named MMA wrappers (thin wrappers over rocke_b_mma; kept for parity * with the legacy Python helpers so emitters can call them by name). All take - * (src0, src1, src2) and return . The scaled MX atom takes the + * (a, b, c) and return . The scaled MX atom takes the * two extra E8M0 scale operands. */ rocke_value_t* rocke_b_mfma_f32_16x16x16_f16(rocke_ir_builder_t* b, rocke_value_t* a, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h index 39f804f813db..8d3be8d5ff1d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h @@ -48,26 +48,27 @@ static inline rocke_scaled_wmma_op_t rocke_scaled_wmma_contract(const rocke_mma_ { rocke_scaled_wmma_op_t spec = {}; spec.op_id = atom->op_id; + const char* dtypes[2] = {atom->a_dtype, atom->b_dtype}; + const int words[2] = {atom->a_frag_len, atom->b_frag_len}; + const rocke_mma_scale_operand_t scales[2] = {atom->a_scale, atom->b_scale}; for(int i = 0; i < 2; ++i) { - const rocke_mma_src_t* src = &atom->srcs[i]; - if(strcmp(src->dtype, "fp8e4m3") == 0) + if(strcmp(dtypes[i], "fp8e4m3") == 0) spec.matrix_formats[i] = 0; - else if(strcmp(src->dtype, "bf8e5m2") == 0) + else if(strcmp(dtypes[i], "bf8e5m2") == 0) spec.matrix_formats[i] = 1; else ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA matrix format"); - if(!src->scale_dtype || strcmp(src->scale_dtype, "e8m0") != 0 - || (src->scale_block_size != 16 && src->scale_block_size != 32)) + if(!scales[i].dtype || strcmp(scales[i].dtype, "e8m0") != 0 + || (scales[i].block_size != 16 && scales[i].block_size != 32)) ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA scale format"); spec.scale_formats[i] = 0; // E8M0. - spec.matrix_words[i] = src->frag_len; + spec.matrix_words[i] = words[i]; } - if(atom->srcs[0].scale_block_size != atom->srcs[1].scale_block_size || atom->srcs[2].scale_dtype - || strcmp(atom->srcs[2].dtype, "fp32") != 0 || strcmp(atom->dst.dtype, "fp32") != 0 + if(atom->a_scale.block_size != atom->b_scale.block_size || strcmp(atom->c_dtype, "fp32") != 0 || atom->m != 16 || atom->n != 16 || atom->k != 128) ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA backend contract"); - spec.scales.block_k = atom->srcs[0].scale_block_size; + spec.scales.block_k = atom->a_scale.block_size; spec.scales.count = atom->k / spec.scales.block_k; char suffix[96]; snprintf(suffix, @@ -76,7 +77,7 @@ static inline rocke_scaled_wmma_op_t rocke_scaled_wmma_contract(const rocke_mma_ atom->m, atom->n, atom->k, - atom->dst.frag_len, + atom->c_frag_len, spec.matrix_words[0], spec.matrix_words[1]); snprintf(spec.intrinsic, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_build_glue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_build_glue.cpp index eebb5653ce27..c4c8a1929394 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_build_glue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_build_glue.cpp @@ -236,9 +236,9 @@ bool rocke_conv_build_ctx_init(rocke_conv_build_ctx_t* ctx, ? NULL : rocke_mfma_atom("f16", spec->warp_tile_m, spec->warp_tile_n, spec->warp_tile_k); - ctx->a_per_lane = ctx->op->srcs[0].frag_len; - ctx->b_per_lane = ctx->op->srcs[1].frag_len; - ctx->c_per_lane = ctx->op->dst.frag_len; + ctx->a_per_lane = ctx->op->a_frag_len; + ctx->b_per_lane = ctx->op->b_frag_len; + ctx->c_per_lane = ctx->op->c_frag_len; /* ---- block tile dims ---- (817) */ ctx->block_m = spec->tile_m; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_epilogue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_epilogue.cpp index 23d266352638..b7389eb0b7dc 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_epilogue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_epilogue.cpp @@ -321,7 +321,7 @@ void rocke_conv_emit_direct_epilogue_wmma(rocke_ir_builder_t* b, n_const = rocke_b_const_i32(b, ni * spec->warp_tile_n); atom_n_off = rocke_b_add(b, n_inner, n_const); - for(i = 0; i < op->dst.frag_len; ++i) + for(i = 0; i < op->c_frag_len; ++i) { rocke_value_t* row_off = NULL; rocke_value_t* col_off = NULL; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp index 51dfe0498ede..52f30b6e97e9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp @@ -1016,7 +1016,7 @@ static void _emit_dgrad_direct_epilogue_wmma(rocke_ir_builder_t* b, = rocke_b_add(b, _bwn, rocke_b_const_i32(b, ni * spec->warp_tile_n)); const rocke_arch_layout_map_t* c_map = rocke_mmaop_c_layout(op, b); - for(int i = 0; i < op->dst.frag_len; i++) + for(int i = 0; i < op->c_frag_len; i++) { rocke_value_t* row_off; rocke_value_t* col_off; @@ -1501,7 +1501,7 @@ static void _emit_dgrad_tilde_direct_epilogue_wmma(rocke_ir_builder_t* b, rocke_b_add(b, block_n_off, warp_n_off), rocke_b_const_i32(b, ni * spec->warp_tile_n)); - for(int i = 0; i < op->dst.frag_len; i++) + for(int i = 0; i < op->c_frag_len; i++) { rocke_value_t* row_off; rocke_value_t* col_off; @@ -1807,9 +1807,9 @@ static rocke_kernel_def_t* = is_wmma ? NULL : rocke_mfma_atom( spec->dtype_a, spec->warp_tile_m, spec->warp_tile_n, spec->warp_tile_k); - int a_per_lane = op->srcs[0].frag_len; - int b_per_lane = op->srcs[1].frag_len; - int c_per_lane = op->dst.frag_len; + int a_per_lane = op->a_frag_len; + int b_per_lane = op->b_frag_len; + int c_per_lane = op->c_frag_len; // ---- 1D grid: block_id_x covers all sub-GEMMs' tiles ---- rocke_value_t* flat_block_id = rocke_b_block_id_x(b); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_wgrad.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_wgrad.cpp index 8e54b541affe..b56df3714918 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_wgrad.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_wgrad.cpp @@ -1605,7 +1605,7 @@ static void wgrad_emit_direct_epilogue(rocke_ir_builder_t* b, rocke_value_t* n_const = rocke_b_const_i32(b, ni * spec->warp_tile_n); rocke_value_t* atom_n_off = rocke_b_add(b, n_inner, n_const); - for(int i = 0; i < op->dst.frag_len; ++i) + for(int i = 0; i < op->c_frag_len; ++i) { rocke_value_t* row_off = NULL; rocke_value_t* col_off = NULL; @@ -1801,9 +1801,9 @@ static bool wgrad_build_ctx_init(rocke_conv_build_ctx_t* ctx, spec->warp_tile_n, spec->warp_tile_k); } - ctx->a_per_lane = ctx->op->srcs[0].frag_len; - ctx->b_per_lane = ctx->op->srcs[1].frag_len; - ctx->c_per_lane = ctx->op->dst.frag_len; + ctx->a_per_lane = ctx->op->a_frag_len; + ctx->b_per_lane = ctx->op->b_frag_len; + ctx->c_per_lane = ctx->op->c_frag_len; /* Block tile dims */ ctx->block_m = spec->tile_m; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/deep_fused_conv_pool_conv1_gemm_and_lds_staging.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/deep_fused_conv_pool_conv1_gemm_and_lds_staging.cpp index c5c7448ebe46..58ca7fbc758c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/deep_fused_conv_pool_conv1_gemm_and_lds_staging.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/deep_fused_conv_pool_conv1_gemm_and_lds_staging.cpp @@ -132,7 +132,7 @@ static rocke_value_t* dfcp_load_smem_frag_contiguous_f16(rocke_ir_builder_t* b, * _stage_accumulators_to_cshuffle_lds (py 388-456) * * Publish MMA accumulators to a row-major [tile_m, tile_n] LDS tile, fully - * op-driven via op->dst.frag_len + rocke_mma_op_dst_layout(op)->coord + the per-warp tile + * op-driven via op->c_frag_len + op->c_layout()->coord + the per-warp tile * geometry. sync=false defers the trailing barrier so the caller can merge it * with the disjoint W1 producer barrier. * ================================================================== */ @@ -160,7 +160,7 @@ rocke_value_t* rocke_dfcp_stage_accumulators_to_cshuffle_lds(rocke_ir_builder_t* return NULL; } - c_frag_len = op->dst.frag_len; + c_frag_len = op->c_frag_len; mfmas_m = rocke_warp_grid_mfmas_per_warp_m(b, grid); mfmas_n = rocke_warp_grid_mfmas_per_warp_n(b, grid); if(!rocke_ir_builder_ok(b)) @@ -360,7 +360,7 @@ rocke_value_t* rocke_dfcp_load_conv1_weights_to_lds(rocke_ir_builder_t* b, * _emit_conv1_1x1 (py 816-910) * * Compute conv1 as a 1x1 GEMM over the staged conv0 activations, op-driven: - * operand lane->(mn,k) decomposition from rocke_mma_op_src_layout(op, 0/1)->coord, + * operand lane->(mn,k) decomposition from op->a_layout()/b_layout()->coord, * fragment widths op->{a,b}_frag_len, matmul via the target-neutral mma. The * K-tail mask is handled by load_smem_frag_contiguous_f16. defer_epilogue=true * returns the raw fp32 accumulators; else applies spec.conv1_epilogue. @@ -428,8 +428,8 @@ rocke_status_t rocke_dfcp_emit_conv1_1x1(rocke_ir_builder_t* b, return rocke_ir_builder_status(b); } - a_frag = op->srcs[0].frag_len; - b_frag = op->srcs[1].frag_len; + a_frag = op->a_frag_len; + b_frag = op->b_frag_len; mfmas_m = rocke_warp_grid_mfmas_per_warp_m(b, grid); mfmas_n = rocke_warp_grid_mfmas_per_warp_n(b, grid); if(!rocke_ir_builder_ok(b)) @@ -476,7 +476,7 @@ rocke_status_t rocke_dfcp_emit_conv1_1x1(rocke_ir_builder_t* b, int idx; for(idx = 0; idx < num_accs; ++idx) { - accs[idx] = rocke_b_zero_vec_f32(b, op->srcs[2].frag_len); + accs[idx] = rocke_b_zero_vec_f32(b, op->c_frag_len); } } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_build_compute.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_build_compute.cpp index d858f214cb08..b99998146b46 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_build_compute.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_build_compute.cpp @@ -30,7 +30,7 @@ * atom step inside this K-tile. On WMMA (RDNA) this delegates to the fully * contract-driven _emit_wmma_phase. On MFMA (CDNA) it keeps the byte-identical * arch-formula fragment load; the matmul + accumulator length are - * contract-driven (rocke_gemm_emit_mma / op->dst.frag_len). + * contract-driven (rocke_gemm_emit_mma / op->c_frag_len). * * lds_parity is split (per the internal-header convention) into a compile-time * int (parity_imm) OR a runtime Value (parity_v != NULL == Python isinstance diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_mma.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_mma.cpp index f403ec3fe83a..f057d592a199 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_mma.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_mma.cpp @@ -135,11 +135,11 @@ rocke_value_t* rocke_gemm_emit_zero_acc(rocke_ir_builder_t* b, void rocke_gemm_atom_frag_lengths(const rocke_mmaop_t* op, int* a_frag, int* b_frag, int* c_frag) { if(a_frag) - *a_frag = op->srcs[0].frag_len; + *a_frag = op->a_frag_len; if(b_frag) - *b_frag = op->srcs[1].frag_len; + *b_frag = op->b_frag_len; if(c_frag) - *c_frag = op->dst.frag_len; + *c_frag = op->c_frag_len; } /* ====================================================================== * @@ -163,7 +163,7 @@ rocke_value_t* rocke_gemm_emit_mma(rocke_ir_builder_t* b, * ====================================================================== */ rocke_value_t* rocke_gemm_emit_zero_acc_op(rocke_ir_builder_t* b, const rocke_mmaop_t* op) { - return rocke_b_zero_vec_f32(b, op->srcs[2].frag_len); + return rocke_b_zero_vec_f32(b, op->c_frag_len); } /* ====================================================================== * diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_scatter_maxpool_finalization_phases.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_scatter_maxpool_finalization_phases.cpp index 0b9afe6484e5..85672b4c693c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_scatter_maxpool_finalization_phases.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_scatter_maxpool_finalization_phases.cpp @@ -54,7 +54,7 @@ void rocke_gfx1151_dfcp_scatter_codes_to_lds(rocke_gfx1151_dfcp_build_ctx_t* ctx const rocke_layout_map_t* c_map = rocke_mma_op_c_layout(op, b); rocke_value_t* warp_m_off = rocke_warp_grid_warp_m_off(b, grid); rocke_value_t* warp_n_off = rocke_warp_grid_warp_n_off(b, grid); - int frag_len = op->dst.frag_len; + int frag_len = op->c_frag_len; int flat = 0; int mi, ni, i; (void)num_accs; @@ -105,7 +105,7 @@ void rocke_gfx1151_dfcp_scatter_codes_to_i8_lds(rocke_gfx1151_dfcp_build_ctx_t* const rocke_layout_map_t* c_map = rocke_mma_op_c_layout(op, b); rocke_value_t* warp_m_off = rocke_warp_grid_warp_m_off(b, grid); rocke_value_t* warp_n_off = rocke_warp_grid_warp_n_off(b, grid); - int frag_len = op->dst.frag_len; + int frag_len = op->c_frag_len; int flat = 0; int mi, ni, i; (void)num_accs; @@ -156,7 +156,7 @@ void rocke_gfx1151_dfcp_scatter_vec_codes_to_i8_lds(rocke_gfx1151_dfcp_build_ctx const rocke_layout_map_t* c_map = rocke_mma_op_c_layout(op, b); rocke_value_t* warp_m_off = rocke_warp_grid_warp_m_off(b, grid); rocke_value_t* warp_n_off = rocke_warp_grid_warp_n_off(b, grid); - int frag_len = op->dst.frag_len; + int frag_len = op->c_frag_len; int flat = 0; int mi, ni, i; (void)num_accs; @@ -208,7 +208,7 @@ void rocke_gfx1151_dfcp_scatter_packed_i4_codes_to_lds(rocke_gfx1151_dfcp_build_ const rocke_layout_map_t* c_map = rocke_mma_op_c_layout(op, b); rocke_value_t* warp_m_off = rocke_warp_grid_warp_m_off(b, grid); rocke_value_t* warp_n_off = rocke_warp_grid_warp_n_off(b, grid); - int frag_len = op->dst.frag_len; + int frag_len = op->c_frag_len; int flat = 0; int mi, ni, i; rocke_value_t* c0 = rocke_b_const_i32(b, 0); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_wmma_gemm_fragment_loaders_register_handoff.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_wmma_gemm_fragment_loaders_register_handoff.cpp index 13e71fef557b..bcc283f605fb 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_wmma_gemm_fragment_loaders_register_handoff.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_wmma_gemm_fragment_loaders_register_handoff.cpp @@ -372,12 +372,12 @@ void rocke_gfx1151_dfcp_wmma_gemm_from_lds(rocke_gfx1151_dfcp_build_ctx_t* ctx, dfcp_layout_coord0(b, b_map, grid->lane, &b_k, &b_col); rocke_value_t* warp_m_off = rocke_warp_grid_warp_m_off(b, grid); rocke_value_t* warp_n_off = rocke_warp_grid_warp_n_off(b, grid); - int ds_per_frag = (op->srcs[0].frag_len + 7) / 8; + int ds_per_frag = (op->a_frag_len + 7) / 8; int n_ds = ds_per_frag * (mfmas_m + mfmas_n); int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec_f32(b, op->srcs[2].frag_len); + out_accs[i] = rocke_b_zero_vec_f32(b, op->c_frag_len); for(int kk = 0; kk < k_atoms; ++kk) { @@ -389,14 +389,14 @@ void rocke_gfx1151_dfcp_wmma_gemm_from_lds(rocke_gfx1151_dfcp_build_ctx_t* ctx, rocke_value_t* atom_row = rocke_b_add(b, warp_m_off, rocke_b_const_i32(b, mi * DFCP_WMMA)); a_rows[mi] = rocke_conv_emit_frag_smem_load( - b, a_smem, a_row, a_k, atom_row, k_tile_base, op->srcs[0].frag_len); + b, a_smem, a_row, a_k, atom_row, k_tile_base, op->a_frag_len); } for(int ni = 0; ni < mfmas_n; ++ni) { rocke_value_t* atom_row = rocke_b_add(b, warp_n_off, rocke_b_const_i32(b, ni * DFCP_WMMA)); b_cols[ni] = rocke_conv_emit_frag_smem_load( - b, b_smem, b_col, b_k, atom_row, k_tile_base, op->srcs[1].frag_len); + b, b_smem, b_col, b_k, atom_row, k_tile_base, op->b_frag_len); } int flat = 0; for(int mi = 0; mi < mfmas_m; ++mi) @@ -436,7 +436,7 @@ void rocke_gfx1151_dfcp_wmma_gemm_from_lds_int(rocke_gfx1151_dfcp_build_ctx_t* c int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->srcs[2].frag_len); + out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->c_frag_len); for(int kk = 0; kk < k_atoms; ++kk) { @@ -490,11 +490,11 @@ void rocke_gfx1151_dfcp_wmma_gemm_conv0_direct(rocke_gfx1151_dfcp_build_ctx_t* c dfcp_layout_coord0(b, b_map, grid->lane, &b_k, &b_col); rocke_value_t* warp_m_off = rocke_warp_grid_warp_m_off(b, grid); rocke_value_t* warp_n_off = rocke_warp_grid_warp_n_off(b, grid); - int n_ds = op->srcs[0].frag_len * mfmas_m + ((op->srcs[1].frag_len + 7) / 8) * mfmas_n; + int n_ds = op->a_frag_len * mfmas_m + ((op->b_frag_len + 7) / 8) * mfmas_n; int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec_f32(b, op->srcs[2].frag_len); + out_accs[i] = rocke_b_zero_vec_f32(b, op->c_frag_len); for(int kk = 0; kk < k_atoms; ++kk) { @@ -508,14 +508,14 @@ void rocke_gfx1151_dfcp_wmma_gemm_conv0_direct(rocke_gfx1151_dfcp_build_ctx_t* c = rocke_b_add(b, warp_m_off, rocke_b_const_i32(b, mi * DFCP_WMMA)); rocke_value_t* m_row = rocke_b_add(b, atom_row, a_row); a_rows[mi] = rocke_gfx1151_dfcp_load_conv0_a_frag_from_footprint( - ctx, inp_smem, m_row, k_base, op->srcs[0].frag_len); + ctx, inp_smem, m_row, k_base, op->a_frag_len); } for(int ni = 0; ni < mfmas_n; ++ni) { rocke_value_t* atom_row = rocke_b_add(b, warp_n_off, rocke_b_const_i32(b, ni * DFCP_WMMA)); b_cols[ni] = rocke_conv_emit_frag_smem_load( - b, w0_smem, b_col, b_k, atom_row, k_tile_base, op->srcs[1].frag_len); + b, w0_smem, b_col, b_k, atom_row, k_tile_base, op->b_frag_len); } int flat = 0; for(int mi = 0; mi < mfmas_m; ++mi) @@ -555,7 +555,7 @@ void rocke_gfx1151_dfcp_wmma_gemm_conv0_direct_int(rocke_gfx1151_dfcp_build_ctx_ int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->srcs[2].frag_len); + out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->c_frag_len); for(int kk = 0; kk < k_atoms; ++kk) { @@ -626,7 +626,7 @@ void rocke_gfx1151_dfcp_wmma_gemm_conv1_i4_from_lds(rocke_gfx1151_dfcp_build_ctx int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->srcs[2].frag_len); + out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->c_frag_len); const void* per_step = sched_fuse ? NULL : policy; @@ -734,7 +734,7 @@ void rocke_gfx1151_dfcp_wmma_gemm_conv1_i4_packed_from_lds(rocke_gfx1151_dfcp_bu int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->srcs[2].frag_len); + out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->c_frag_len); for(int kk = 0; kk < k_atoms; ++kk) { @@ -796,7 +796,7 @@ void rocke_gfx1151_dfcp_wmma_gemm_conv1_i4_from_regs(rocke_gfx1151_dfcp_build_ct int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->srcs[2].frag_len); + out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->c_frag_len); const void* per_step = sched_fuse ? NULL : policy; @@ -882,7 +882,7 @@ void rocke_gfx1151_dfcp_wmma_gemm_conv1_i8_from_regs(rocke_gfx1151_dfcp_build_ct int n_acc = mfmas_m * mfmas_n; for(int i = 0; i < n_acc; ++i) - out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->srcs[2].frag_len); + out_accs[i] = rocke_b_zero_vec(b, rocke_i32(), op->c_frag_len); const void* per_step = sched_fuse ? NULL : policy; @@ -968,11 +968,11 @@ void rocke_gfx1151_dfcp_fuse_c0_to_conv1_a_regs(rocke_gfx1151_dfcp_build_ctx_t* rocke_value_t* acc = accs0[mi * mfmas_n + kk]; /* codes = [code_fn(b.vec_extract(acc, s)) for s in range(op0.c_frag_len)] */ rocke_value_t* codes[ROCKE_GFX1151_DFCP_MAX_ACCS]; - for(int s = 0; s < op0->dst.frag_len; ++s) + for(int s = 0; s < op0->c_frag_len; ++s) codes[s] = rocke_gfx1151_dfcp_apply_code_fn( ctx, code_fn, rocke_b_vec_extract(b, acc, s)); /* words = b.bitcast(b.vec_pack(codes, I8), VectorType(I32, 2)) */ - rocke_value_t* packed_i8 = rocke_b_vec_pack(b, codes, op0->dst.frag_len, rocke_i8()); + rocke_value_t* packed_i8 = rocke_b_vec_pack(b, codes, op0->c_frag_len, rocke_i8()); rocke_value_t* words = rocke_b_bitcast(b, packed_i8, rocke_vector_type(b, rocke_i32(), 2)); rocke_value_t* lo = rocke_b_vec_extract(b, words, 0); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md index bbc313f159f7..acfb38ef041c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md @@ -31,10 +31,11 @@ Matrix-operation choices start from the exact gfx target. Resolve and validate the operation's wave and layout contract. The exact catalog is listed in [`kernel_taxonomy.md`](./kernel_taxonomy.md). -MMA metadata uses `srcs[0]`, `srcs[1]`, `srcs[2]`, and `dst`: the first two -sources are multiplicands, the third is the accumulator input, and `dst` is -the result. A source's optional `MmaScaleOperand(dtype, block_size)` describes -its scale values independently of the matrix dtype and packed register type. +MMA metadata uses `a_dtype`, `b_dtype`, and `c_dtype`, with C describing both +the accumulator input and result. Optional `a_scale` and `b_scale` descriptors +(`MmaScaleOperand(dtype, block_size)`) describe scale values independently of +the matrix dtype and packed register type. Fragment and layout accessors +retain their A/B/C roles. For scaled operations, query the complete contract and pass the selected atom to `IRBuilder.mma`: @@ -45,19 +46,18 @@ from rocke.core.arch import ArchTarget, MmaScaleOperand scale = MmaScaleOperand("e8m0", 32) atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( family="wmma_scaled", - src_dtypes=("fp8", "fp8", "fp32"), - dst_dtype="fp32", - src_scales=(scale, scale, None), + a_dtype="fp8", b_dtype="fp8", c_dtype="fp32", + scales=(scale, scale), m=16, n=16, k=128, ) assert atom is not None -# result = builder.mma(atom, src0, src1, src2, scale0, scale1) +# result = builder.mma(atom, a, b, c, scale_a, scale_b) ``` -Omitting `src_scales` leaves scales unconstrained. Passing a three-entry tuple -matches every source exactly; `None` in that tuple means unscaled. Enumeration +Omitting `scales` leaves scales unconstrained. Passing an A/B pair matches both +inputs exactly; `None` in that pair means unscaled. Enumeration and existence queries may match several records, while `op_for_shape` and -`select_largest_k` reject an ambiguous selection. The C indexed query functions +`select_largest_k` reject ambiguous exact matches or largest-K ties. The C queries use the same rules: a NULL scale array is unconstrained, and `{NULL, 0}` requests an unscaled source. Scale value formats are `e8m0`, `e4m3`, and `e5m3`, with `fp8e4m3` accepted as an alias for `e4m3`; actual target support comes from the @@ -66,7 +66,8 @@ catalog. Scaled-WMMA IDs have the form `wmma.scaled..src0___b.src1___b.src2_.dst_`. They describe a semantic contract independently of LLVM intrinsic names and -packed carriers. Existing `wmma_scale*_f32_*` IDs and their dedicated builder +packed carriers. The `src2` and `dst` fields both use the shared C dtype; +they do not imply independent result-type selection. Existing `wmma_scale*_f32_*` IDs and their dedicated builder wrappers are retired; serialized IR using those IDs must be regenerated. Use `tile.mma` with a resolved catalog atom. Other MMA operation IDs retain their existing spelling. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py index 42e940009075..fdada1852122 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py @@ -13,10 +13,8 @@ LayoutMap, MemoryCapabilities, MmaCatalog, - MmaDst, MmaOp, MmaScaleOperand, - MmaSrc, ResourceLimits, arch_from_isa, base_arch_from_target_id, @@ -31,10 +29,8 @@ "LayoutMap", "MemoryCapabilities", "MmaCatalog", - "MmaDst", "MmaOp", "MmaScaleOperand", - "MmaSrc", "ResourceLimits", "arch_from_isa", "base_arch_from_target_id", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json index 7170e799ad49..1663e919166d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json @@ -1,6 +1,6 @@ { - "version": 2, - "_comment": "ROCKE-owned architecture SSOT. Hardware facts only (no pipeline/scheduler vocabulary). MFMA atom sets are hardware-verified on real hardware (see dsl_docs/architecture/multi_arch_data_layout.md). Each MMA row has three matrix srcs and one dst; a src may optionally carry a scale descriptor with dtype (scale value format: e8m0/e4m3/e5m3; fp8e4m3 aliases e4m3) and block_size (16 or 32 source elements along K per scale). op_id is the opaque backend handle (matches IRBuilder.mfma_* today).", + "version": 1, + "_comment": "ROCKE-owned architecture SSOT. Hardware facts only (no pipeline/scheduler vocabulary). MFMA atom sets are hardware-verified on real hardware (see dsl_docs/architecture/multi_arch_data_layout.md). Each MMA row: {family,a,b,c,m,n,k,op_id}; op_id is the opaque backend handle (matches IRBuilder.mfma_* today).", "arches": { "gfx90a": { "_comment": "CDNA2 (gfx90a). Wave64, 64 KB LDS. No fp8/bf8 native MFMA (those arrived with gfx940/gfx942). No async_lds. bf16 32x32x8 is hardware-verified on gfx90a series. Hardware-verified MMA set matches AMD ISA manual v3 (GFX9 MFMA section).", @@ -21,10 +21,10 @@ "sgprs": 102 }, "mma": [ - {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_f16"}, - {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_f16"}, - {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_bf16"}, - {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_bf16"} + {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_f16"}, + {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_f16"}, + {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_bf16"}, + {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_bf16"} ] }, "gfx942": { @@ -45,16 +45,16 @@ "sgprs": 102 }, "mma": [ - {"family": "mma", "srcs": [{"dtype": "fp32"}, {"dtype": "fp32"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 4, "op_id": "mfma_f32_16x16x4_f32"}, - {"family": "mma", "srcs": [{"dtype": "fp32"}, {"dtype": "fp32"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 2, "op_id": "mfma_f32_32x32x2_f32"}, - {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_f16"}, - {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_f16"}, - {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_bf16"}, - {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_bf16"}, - {"family": "mma", "srcs": [{"dtype": "fp8"}, {"dtype": "fp8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_fp8"}, - {"family": "mma", "srcs": [{"dtype": "fp8"}, {"dtype": "fp8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_fp8"}, - {"family": "mma", "srcs": [{"dtype": "bf8"}, {"dtype": "bf8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_bf8"}, - {"family": "mma", "srcs": [{"dtype": "bf8"}, {"dtype": "bf8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_bf8"} + {"family": "mma", "a": "fp32", "b": "fp32", "c": "fp32", "m": 16, "n": 16, "k": 4, "op_id": "mfma_f32_16x16x4_f32"}, + {"family": "mma", "a": "fp32", "b": "fp32", "c": "fp32", "m": 32, "n": 32, "k": 2, "op_id": "mfma_f32_32x32x2_f32"}, + {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_f16"}, + {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_f16"}, + {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_bf16"}, + {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_bf16"}, + {"family": "mma", "a": "fp8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_fp8"}, + {"family": "mma", "a": "fp8", "b": "fp8", "c": "fp32", "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_fp8"}, + {"family": "mma", "a": "bf8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_bf8"}, + {"family": "mma", "a": "bf8", "b": "bf8", "c": "fp32", "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_bf8"} ] }, "gfx950": { @@ -75,22 +75,22 @@ "sgprs": 102 }, "mma": [ - {"family": "mma", "srcs": [{"dtype": "fp32"}, {"dtype": "fp32"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 4, "op_id": "mfma_f32_16x16x4_f32"}, - {"family": "mma", "srcs": [{"dtype": "fp32"}, {"dtype": "fp32"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 2, "op_id": "mfma_f32_32x32x2_f32"}, - {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_f16"}, - {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_f16"}, - {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_f16"}, - {"family": "mma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_f16"}, - {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_bf16"}, - {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_bf16"}, - {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_bf16"}, - {"family": "mma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_bf16"}, - {"family": "mma", "srcs": [{"dtype": "fp8"}, {"dtype": "fp8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_fp8"}, - {"family": "mma", "srcs": [{"dtype": "fp8"}, {"dtype": "fp8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_fp8"}, - {"family": "mma", "srcs": [{"dtype": "bf8"}, {"dtype": "bf8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_bf8"}, - {"family": "mma", "srcs": [{"dtype": "bf8"}, {"dtype": "bf8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_bf8"}, - {"family": "mma", "srcs": [{"dtype": "fp4"}, {"dtype": "fp4"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 128, "op_id": "mfma_f32_16x16x128_fp4"}, - {"family": "mma", "srcs": [{"dtype": "fp6"}, {"dtype": "fp6"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 96, "op_id": "mfma_f32_16x16x96_fp6"} + {"family": "mma", "a": "fp32", "b": "fp32", "c": "fp32", "m": 16, "n": 16, "k": 4, "op_id": "mfma_f32_16x16x4_f32"}, + {"family": "mma", "a": "fp32", "b": "fp32", "c": "fp32", "m": 32, "n": 32, "k": 2, "op_id": "mfma_f32_32x32x2_f32"}, + {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_f16"}, + {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_f16"}, + {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_f16"}, + {"family": "mma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_f16"}, + {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "mfma_f32_16x16x16_bf16"}, + {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_bf16"}, + {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 32, "n": 32, "k": 8, "op_id": "mfma_f32_32x32x8_bf16"}, + {"family": "mma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_bf16"}, + {"family": "mma", "a": "fp8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_fp8"}, + {"family": "mma", "a": "fp8", "b": "fp8", "c": "fp32", "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_fp8"}, + {"family": "mma", "a": "bf8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "mfma_f32_16x16x32_bf8"}, + {"family": "mma", "a": "bf8", "b": "bf8", "c": "fp32", "m": 32, "n": 32, "k": 16, "op_id": "mfma_f32_32x32x16_bf8"}, + {"family": "mma", "a": "fp4", "b": "fp4", "c": "fp32", "m": 16, "n": 16, "k": 128, "op_id": "mfma_f32_16x16x128_fp4"}, + {"family": "mma", "a": "fp6", "b": "fp6", "c": "fp32", "m": 16, "n": 16, "k": 96, "op_id": "mfma_f32_16x16x96_fp6"} ] }, "gfx1151": { @@ -112,10 +112,10 @@ "sgprs": 106 }, "mma": [ - {"family": "wmma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_f16"}, - {"family": "wmma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_bf16"}, - {"family": "wmma", "srcs": [{"dtype": "iu8"}, {"dtype": "iu8"}, {"dtype": "i32"}], "dst": {"dtype": "i32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu8"}, - {"family": "wmma", "srcs": [{"dtype": "iu4"}, {"dtype": "iu4"}, {"dtype": "i32"}], "dst": {"dtype": "i32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu4"} + {"family": "wmma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_f16"}, + {"family": "wmma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_bf16"}, + {"family": "wmma", "a": "iu8", "b": "iu8", "c": "i32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu8"}, + {"family": "wmma", "a": "iu4", "b": "iu4", "c": "i32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu4"} ] }, "gfx1201": { @@ -137,12 +137,12 @@ "sgprs": 106 }, "mma": [ - {"family": "wmma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_gfx12_f32_16x16x16_f16"}, - {"family": "wmma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_gfx12_f32_16x16x16_bf16"} + {"family": "wmma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_gfx12_f32_16x16x16_f16"}, + {"family": "wmma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_gfx12_f32_16x16x16_bf16"} ] }, "gfx1250": { - "_comment": "gfx1250-class CDNA multi-chip device on the GFX12 programming model (wave32, WMMA, no MFMA). Primary fp16/bf16 WMMA atom is 16x16x32 (NOT gfx1201's 16x16x16). ABI empirically confirmed on ROCm 7.13 clang 23 (gfx1250 device target): __builtin_amdgcn_wmma_f32_16x16x32_f16 -> llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1,<16 x half>,i1,<16 x half>,i16,<8 x float>,i1,i1); bf16 uses v16bf16 (<16 x bfloat>) directly, not the gfx11 <16 x i16> bitcast. Datalayout/triple byte-identical to gfx950/gfx1201. The MMA entries below enumerate supported operand formats and shapes. Dtype-name recognition alone does not imply an available atom. Lane maps are a HYPOTHESIS verified by examples/gfx1250/wmma_probe.py. MEMORY: gfx1250 HAS async global->LDS DMA (its own GFX12 family: global_load_async_to_lds_b128 + s_wait_asynccnt -- NOT the gfx9 buffer_load_lds/global_load_lds, which do not select), so has_async_global_lds=true (GPU-validated via the DTLA path). It also HAS a hardware transpose-LDS read, ds_load_tr16_b128 (GPU-validated via compute_pv_dstr). The has_async_lds and has_ds_read_tr flags stay false on purpose: in this catalog they denote the gfx9/gfx950 ds_read_*_tr_* / async-LDS *ABI* used to admit the gfx950 MFMA tiled builder (attention_arch.py), which gfx1250 does NOT have -- its async/transpose use distinct GFX12 opcodes gated directly in the ISA backend, not via these MFMA-pipeline bits. has_tdm=false: TDM not verified on this part.", + "_comment": "gfx1250-class CDNA multi-chip device on the GFX12 programming model (wave32, WMMA, no MFMA). Primary fp16/bf16 WMMA atom is 16x16x32 (NOT gfx1201's 16x16x16). ABI empirically confirmed on ROCm 7.13 clang 23 (gfx1250 device target): __builtin_amdgcn_wmma_f32_16x16x32_f16 -> llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1,<16 x half>,i1,<16 x half>,i16,<8 x float>,i1,i1); bf16 uses v16bf16 (<16 x bfloat>) directly, not the gfx11 <16 x i16> bitcast. Datalayout/triple byte-identical to gfx950/gfx1201. This catalog ships validated fp16/bf16 and fp8/bf8 WMMA atoms plus the FP8xFP8 SCALE/SCALE16 prerequisite slice; FP4xFP4 SCALE/SCALE16 accepts prepacked E2M1 bytes; FP6/BF6 forms remain omitted. Lane maps are a HYPOTHESIS verified by examples/gfx1250/wmma_probe.py. MEMORY: gfx1250 HAS async global->LDS DMA (its own GFX12 family: global_load_async_to_lds_b128 + s_wait_asynccnt -- NOT the gfx9 buffer_load_lds/global_load_lds, which do not select), so has_async_global_lds=true (GPU-validated via the DTLA path). It also HAS a hardware transpose-LDS read, ds_load_tr16_b128 (GPU-validated via compute_pv_dstr). The has_async_lds and has_ds_read_tr flags stay false on purpose: in this catalog they denote the gfx9/gfx950 ds_read_*_tr_* / async-LDS *ABI* used to admit the gfx950 MFMA tiled builder (attention_arch.py), which gfx1250 does NOT have -- its async/transpose use distinct GFX12 opcodes gated directly in the ISA backend, not via these MFMA-pipeline bits. has_tdm=false: TDM not verified on this part.", "family": "cdna", "target_family": "gfx12_cdna", "matrix_path": "wmma", @@ -170,17 +170,17 @@ "sgprs": 106 }, "mma": [ - {"family": "wmma", "srcs": [{"dtype": "fp32"}, {"dtype": "fp32"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 4, "op_id": "wmma_gfx1250_f32_16x16x4_f32"}, - {"family": "wmma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "wmma_gfx1250_f32_16x16x32_f16"}, - {"family": "wmma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 32, "op_id": "wmma_gfx1250_f32_16x16x32_bf16"}, - {"family": "wmma", "srcs": [{"dtype": "fp8"}, {"dtype": "fp8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_fp8_fp8"}, - {"family": "wmma", "srcs": [{"dtype": "fp8"}, {"dtype": "bf8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_fp8_bf8"}, - {"family": "wmma", "srcs": [{"dtype": "bf8"}, {"dtype": "fp8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_fp8"}, - {"family": "wmma", "srcs": [{"dtype": "bf8"}, {"dtype": "bf8"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_bf8"}, - {"family": "wmma_scaled", "srcs": [{"dtype": "fp8", "scale": {"dtype": "e8m0", "block_size": 32}}, {"dtype": "fp8", "scale": {"dtype": "e8m0", "block_size": 32}}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32"}, - {"family": "wmma_scaled", "srcs": [{"dtype": "bf8", "scale": {"dtype": "e8m0", "block_size": 32}}, {"dtype": "bf8", "scale": {"dtype": "e8m0", "block_size": 32}}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32"}, - {"family": "wmma_scaled", "srcs": [{"dtype": "fp8", "scale": {"dtype": "e8m0", "block_size": 16}}, {"dtype": "fp8", "scale": {"dtype": "e8m0", "block_size": 16}}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32"}, - {"family": "wmma_scaled", "srcs": [{"dtype": "bf8", "scale": {"dtype": "e8m0", "block_size": 16}}, {"dtype": "bf8", "scale": {"dtype": "e8m0", "block_size": 16}}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32"} + {"family": "wmma", "a": "fp32", "b": "fp32", "c": "fp32", "m": 16, "n": 16, "k": 4, "op_id": "wmma_gfx1250_f32_16x16x4_f32"}, + {"family": "wmma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "wmma_gfx1250_f32_16x16x32_f16"}, + {"family": "wmma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 32, "op_id": "wmma_gfx1250_f32_16x16x32_bf16"}, + {"family": "wmma", "a": "fp8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_fp8_fp8"}, + {"family": "wmma", "a": "fp8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_fp8_bf8"}, + {"family": "wmma", "a": "bf8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_fp8"}, + {"family": "wmma", "a": "bf8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_bf8"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale": {"dtype": "e8m0", "block_size": 32}, "b_scale": {"dtype": "e8m0", "block_size": 32}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale": {"dtype": "e8m0", "block_size": 32}, "b_scale": {"dtype": "e8m0", "block_size": 32}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale": {"dtype": "e8m0", "block_size": 16}, "b_scale": {"dtype": "e8m0", "block_size": 16}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale": {"dtype": "e8m0", "block_size": 16}, "b_scale": {"dtype": "e8m0", "block_size": 16}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32"} ] }, "gfx11-generic": { @@ -202,10 +202,10 @@ "sgprs": 106 }, "mma": [ - {"family": "wmma", "srcs": [{"dtype": "fp16"}, {"dtype": "fp16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_f16"}, - {"family": "wmma", "srcs": [{"dtype": "bf16"}, {"dtype": "bf16"}, {"dtype": "fp32"}], "dst": {"dtype": "fp32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_bf16"}, - {"family": "wmma", "srcs": [{"dtype": "iu8"}, {"dtype": "iu8"}, {"dtype": "i32"}], "dst": {"dtype": "i32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu8"}, - {"family": "wmma", "srcs": [{"dtype": "iu4"}, {"dtype": "iu4"}, {"dtype": "i32"}], "dst": {"dtype": "i32"}, "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu4"} + {"family": "wmma", "a": "fp16", "b": "fp16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_f16"}, + {"family": "wmma", "a": "bf16", "b": "bf16", "c": "fp32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_f32_16x16x16_bf16"}, + {"family": "wmma", "a": "iu8", "b": "iu8", "c": "i32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu8"}, + {"family": "wmma", "a": "iu4", "b": "iu4", "c": "i32", "m": 16, "n": 16, "k": 16, "op_id": "wmma_i32_16x16x16_iu4"} ] } } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py index 044c5914d0d1..0ff4916ea99d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py @@ -32,12 +32,12 @@ # A callable that, given an :class:`~rocke.core.ir.IRBuilder`, a runtime lane # ``Value`` (0..wave_size-1) and a compile-time fragment slot index, emits the # index arithmetic for the two tile coordinates of that fragment element and -# returns them as a ``(Value, Value)`` pair. The coordinate meaning depends on -# the machine operand position (see :class:`LayoutMap.role`): +# returns them as a ``(Value, Value)`` pair. The coordinate *meaning* depends on +# the operand role (see :class:`LayoutMap.role`): # -# * src0 -> ``(row, k)`` in the atom's M x K input tile -# * src1 -> ``(k, col)`` in the atom's K x N input tile -# * src2/dst -> ``(row, col)`` in the atom's M x N tile +# * accumulator (C/D) -> ``(row, col)`` in the atom's M x N output tile +# * A operand -> ``(row, k)`` in the atom's M x K input tile +# * B operand -> ``(k, col)`` in the atom's K x N input tile # # The map is *pure* with respect to the IR: it only emits ``arith.*`` ops via the # builder, so the same map drives both fragment loads (compute the source @@ -60,9 +60,9 @@ class LayoutMap: Attributes ---------- role - ``"src0"``, ``"src1"``, ``"src2"``, or ``"dst"``. ``src0`` uses - ``(row, k)``, ``src1`` uses ``(k, col)``, and ``src2``/``dst`` use - ``(row, col)`` coordinates. + ``"acc"`` (accumulator C/D, coords are ``(row, col)``), ``"a"`` (A + operand, coords are ``(row, k)``) or ``"b"`` (B operand, coords are + ``(k, col)``). frag_len Number of fragment slots per lane for this role (the per-lane vector length: e.g. 4 for an MFMA 16x16x16 accumulator, 8 for the WMMA @@ -123,142 +123,88 @@ def __post_init__(self) -> None: ) -@dataclass(frozen=True) -class MmaDst: - """Metadata for the matrix ``dst`` operand.""" - - dtype: str - frag_len: int = 0 - layout: Optional[LayoutMap] = field(default=None, repr=False, compare=False) - - def __post_init__(self) -> None: - object.__setattr__(self, "dtype", normalize_dtype(self.dtype)) - - def require_layout(self, name: str, op: "MmaOp") -> LayoutMap: - if self.layout is None: - raise NotImplementedError( - f"no verified {name!r} layout map for MMA op_id {op.op_id!r} " - f"({op.m}x{op.n}x{op.k}); add one to " - f"_MMA_FRAGMENT_INFO before consuming it" - ) - return self.layout - - -@dataclass(frozen=True) -class MmaSrc: - """Metadata for one matrix ``src``, with an optional scale operand.""" - - dtype: str - frag_len: int = 0 - layout: Optional[LayoutMap] = field(default=None, repr=False, compare=False) - scale: Optional[MmaScaleOperand] = None - - def __post_init__(self) -> None: - object.__setattr__(self, "dtype", normalize_dtype(self.dtype)) - - def require_layout(self, name: str, op: "MmaOp") -> LayoutMap: - if self.layout is None: - raise NotImplementedError( - f"no verified {name!r} layout map for MMA op_id {op.op_id!r} " - f"({op.m}x{op.n}x{op.k}); add one to " - f"_MMA_FRAGMENT_INFO before consuming it" - ) - return self.layout - - @dataclass(frozen=True) class MmaOp: """A single supported matrix-multiply-accumulate atom on a target. ``op_id`` identifies a concrete operand contract. Scaled WMMA uses semantic - IDs including source and result dtypes and per-source scales. The backend - maps this identity to an intrinsic and format selectors. - - ``srcs`` contains ``src0``, ``src1``, and ``src2`` in machine order. The - first two are multiplicands, ``src2`` is the accumulator input, and ``dst`` - is the matrix result. - Optional scale operands belong to the source they scale; they are not extra - matrix fragments in ``srcs``. + IDs including matrix dtypes and per-input scales. The backend maps this + identity to an intrinsic and format selectors. + + The ``*_frag_len`` fields and the layout-map accessors describe the + *physical* register fragmentation of the atom: how many values of each + operand a single lane holds, and where (in the logical M/N/K tile) each of + those values lives. They are the cross-arch data-layout contract that lets + one kernel body drive both MFMA (wave64) and WMMA (wave32). They are derived + from ``(op_id, family, shape)`` at load time via :data:`_MMA_FRAGMENT_INFO`; + op_ids without verified layout maps still carry frag lengths but raise from + the accessor until a map is added. """ family: str # "mma" | "wmma" | "wmma_scaled" - srcs: Tuple[MmaSrc, MmaSrc, MmaSrc] - dst: MmaDst + a_dtype: str + b_dtype: str + c_dtype: str m: int n: int k: int op_id: str + a_frag_len: int = 0 + b_frag_len: int = 0 + c_frag_len: int = 0 wave_size: int = 64 + # Layout maps are kept off the public field list (repr=False, compare=False) + # so two MmaOps with the same shape still compare equal regardless of the + # closure identity of their maps. + _a_layout: Optional[LayoutMap] = field(default=None, repr=False, compare=False) + _b_layout: Optional[LayoutMap] = field(default=None, repr=False, compare=False) + _c_layout: Optional[LayoutMap] = field(default=None, repr=False, compare=False) - def __post_init__(self) -> None: - if len(self.srcs) != 3: - raise ValueError(f"MMA op {self.op_id!r} requires exactly 3 matrix sources") + a_scale: MmaScaleOperand | None = None + b_scale: MmaScaleOperand | None = None def semantic_id(self) -> str: - """Stable semantic identity, independent of backend packing and names.""" + """Logical identity; C describes both accumulator input and result.""" fields = [self.family.replace("_", "."), f"{self.m}x{self.n}x{self.k}"] - for i, src in enumerate(self.srcs): - field = f"src{i}_{normalize_dtype(src.dtype)}" - if src.scale is not None: - field += f"_{src.scale.dtype}_b{src.scale.block_size}" + for i, (dtype, scale) in enumerate( + ((self.a_dtype, self.a_scale), (self.b_dtype, self.b_scale)) + ): + field = f"src{i}_{normalize_dtype(dtype)}" + if scale is not None: + field += f"_{scale.dtype}_b{scale.block_size}" fields.append(field) - fields.append(f"dst_{normalize_dtype(self.dst.dtype)}") - return ".".join(fields) + acc = normalize_dtype(self.c_dtype) + return ".".join(fields + [f"src2_{acc}", f"dst_{acc}"]) @property def shape(self) -> Tuple[int, int, int]: return (self.m, self.n, self.k) - def src(self, index: int) -> MmaSrc: - if not 0 <= index < len(self.srcs): - raise IndexError(f"MMA src index {index} is outside [0, {len(self.srcs)})") - return self.srcs[index] - - def src_layout(self, index: int) -> LayoutMap: - return self.src(index).require_layout(f"src{index}", self) - - def dst_layout(self) -> LayoutMap: - return self.dst.require_layout("dst", self) - - # Compatibility projections for existing mathematical A/B/C callers. The - # historical C surface described the accumulator result, so C projects dst; - # src2 is available only through the canonical indexed interface. - @property - def a_dtype(self) -> str: - return self.srcs[0].dtype - - @property - def b_dtype(self) -> str: - return self.srcs[1].dtype - - @property - def c_dtype(self) -> str: - return self.dst.dtype - - @property - def a_frag_len(self) -> int: - return self.srcs[0].frag_len - - @property - def b_frag_len(self) -> int: - return self.srcs[1].frag_len - - @property - def c_frag_len(self) -> int: - return self.dst.frag_len - - # --- physical-layout compatibility accessors ------------------------- + # --- physical-layout accessors --------------------------------------- def a_layout(self) -> LayoutMap: """The A-operand ``(row, k)`` lane/slot -> coordinate map.""" - return self.src_layout(0) + return self._require_layout(self._a_layout, "a") def b_layout(self) -> LayoutMap: """The B-operand ``(k, col)`` lane/slot -> coordinate map.""" - return self.src_layout(1) + return self._require_layout(self._b_layout, "b") def c_layout(self) -> LayoutMap: - """Legacy result ``(row, col)`` lane/slot -> coordinate map.""" - return self.dst_layout() + """The accumulator (C/D) ``(row, col)`` lane/slot -> coordinate map.""" + return self._require_layout(self._c_layout, "c") + + # Convenience aliases for the accumulator (the most-used map). + def acc_layout(self) -> LayoutMap: + return self.c_layout() + + def _require_layout(self, layout: Optional[LayoutMap], role: str) -> LayoutMap: + if layout is None: + raise NotImplementedError( + f"no verified {role!r} layout map for MMA op_id {self.op_id!r} " + f"({self.m}x{self.n}x{self.k}); add one to " + f"_MMA_FRAGMENT_INFO before consuming it" + ) + return layout # --------------------------------------------------------------------------- @@ -276,8 +222,8 @@ def c_layout(self) -> LayoutMap: # single LayoutMap covers all slots of the fragment. -def _mfma_row_col_16x16(builder, lane, slot): - """MFMA 16x16 src2/dst row-column map. +def _mfma_acc_16x16(builder, lane, slot): + """MFMA 16x16 accumulator: slot ``i`` -> ``(m_blk*4 + i, lane % 16)``. Mirrors ``MfmaAtom.lane_to_output`` for the (16, 16) case with ``c_per_lane == 4`` (``row = m_blk * 4 + i``, ``col = lane % 16``). @@ -289,8 +235,8 @@ def _mfma_row_col_16x16(builder, lane, slot): return row, n_in_atom -def _mfma_row_col_32x32(builder, lane, slot): - """MFMA 32x32 src2/dst row-column map. +def _mfma_acc_32x32(builder, lane, slot): + """MFMA 32x32 accumulator (``c_per_lane == 16``). Mirrors ``MfmaAtom.lane_to_output`` for the (32, 32) case: ``row = (i // 4) * 8 + (lane // 32) * 4 + (i % 4)``, ``col = lane % 32``. @@ -467,8 +413,8 @@ def _mfma_b_32x32x16(builder, lane, slot): return k, n_in_atom -def _wmma_row_col_16x16(builder, lane, slot): - """WMMA 16x16x16 src2/dst row-column map. +def _wmma_acc_16x16(builder, lane, slot): + """WMMA 16x16x16 accumulator (wave32, hardware-verified gfx1151). ``<8 x float>`` per lane; slot ``i`` -> ``(row 2*i + lane // 16, col lane % 16)``. Returns ``(row, col)``. @@ -503,7 +449,7 @@ def _wmma_b_16x16(builder, lane, slot): # fragment: slot j (0..3) is one i32 holding the four int8 K-values # [4j, 4j+1, 4j+2, 4j+3]. The lane map therefore returns the K *base* of the # slot (4*j); the kernel/staging code packs the four consecutive K bytes from -# there. The accumulator is identical to f16 WMMA (_wmma_row_col_16x16), only the +# there. The accumulator is identical to f16 WMMA (_wmma_acc_16x16), only the # element type is i32 instead of f32. def _wmma_a_16x16_iu8(builder, lane, slot): """iu8 WMMA A operand (wave32): lane ``l`` holds row ``l % 16``; A fragment @@ -547,8 +493,8 @@ def _wmma_b_16x16_iu4(builder, lane, slot): # accumulator is column-distributed (CDNA/MFMA-style): lanes index columns, # registers index rows. These maps are the *hypothesis* verified empirically by # examples/gfx1201/wmma_probe.py before matmul_nbits is trusted on gfx1201. -def _wmma_gfx12_row_col_16x16(builder, lane, slot): - """RDNA4 WMMA 16x16x16 src2/dst row-column map: +def _wmma_gfx12_acc_16x16(builder, lane, slot): + """RDNA4 WMMA 16x16x16 accumulator (wave32): ``<8 x float>`` per lane; slot ``i`` -> ``(row (lane // 16) * 8 + i, col lane % 16)``. Returns ``(row, col)``.""" c16 = builder.const_i32(16) @@ -632,53 +578,18 @@ def _wmma_gfx1250_b_16x16x32(builder, lane, slot): return k, col -@dataclass(frozen=True) -class _FragOperand: - """Physical fragment metadata for one machine operand position.""" - - frag_len: int - fn: _LaneCoordFn = None - - @dataclass(frozen=True) class _FragInfo: - """Per-op_id physical metadata for ``srcs[0..2]`` and ``dst``.""" + """Per-op_id fragment metadata: per-lane vector lengths, wave size, and the + (optional) lane/slot coordinate functions for A / B / accumulator.""" - srcs: Tuple[_FragOperand, _FragOperand, _FragOperand] - dst: _FragOperand + a_frag_len: int + b_frag_len: int + c_frag_len: int wave_size: int - - def __post_init__(self) -> None: - if len(self.srcs) != 3: - raise ValueError("MMA fragment metadata requires exactly 3 sources") - - -def _shared_src2_dst_frag_info( - src0_frag_len: int, - src1_frag_len: int, - row_col_frag_len: int, - wave_size: int, - src0_fn: _LaneCoordFn = None, - src1_fn: _LaneCoordFn = None, - row_col_fn: _LaneCoordFn = None, -) -> _FragInfo: - """Build metadata whose ``src2`` and ``dst`` layouts currently match. - - ``src2`` and ``dst`` get separate descriptors even when the current ISA - gives them the same fragment width and row/column coordinate function. - This is a catalog-construction convenience, not a constraint on - :class:`_FragInfo`; distinct contracts use that type directly. - """ - - return _FragInfo( - srcs=( - _FragOperand(src0_frag_len, src0_fn), - _FragOperand(src1_frag_len, src1_fn), - _FragOperand(row_col_frag_len, row_col_fn), - ), - dst=_FragOperand(row_col_frag_len, row_col_fn), - wave_size=wave_size, - ) + a_fn: _LaneCoordFn = None + b_fn: _LaneCoordFn = None + c_fn: _LaneCoordFn = None # op_id -> physical fragment metadata. Frag lengths are populated for every @@ -689,57 +600,57 @@ def _shared_src2_dst_frag_info( # --- MFMA fp32 (wave64) ----------------------------------------------- # A/B are scalar float per lane (a_frag_len=b_frag_len=1); accumulator # shares the standard 16x16 / 32x32 layout (c_frag_len=4 / 16). - "mfma_f32_16x16x4_f32": _shared_src2_dst_frag_info( - 1, 1, 4, 64, _mfma_a_16x16x4_f32, _mfma_b_16x16x4_f32, _mfma_row_col_16x16 + "mfma_f32_16x16x4_f32": _FragInfo( + 1, 1, 4, 64, _mfma_a_16x16x4_f32, _mfma_b_16x16x4_f32, _mfma_acc_16x16 ), - "mfma_f32_32x32x2_f32": _shared_src2_dst_frag_info( - 1, 1, 16, 64, _mfma_a_32x32x2_f32, _mfma_b_32x32x2_f32, _mfma_row_col_32x32 + "mfma_f32_32x32x2_f32": _FragInfo( + 1, 1, 16, 64, _mfma_a_32x32x2_f32, _mfma_b_32x32x2_f32, _mfma_acc_32x32 ), # --- MFMA f16 (wave64) ------------------------------------------------ - "mfma_f32_16x16x16_f16": _shared_src2_dst_frag_info( - 4, 4, 4, 64, _mfma_a_16x16, _mfma_b_16x16, _mfma_row_col_16x16 + "mfma_f32_16x16x16_f16": _FragInfo( + 4, 4, 4, 64, _mfma_a_16x16, _mfma_b_16x16, _mfma_acc_16x16 ), - "mfma_f32_16x16x32_f16": _shared_src2_dst_frag_info( - 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_row_col_16x16 + "mfma_f32_16x16x32_f16": _FragInfo( + 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_acc_16x16 ), - "mfma_f32_32x32x8_f16": _shared_src2_dst_frag_info( - 4, 4, 16, 64, _mfma_a_32x32x8, _mfma_b_32x32x8, _mfma_row_col_32x32 + "mfma_f32_32x32x8_f16": _FragInfo( + 4, 4, 16, 64, _mfma_a_32x32x8, _mfma_b_32x32x8, _mfma_acc_32x32 ), - "mfma_f32_32x32x16_f16": _shared_src2_dst_frag_info( - 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_row_col_32x32 + "mfma_f32_32x32x16_f16": _FragInfo( + 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_acc_32x32 ), - "mfma_f32_4x4x4_f16": _shared_src2_dst_frag_info(4, 4, 4, 64), + "mfma_f32_4x4x4_f16": _FragInfo(4, 4, 4, 64), # --- MFMA bf16 (wave64) ---------------------------------------------- - "mfma_f32_16x16x16_bf16": _shared_src2_dst_frag_info( - 4, 4, 4, 64, _mfma_a_16x16, _mfma_b_16x16, _mfma_row_col_16x16 + "mfma_f32_16x16x16_bf16": _FragInfo( + 4, 4, 4, 64, _mfma_a_16x16, _mfma_b_16x16, _mfma_acc_16x16 ), - "mfma_f32_16x16x32_bf16": _shared_src2_dst_frag_info( - 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_row_col_16x16 + "mfma_f32_16x16x32_bf16": _FragInfo( + 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_acc_16x16 ), - "mfma_f32_32x32x8_bf16": _shared_src2_dst_frag_info( - 4, 4, 16, 64, _mfma_a_32x32x8, _mfma_b_32x32x8, _mfma_row_col_32x32 + "mfma_f32_32x32x8_bf16": _FragInfo( + 4, 4, 16, 64, _mfma_a_32x32x8, _mfma_b_32x32x8, _mfma_acc_32x32 ), - "mfma_f32_32x32x16_bf16": _shared_src2_dst_frag_info( - 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_row_col_32x32 + "mfma_f32_32x32x16_bf16": _FragInfo( + 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_acc_32x32 ), # --- MFMA fp8 / bf8 (wave64) ----------------------------------------- # fp8/bf8 share the f16 operand lane layout (a_per_lane=8, same K-packing); # only the element type / intrinsic mangling differ. - "mfma_f32_16x16x32_fp8": _shared_src2_dst_frag_info( - 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_row_col_16x16 + "mfma_f32_16x16x32_fp8": _FragInfo( + 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_acc_16x16 ), - "mfma_f32_16x16x32_bf8": _shared_src2_dst_frag_info( - 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_row_col_16x16 + "mfma_f32_16x16x32_bf8": _FragInfo( + 8, 8, 4, 64, _mfma_a_16x16x32, _mfma_b_16x16x32, _mfma_acc_16x16 ), - "mfma_f32_32x32x16_fp8": _shared_src2_dst_frag_info( - 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_row_col_32x32 + "mfma_f32_32x32x16_fp8": _FragInfo( + 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_acc_32x32 ), - "mfma_f32_32x32x16_bf8": _shared_src2_dst_frag_info( - 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_row_col_32x32 + "mfma_f32_32x32x16_bf8": _FragInfo( + 8, 8, 16, 64, _mfma_a_32x32x16, _mfma_b_32x32x16, _mfma_acc_32x32 ), # --- MFMA MX (wave64), frag lengths only ----------------------------- - "mfma_f32_16x16x128_fp4": _shared_src2_dst_frag_info(16, 16, 4, 64), - "mfma_f32_16x16x96_fp6": _shared_src2_dst_frag_info(12, 12, 4, 64), + "mfma_f32_16x16x128_fp4": _FragInfo(16, 16, 4, 64), + "mfma_f32_16x16x96_fp6": _FragInfo(12, 12, 4, 64), # Unscaled fp8 K=128 hero atom (lowers through the f8f6f4 scale-MFMA # intrinsic; there is no dense plain fp8 K=128). A/B are 32 fp8 bytes per # lane (<8 x i32> at the intrinsic boundary), accumulator <4 x float> -- @@ -747,37 +658,37 @@ def _shared_src2_dst_frag_info( # op_id resolves to correct fragment lengths (this table is the SSOT that # ir.IRBuilder.mma consults) even before it is added to the JSON catalog, # instead of hitting the zero-length _frag_info fallback. - "mfma_f32_16x16x128_fp8": _shared_src2_dst_frag_info(32, 32, 4, 64), - "mfma_scale_f32_16x16x128_f8f6f4": _shared_src2_dst_frag_info(32, 32, 4, 64), + "mfma_f32_16x16x128_fp8": _FragInfo(32, 32, 4, 64), + "mfma_scale_f32_16x16x128_f8f6f4": _FragInfo(32, 32, 4, 64), # --- WMMA f16 / bf16 (wave32, RDNA) ---------------------------------- - "wmma_f32_16x16x16_f16": _shared_src2_dst_frag_info( - 16, 16, 8, 32, _wmma_a_16x16, _wmma_b_16x16, _wmma_row_col_16x16 + "wmma_f32_16x16x16_f16": _FragInfo( + 16, 16, 8, 32, _wmma_a_16x16, _wmma_b_16x16, _wmma_acc_16x16 ), # bf16 shares the f16 fragment layout (same 16x16x16 lane math; only the # element type / intrinsic mangling differ — operands lower as <16 x i16>). - "wmma_f32_16x16x16_bf16": _shared_src2_dst_frag_info( - 16, 16, 8, 32, _wmma_a_16x16, _wmma_b_16x16, _wmma_row_col_16x16 + "wmma_f32_16x16x16_bf16": _FragInfo( + 16, 16, 8, 32, _wmma_a_16x16, _wmma_b_16x16, _wmma_acc_16x16 ), # --- WMMA iu8 (wave32, RDNA3/3.5) ------------------------------------------ # A/B fragments are <4 x i32> (16 int8 packed 4-per-i32); accumulator is # <8 x i32> with the same lane math as the f16 WMMA accumulator. - "wmma_i32_16x16x16_iu8": _shared_src2_dst_frag_info( - 4, 4, 8, 32, _wmma_a_16x16_iu8, _wmma_b_16x16_iu8, _wmma_row_col_16x16 + "wmma_i32_16x16x16_iu8": _FragInfo( + 4, 4, 8, 32, _wmma_a_16x16_iu8, _wmma_b_16x16_iu8, _wmma_acc_16x16 ), # --- WMMA iu4 (wave32, RDNA3/3.5) ------------------------------------------ # A/B fragments are <2 x i32> (16 int4 packed 8-per-i32); accumulator is # <8 x i32> with the same lane math as the f16 WMMA accumulator. - "wmma_i32_16x16x16_iu4": _shared_src2_dst_frag_info( - 2, 2, 8, 32, _wmma_a_16x16_iu4, _wmma_b_16x16_iu4, _wmma_row_col_16x16 + "wmma_i32_16x16x16_iu4": _FragInfo( + 2, 2, 8, 32, _wmma_a_16x16_iu4, _wmma_b_16x16_iu4, _wmma_acc_16x16 ), # --- WMMA f16 / bf16 (wave32, RDNA4 / gfx12) ------------------------------- # No cross-half duplication: A/B are <8 x half> per lane; column-distributed # accumulator. Lane maps verified by examples/gfx1201/wmma_probe.py. - "wmma_gfx12_f32_16x16x16_f16": _shared_src2_dst_frag_info( - 8, 8, 8, 32, _wmma_gfx12_a_16x16, _wmma_gfx12_b_16x16, _wmma_gfx12_row_col_16x16 + "wmma_gfx12_f32_16x16x16_f16": _FragInfo( + 8, 8, 8, 32, _wmma_gfx12_a_16x16, _wmma_gfx12_b_16x16, _wmma_gfx12_acc_16x16 ), - "wmma_gfx12_f32_16x16x16_bf16": _shared_src2_dst_frag_info( - 8, 8, 8, 32, _wmma_gfx12_a_16x16, _wmma_gfx12_b_16x16, _wmma_gfx12_row_col_16x16 + "wmma_gfx12_f32_16x16x16_bf16": _FragInfo( + 8, 8, 8, 32, _wmma_gfx12_a_16x16, _wmma_gfx12_b_16x16, _wmma_gfx12_acc_16x16 ), # --- WMMA fp32 (wave32, gfx1250, CDNA) -------------------------------- # K=4 atom: A/B are <2 x fp32> per lane (a_frag_len=b_frag_len=2). @@ -785,116 +696,116 @@ def _shared_src2_dst_frag_info( # Accumulator is the gfx12 column-distributed <8 x float> (c_frag_len=8). # Intrinsic: __builtin_amdgcn_wmma_f32_16x16x4_f32 (verified in CK Tile # wmma_gfx12.hpp lines 355-373; amdgcn_mma_base kABKPerLane=2, kCMPerLane=8). - "wmma_gfx1250_f32_16x16x4_f32": _shared_src2_dst_frag_info( + "wmma_gfx1250_f32_16x16x4_f32": _FragInfo( 2, 2, 8, 32, _wmma_gfx1250_a_16x16x4_f32, _wmma_gfx1250_b_16x16x4_f32, - _wmma_gfx12_row_col_16x16, + _wmma_gfx12_acc_16x16, ), # --- WMMA f16 / bf16 (wave32, gfx1250, CDNA) ---------------------- # K=32 atom: A/B are <16 x half> per lane (K split across lane-halves, 16 # each); accumulator is the same 16x16 column-distributed <8 x float> as # gfx12. Lane maps verified by examples/gfx1250/wmma_probe.py. - "wmma_gfx1250_f32_16x16x32_f16": _shared_src2_dst_frag_info( + "wmma_gfx1250_f32_16x16x32_f16": _FragInfo( 16, 16, 8, 32, _wmma_gfx1250_a_16x16x32, _wmma_gfx1250_b_16x16x32, - _wmma_gfx12_row_col_16x16, + _wmma_gfx12_acc_16x16, ), # gfx1250 FP8/BF8 K=64 WMMA. A/B carry 32 low-bit bytes per lane presented # as <8 x i32>; accumulator is the same 16x16 column-distributed <8 x float> # as the f16/bf16 K=32 atom. The block-scaled GEMM kernel computes operand # offsets directly (no LayoutMap), so only frag lengths + wave size are # registered here; the accumulator map is shared with the f16 path. - "wmma_gfx1250_f32_16x16x64_fp8_fp8": _shared_src2_dst_frag_info( + "wmma_gfx1250_f32_16x16x64_fp8_fp8": _FragInfo( 8, 8, 8, 32, None, None, - _wmma_gfx12_row_col_16x16, + _wmma_gfx12_acc_16x16, ), - "wmma_gfx1250_f32_16x16x64_fp8_bf8": _shared_src2_dst_frag_info( + "wmma_gfx1250_f32_16x16x64_fp8_bf8": _FragInfo( 8, 8, 8, 32, None, None, - _wmma_gfx12_row_col_16x16, + _wmma_gfx12_acc_16x16, ), - "wmma_gfx1250_f32_16x16x64_bf8_fp8": _shared_src2_dst_frag_info( + "wmma_gfx1250_f32_16x16x64_bf8_fp8": _FragInfo( 8, 8, 8, 32, None, None, - _wmma_gfx12_row_col_16x16, + _wmma_gfx12_acc_16x16, ), - "wmma_gfx1250_f32_16x16x64_bf8_bf8": _shared_src2_dst_frag_info( + "wmma_gfx1250_f32_16x16x64_bf8_bf8": _FragInfo( 8, 8, 8, 32, None, None, - _wmma_gfx12_row_col_16x16, + _wmma_gfx12_acc_16x16, ), # Native gfx1250 scaled WMMA. FP8/BF8 use 64 bytes per lane as <16 x i32>. # SCALE packs four K=32 E8M0 bytes in i32 and SCALE16 packs eight K=16 # bytes in i64. Both share the gfx12 column-distributed accumulator. - "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32": _shared_src2_dst_frag_info( + "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32": _FragInfo( 16, 16, 8, 32, None, None, - _wmma_gfx12_row_col_16x16, + _wmma_gfx12_acc_16x16, ), - "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32": _shared_src2_dst_frag_info( + "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32": _FragInfo( 16, 16, 8, 32, None, None, - _wmma_gfx12_row_col_16x16, + _wmma_gfx12_acc_16x16, ), - "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32": _shared_src2_dst_frag_info( + "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32": _FragInfo( 16, 16, 8, 32, None, None, - _wmma_gfx12_row_col_16x16, + _wmma_gfx12_acc_16x16, ), - "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32": _shared_src2_dst_frag_info( + "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32": _FragInfo( 16, 16, 8, 32, None, None, - _wmma_gfx12_row_col_16x16, + _wmma_gfx12_acc_16x16, ), - "wmma_gfx1250_f32_16x16x32_bf16": _shared_src2_dst_frag_info( + "wmma_gfx1250_f32_16x16x32_bf16": _FragInfo( 16, 16, 8, 32, _wmma_gfx1250_a_16x16x32, _wmma_gfx1250_b_16x16x32, - _wmma_gfx12_row_col_16x16, + _wmma_gfx12_acc_16x16, ), } @@ -904,7 +815,7 @@ def _frag_info(op_id: str) -> _FragInfo: if info is None: # Unknown atoms still load (e.g. a future JSON-only op_id); they carry # zero frag lengths and no maps until registered here. - return _shared_src2_dst_frag_info(0, 0, 0, 64) + return _FragInfo(0, 0, 0, 64) return info @@ -926,7 +837,11 @@ class ResourceLimits: class MmaCatalog: - """Query concrete operand contracts; omitted scales are a wildcard.""" + """The arch-selected MMA atoms, with optional exact A/B scale filtering. + + ``scales=None`` leaves scales unconstrained; ``(None, None)`` selects + unscaled inputs. Exact selection and largest-K ties must be unambiguous. + """ def __init__(self, ops: List[MmaOp]) -> None: self._ops = tuple(ops) @@ -939,39 +854,27 @@ def enumerate( self, *, family: str = "mma", - src_dtypes: Optional[Tuple[str, str, str]] = None, - dst_dtype: Optional[str] = None, - src_scales: Optional[Tuple[Optional[MmaScaleOperand], ...]] = None, - a_dtype: Optional[str] = None, - b_dtype: Optional[str] = None, - c_dtype: Optional[str] = None, + a_dtype: str, + b_dtype: str, + c_dtype: str, + scales: tuple[MmaScaleOperand | None, MmaScaleOperand | None] | None = None, m: Optional[int] = None, n: Optional[int] = None, ) -> List[MmaOp]: - if src_dtypes is None: - if a_dtype is None or b_dtype is None or c_dtype is None: - raise TypeError("enumerate requires src_dtypes or a/b/c_dtype") - src_dtypes = (a_dtype, b_dtype, c_dtype) - if len(src_dtypes) != 3: - raise ValueError("src_dtypes must contain exactly 3 entries") - if src_scales is not None and len(src_scales) != 3: - raise ValueError("src_scales must contain exactly 3 entries") - src_keys = tuple(normalize_dtype(dtype) for dtype in src_dtypes) - # Preserve the historical three-dtype query by defaulting dst to src2. - # Indexed callers can state a distinct result. - dst_key = normalize_dtype(dst_dtype or src_dtypes[2]) + if scales is not None and len(scales) != 2: + raise ValueError("scales must contain exactly 2 entries") + a, b, c = ( + normalize_dtype(a_dtype), + normalize_dtype(b_dtype), + normalize_dtype(c_dtype), + ) out = [] for op in self._ops: if op.family != family: continue - if ( - tuple(src.dtype for src in op.srcs) != src_keys - or op.dst.dtype != dst_key - ): + if (op.a_dtype, op.b_dtype, op.c_dtype) != (a, b, c): continue - if src_scales is not None and tuple(src.scale for src in op.srcs) != tuple( - src_scales - ): + if scales is not None and (op.a_scale, op.b_scale) != tuple(scales): continue if m is not None and op.m != m: continue @@ -984,12 +887,10 @@ def has_shape( self, *, family: str = "mma", - src_dtypes: Optional[Tuple[str, str, str]] = None, - a_dtype: Optional[str] = None, - b_dtype: Optional[str] = None, - c_dtype: Optional[str] = None, - dst_dtype: Optional[str] = None, - src_scales: Optional[Tuple[Optional[MmaScaleOperand], ...]] = None, + a_dtype: str, + b_dtype: str, + c_dtype: str, + scales: tuple[MmaScaleOperand | None, MmaScaleOperand | None] | None = None, m: int, n: int, k: int, @@ -998,12 +899,10 @@ def has_shape( op.shape == (m, n, k) for op in self.enumerate( family=family, - src_dtypes=src_dtypes, - src_scales=src_scales, a_dtype=a_dtype, b_dtype=b_dtype, c_dtype=c_dtype, - dst_dtype=dst_dtype, + scales=scales, m=m, n=n, ) @@ -1013,12 +912,10 @@ def select_largest_k( self, *, family: str = "mma", - src_dtypes: Optional[Tuple[str, str, str]] = None, - a_dtype: Optional[str] = None, - b_dtype: Optional[str] = None, - c_dtype: Optional[str] = None, - dst_dtype: Optional[str] = None, - src_scales: Optional[Tuple[Optional[MmaScaleOperand], ...]] = None, + a_dtype: str, + b_dtype: str, + c_dtype: str, + scales: tuple[MmaScaleOperand | None, MmaScaleOperand | None] | None = None, m: int, n: int, k_max: Optional[int] = None, @@ -1027,12 +924,10 @@ def select_largest_k( op for op in self.enumerate( family=family, - src_dtypes=src_dtypes, - src_scales=src_scales, a_dtype=a_dtype, b_dtype=b_dtype, c_dtype=c_dtype, - dst_dtype=dst_dtype, + scales=scales, m=m, n=n, ) @@ -1044,7 +939,7 @@ def select_largest_k( return self._unique([op for op in cands if op.k == largest_k]) @staticmethod - def _unique(ops: List[MmaOp]) -> Optional[MmaOp]: + def _unique(ops: list[MmaOp]) -> MmaOp | None: if len(ops) > 1: raise ValueError("ambiguous MMA query; specify the full operand contract") return ops[0] if ops else None @@ -1060,24 +955,20 @@ def op_for_shape( self, *, family: str = "mma", - src_dtypes: Optional[Tuple[str, str, str]] = None, - a_dtype: Optional[str] = None, - b_dtype: Optional[str] = None, - c_dtype: Optional[str] = None, - dst_dtype: Optional[str] = None, - src_scales: Optional[Tuple[Optional[MmaScaleOperand], ...]] = None, + a_dtype: str, + b_dtype: str, + c_dtype: str, + scales: tuple[MmaScaleOperand | None, MmaScaleOperand | None] | None = None, m: int, n: int, k: int, ) -> Optional[MmaOp]: candidates = self.enumerate( family=family, - src_dtypes=src_dtypes, - src_scales=src_scales, a_dtype=a_dtype, b_dtype=b_dtype, c_dtype=c_dtype, - dst_dtype=dst_dtype, + scales=scales, m=m, n=n, ) @@ -1117,27 +1008,10 @@ def fits_lds(self, bytes_in_use: int) -> bool: return bytes_in_use <= self.lds_capacity_bytes def supports_dtype_combo( - self, - a: str, - b: str, - c: str, - dst: Optional[str] = None, - *, - family: str = "mma", - src_scales: Optional[Tuple[Optional[MmaScaleOperand], ...]] = None, + self, a: str, b: str, c: str, *, family: str = "mma" ) -> bool: return ( - len( - self.mma.enumerate( - family=family, - a_dtype=a, - b_dtype=b, - c_dtype=c, - dst_dtype=dst, - src_scales=src_scales, - ) - ) - > 0 + len(self.mma.enumerate(family=family, a_dtype=a, b_dtype=b, c_dtype=c)) > 0 ) def max_vector_load_dwords(self, dtype: str) -> int: @@ -1168,21 +1042,21 @@ def _load_specs() -> Dict[str, dict]: @lru_cache(maxsize=1) -def _op_id_dst_dtype() -> Dict[str, str]: - """``op_id -> normalized dst dtype``, aggregated across every arch +def _op_id_c_dtype() -> Dict[str, str]: + """``op_id -> normalized accumulator dtype``, aggregated across every arch row in the JSON catalog. - An ``op_id`` names a specific atom, so its ``dst`` dtype is invariant + An ``op_id`` names a specific atom, so its accumulator dtype is invariant across the arches that list it. This lets a caller that only has a bare ``op_id`` string (e.g. :meth:`rocke.core.ir.IRBuilder.mma`) recover the - ``dst`` dtype from the SSOT without holding an :class:`ArchTarget`. + accumulator dtype from the SSOT without holding an :class:`ArchTarget`. Op_ids that are frag-registered but not yet in the JSON catalog are absent - (callers should treat a miss as the default f32 ``dst``). + (callers should treat a miss as the default f32 accumulator). The **first** catalog hit for an op_id wins, matching the C implementation - (``rocke_arch_mma_op_id_dst_dtype`` in ``query.cpp``, which returns the first + (``rocke_arch_mma_op_id_c_dtype`` in ``query.cpp``, which returns the first registry hit). If a later arch lists the same op_id with a *different* - ``dst`` dtype we raise instead of silently overwriting: that would be + accumulator dtype we raise instead of silently overwriting: that would be SSOT drift (the invariant above is broken) and must be fixed in the catalog, not masked. Raising here also keeps the Python and C engines deterministic and byte-identical rather than diverging on catalog ordering. @@ -1191,26 +1065,21 @@ def _op_id_dst_dtype() -> Dict[str, str]: for row in _load_specs().values(): for o in row["mma"]: op_id = o["op_id"] - dst_row = o["dst"] if "dst" in o else {"dtype": o["c"]} - c = normalize_dtype(dst_row["dtype"]) + c = normalize_dtype(o["c"]) prev = out.get(op_id) if prev is None: out[op_id] = c # first hit wins elif prev != c: raise ValueError( f"arch SSOT drift in {_DATA_FILE.name}: op_id {op_id!r} has " - f"inconsistent dst dtype across arches " + f"inconsistent accumulator dtype across arches " f"({prev!r} vs {c!r}). An op_id names a specific atom, so its " - f"dst dtype must be invariant across the arches that " + f"accumulator dtype must be invariant across the arches that " f"list it; fix the catalog so every row agrees." ) return out -# Compatibility for callers that used C to mean the instruction result. -_op_id_c_dtype = _op_id_dst_dtype - - def _build_mma_op(o: dict) -> MmaOp: """Construct an :class:`MmaOp` from one catalog JSON row, attaching the physical fragment lengths and layout maps registered for its op_id.""" @@ -1222,55 +1091,25 @@ def _mk(role: str, frag_len: int, fn: _LaneCoordFn) -> Optional[LayoutMap]: return None return LayoutMap(role=role, frag_len=frag_len, wave_size=info.wave_size, fn=fn) - src_rows = o.get("srcs") - if src_rows is None: - src_rows = ({"dtype": o["a"]}, {"dtype": o["b"]}, {"dtype": o["c"]}) - if len(src_rows) != 3: - raise ValueError( - f"MMA catalog op {op_id!r} must define exactly 3 matrix sources" - ) - - def _source(row: dict, role: str, frag_len: int, fn: _LaneCoordFn) -> MmaSrc: - scale_row = row.get("scale") - scale = ( - MmaScaleOperand( - dtype=normalize_dtype(scale_row["dtype"]), - block_size=scale_row["block_size"], - ) - if scale_row is not None - else None - ) - return MmaSrc( - dtype=normalize_dtype(row["dtype"]), - frag_len=frag_len, - layout=_mk(role, frag_len, fn), - scale=scale, - ) - - dst_row = o["dst"] if "dst" in o else {"dtype": o["c"]} - op = MmaOp( + return MmaOp( family=o["family"], - srcs=( - _source(src_rows[0], "src0", info.srcs[0].frag_len, info.srcs[0].fn), - _source(src_rows[1], "src1", info.srcs[1].frag_len, info.srcs[1].fn), - _source(src_rows[2], "src2", info.srcs[2].frag_len, info.srcs[2].fn), - ), - dst=MmaDst( - dtype=normalize_dtype(dst_row["dtype"]), - frag_len=info.dst.frag_len, - layout=_mk("dst", info.dst.frag_len, info.dst.fn), - ), + a_dtype=normalize_dtype(o["a"]), + b_dtype=normalize_dtype(o["b"]), + c_dtype=normalize_dtype(o["c"]), m=o["m"], n=o["n"], k=o["k"], op_id=op_id, + a_frag_len=info.a_frag_len, + b_frag_len=info.b_frag_len, + c_frag_len=info.c_frag_len, wave_size=info.wave_size, + _a_layout=_mk("a", info.a_frag_len, info.a_fn), + _b_layout=_mk("b", info.b_frag_len, info.b_fn), + _c_layout=_mk("c", info.c_frag_len, info.c_fn), + a_scale=MmaScaleOperand(**o["a_scale"]) if o.get("a_scale") else None, + b_scale=MmaScaleOperand(**o["b_scale"]) if o.get("b_scale") else None, ) - if op.family == "wmma_scaled" and op.op_id != op.semantic_id(): - raise ValueError( - f"scaled MMA catalog ID does not match its contract: {op.op_id}" - ) - return op @lru_cache(maxsize=None) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py index 78f86df448ed..c4aeb359eafd 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py @@ -52,14 +52,16 @@ def scale16(self) -> bool: @property def matrix_llvm_types(self) -> tuple[str, str]: - return tuple(f"<{src.frag_len} x i32>" for src in self.atom.srcs[:2]) + return tuple( + f"<{n} x i32>" for n in (self.atom.a_frag_len, self.atom.b_frag_len) + ) @property def intrinsic_suffix(self) -> str: atom = self.atom return ( - f"f32.{atom.m}x{atom.n}x{atom.k}.f8f6f4.v{atom.dst.frag_len}f32." - f"v{atom.srcs[0].frag_len}i32.v{atom.srcs[1].frag_len}i32" + f"f32.{atom.m}x{atom.n}x{atom.k}.f8f6f4.v{atom.c_frag_len}f32." + f"v{atom.a_frag_len}i32.v{atom.b_frag_len}i32" ) @property @@ -78,26 +80,23 @@ def gfx1250_scaled_wmma(op_id: str) -> ScaledWmmaOp | None: if atom is None or atom.family != "wmma_scaled": return None formats = {"fp8e4m3": 0, "bf8e5m2": 1} - src0, src1, src2 = atom.srcs # The current backend signatures require matching E8M0 scale packing. # Keep these restrictions here, independently of the catalog query model. if ( - src0.dtype not in formats - or src1.dtype not in formats - or src0.scale is None - or src0.scale != src1.scale - or src0.scale.dtype != "e8m0" - or src2.scale is not None - or src2.dtype != "fp32" - or atom.dst.dtype != "fp32" + atom.a_dtype not in formats + or atom.b_dtype not in formats + or atom.a_scale is None + or atom.a_scale != atom.b_scale + or atom.a_scale.dtype != "e8m0" + or atom.c_dtype != "fp32" or atom.shape != (16, 16, 128) ): raise ValueError(f"unsupported scaled WMMA backend contract: {atom.op_id}") return ScaledWmmaOp( atom=atom, - matrix_formats=(formats[src0.dtype], formats[src1.dtype]), + matrix_formats=(formats[atom.a_dtype], formats[atom.b_dtype]), scale_formats=(0, 0), # E8M0 for each source. scales=E8M0ScalePacking( - count=atom.k // src0.scale.block_size, block_k=src0.scale.block_size + count=atom.k // atom.a_scale.block_size, block_k=atom.a_scale.block_size ), ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py index 1f1eb493526a..7b6b47918382 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py @@ -130,8 +130,8 @@ def _check_cachepolicy(op: str, value: int) -> int: return v -def _mma_dst_frag_len(op_id: str) -> int: - """``dst`` fragment length for ``op_id`` from the arch SSOT. +def _mma_c_frag_len(op_id: str) -> int: + """Accumulator fragment length for ``op_id`` from the arch SSOT. Resolved through ``core/arch/target._MMA_FRAGMENT_INFO`` (imported lazily); ir.py holds no private copy. Unknown op_ids (frag length 0) raise, matching @@ -139,7 +139,7 @@ def _mma_dst_frag_len(op_id: str) -> int: """ from rocke.core.arch import target as _arch - frag_len = _arch._frag_info(op_id).dst.frag_len + frag_len = _arch._frag_info(op_id).c_frag_len if frag_len <= 0: raise ValueError( f"unknown MMA op_id {op_id!r}; pass an MmaOp or one of " @@ -148,21 +148,16 @@ def _mma_dst_frag_len(op_id: str) -> int: return frag_len -def _mma_dst_is_int(op_id: str) -> bool: - """True when ``op_id`` produces i32 (integer WMMA). +def _mma_c_is_int(op_id: str) -> bool: + """True when ``op_id`` accumulates in i32 (integer WMMA). - Sourced from the arch catalog's ``dst`` dtype - (``core/arch/data/arch_specs.json`` via ``target._op_id_dst_dtype``), imported - lazily. Op_ids absent from the catalog default to an f32 ``dst``. + Sourced from the arch catalog's accumulator dtype + (``core/arch/data/arch_specs.json`` via ``target._op_id_c_dtype``), imported + lazily. Op_ids absent from the catalog default to the f32 accumulator. """ from rocke.core.arch import target as _arch - return _arch._op_id_dst_dtype().get(op_id) == "i32" - - -# Compatibility aliases for code that historically used C as the result role. -_mma_c_frag_len = _mma_dst_frag_len -_mma_c_is_int = _mma_dst_is_int + return _arch._op_id_c_dtype().get(op_id) == "i32" @dataclass(frozen=True) @@ -1831,15 +1826,15 @@ def mma( """Target-neutral matrix-multiply-accumulate: ``D = A * B + C``. ``op`` is either an :class:`~rocke.core.arch.MmaOp` (preferred — its - ``op_id`` and ``dst`` fragment metadata drive the lowering) or a raw ``op_id`` + ``op_id`` and ``c_frag_len`` drive the lowering) or a raw ``op_id`` string. This emits a single ``tile.mma`` op carrying the ``op_id`` as an attribute; the LLVM lowering dispatches that ``op_id`` through the ISA backend (:meth:`rocke.core.isa.ISABackend.emit_mma`), which emits the matching MFMA call on CDNA or the WMMA call on RDNA. **One kernel body, two ISAs.** - The result vector type is ```` (the per-lane - result length the atom produces). When ``op`` is an ``op_id`` + The result vector type is ```` (the per-lane + accumulator length the atom produces). When ``op`` is an ``op_id`` string the frag length is resolved from the static MMA fragment table. The ISA-named helpers (:meth:`mfma_f32_16x16x16_f16`, @@ -1851,19 +1846,17 @@ def mma( (``a_scale``, ``b_scale``); ordinary atoms take exactly ``a, b, c``. """ op_id = op.op_id if hasattr(op, "op_id") else str(op) - dst_frag_len = ( - op.dst.frag_len - if hasattr(op, "dst") and op.dst.frag_len - else _mma_dst_frag_len(op_id) + c_frag_len = ( + op.c_frag_len + if hasattr(op, "c_frag_len") and op.c_frag_len + else _mma_c_frag_len(op_id) ) # Accumulator element type: integer WMMA atoms (iu8/iu4) accumulate in - # i32; everything else in f32. Prefer the atom's own dst dtype when ``op`` + # i32; everything else in f32. Prefer the atom's own c_dtype when ``op`` # is an MmaOp, else resolve from the arch SSOT via op_id. - dst_dtype = op.dst.dtype if hasattr(op, "dst") else None - is_int_acc = ( - dst_dtype == "i32" if dst_dtype is not None else _mma_dst_is_int(op_id) - ) - dst_elem = I32 if is_int_acc else F32 + c_dtype = getattr(op, "c_dtype", None) + is_int_acc = c_dtype == "i32" if c_dtype is not None else _mma_c_is_int(op_id) + c_elem = I32 if is_int_acc else F32 hint = ( "mxacc" if op_id.startswith("wmma.scaled.") @@ -1872,7 +1865,7 @@ def mma( return self._op( "tile.mma", [a, b, c, *extra], - [VectorType(dst_elem, dst_frag_len)], + [VectorType(c_elem, c_frag_len)], attrs={"op_id": op_id}, result_name_hint=hint, ).result diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py index 0b1ddb1511db..8996d6aeacc9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py @@ -28,7 +28,6 @@ VectorType, ) - _HIP_TYPE = { "i1": "bool", "i8": "int8_t", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py index 4743a1335ea8..e3249b1c64e3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py @@ -56,7 +56,6 @@ split_loc, ) - # Datalayout / triple. Copied verbatim from clang's output for the same # target on this box: clang -target amdgcn-amd-amdhsa -mcpu=gfx950 # -emit-llvm -S. The string is LLVM-version-keyed, not gfx-keyed: every diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py index 8dfdb56dd771..4c73642e4204 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py @@ -128,9 +128,10 @@ def _native_scaled_atom(spec: BlockScaledGemmSpec, target: ArchTarget): ) return target.mma.op_for_shape( family="wmma_scaled", - src_dtypes=(spec.dtype_a, spec.dtype_b, spec.dtype_acc), - dst_dtype=spec.dtype_acc, - src_scales=(scale, scale, None), + a_dtype=spec.dtype_a, + b_dtype=spec.dtype_b, + c_dtype=spec.dtype_acc, + scales=(scale, scale), m=_BLOCK_M, n=_BLOCK_N, k=_WMMA_SCALE_K, @@ -313,7 +314,7 @@ def build_block_scaled_gemm( op_id = atom.op_id if atom is not None else _wmma_op_id(spec.dtype_a, spec.dtype_b) scale_op = gfx1250_scaled_wmma(op_id) atom_k = _WMMA_SCALE_K if native_scale else _WMMA_K - frag_words = atom.srcs[0].frag_len if atom is not None else _ACC + frag_words = atom.a_frag_len if atom is not None else _ACC a_frag_ty = VectorType(I32, frag_words) groups = spec.K // spec.block_k diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp index 19ba29f2502e..e5834c8d24b6 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp @@ -794,13 +794,11 @@ void case_gfx1250_scaled_wmma() rocke_value_t* fragment = rocke_b_vec_concat(b, lo, hi); rocke_value_t* c = rocke_b_global_load_vN(b, accum, lane, rocke_f32(), 8, 0); rocke_value_t* scale = rocke_b_global_load(b, scales, lane, v.scale_type(), 1); - const char* src_dtypes[3] = {"fp8", "fp8", "fp32"}; const int block = v.scale16 ? 16 : 32; - const rocke_mma_scale_operand_t scales_query[3] - = {{"e8m0", block}, {"e8m0", block}, {NULL, 0}}; + const rocke_mma_scale_operand_t scales_query[2] = {{"e8m0", block}, {"e8m0", block}}; const rocke_arch_target_t* target = rocke_arch_target_from_gfx("gfx1250"); - const rocke_mma_op_t* atom = rocke_mma_catalog_op_for_shape_indexed( - &target->mma, "wmma_scaled", src_dtypes, "fp32", scales_query, 16, 16, 128); + const rocke_mma_op_t* atom = rocke_mma_catalog_op_for_shape( + &target->mma, "wmma_scaled", "fp8", "fp8", "fp32", 16, 16, 128, scales_query); rocke_value_t* extra[2] = {scale, scale}; rocke_value_t* d = rocke_b_mma(b, atom->op_id, fragment, fragment, c, extra, 2); rocke_b_global_store(b, accum, lane, rocke_b_vec_extract(b, d, 0), 1); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp index 3a172134c346..7ce214a1e235 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp @@ -4,8 +4,8 @@ * tests/core/mma_frag_ssot.cpp -- host unit test for the IR-layer MMA * frag-length / accumulator-dtype tables consulted by rocke_b_mma. * - * rocke_b_mma sizes its tile.mma result vector as , - * where dst_frag_len comes from the op_id frag-length table and acc_elem is i32 + * rocke_b_mma sizes its tile.mma result vector as , + * where c_frag_len comes from the op_id frag-length table and acc_elem is i32 * for integer WMMA atoms (else f32). This test pins that mapping for a * representative set of atoms and checks the unknown-op_id error path, so a * table edit that changes a result width/dtype is caught here. @@ -16,10 +16,8 @@ */ #include #include -#include #include "rocke/arch_target.h" -#include "rocke/error.hpp" #include "rocke/ir.h" static int g_failures = 0; @@ -62,9 +60,7 @@ static void check_atom(rocke_ir_builder_t* b, const char* op_id, int expect_frag } } -static void check_catalog_fragments(const rocke_mma_catalog_t* catalog, - const char* op_id, - int scale_block_size) +static void check_catalog_fragments(const rocke_mma_catalog_t* catalog, const char* op_id) { const rocke_mma_op_t* op = rocke_mma_catalog_by_op_id(catalog, op_id); CHECK(op != NULL, op_id); @@ -72,203 +68,13 @@ static void check_catalog_fragments(const rocke_mma_catalog_t* catalog, { return; } - CHECK(op->srcs[0].frag_len == 16, op_id); - CHECK(op->srcs[1].frag_len == 16, op_id); - CHECK(op->srcs[2].frag_len == 8, op_id); - CHECK(op->dst.frag_len == 8, op_id); - for(int i = 0; i < 2; ++i) - { - CHECK(strcmp(op->srcs[i].dtype, "fp8e4m3") == 0, op_id); - CHECK(op->srcs[i].scale_dtype != NULL, op_id); - if(op->srcs[i].scale_dtype) - { - CHECK(strcmp(op->srcs[i].scale_dtype, "e8m0") == 0, op_id); - } - CHECK(op->srcs[i].scale_block_size == scale_block_size, op_id); - } - CHECK(op->srcs[2].scale_dtype == NULL && op->srcs[2].scale_block_size == 0, op_id); -} - -static void check_contract_queries() -{ - rocke_mma_op_t rows[10] = {}; - rows[0].family = "wmma_scaled"; - rows[0].srcs[0] = {"fp8e4m3", 16, NULL, "e8m0", 32}; - rows[0].srcs[1] = {"bf8e5m2", 16, NULL, "e8m0", 32}; - rows[0].srcs[2] = {"fp32", 8, NULL, NULL, 0}; - rows[0].dst = {"fp32", 8, NULL}; - rows[0].m = rows[0].n = 16; - rows[0].k = 128; - for(int i = 1; i < 10; ++i) - rows[i] = rows[0]; - for(int src = 0; src < 2; ++src) - { - rows[1 + src * 3].srcs[src].scale_dtype = "e4m3"; - rows[2 + src * 3].srcs[src].scale_block_size = 16; - rows[3 + src * 3].srcs[src].scale_dtype = NULL; - rows[3 + src * 3].srcs[src].scale_block_size = 0; - } - rows[7].srcs[2].dtype = "fp16"; - rows[8].dst.dtype = "fp16"; - for(int i = 0; i < 2; ++i) - { - rows[9].srcs[i].scale_dtype = NULL; - rows[9].srcs[i].scale_block_size = 0; - } - const rocke_mma_catalog_t catalog = {rows, 10}; - char ids[10][256]; - for(int i = 0; i < 10; ++i) - { - rocke_mma_op_t* row = &rows[i]; - CHECK(rocke_mma_op_semantic_id(row, ids[i], sizeof(ids[i])) != NULL, "format contract ID"); - row->op_id = ids[i]; - for(int j = 0; j < i; ++j) - CHECK(strcmp(ids[i], ids[j]) != 0, "different contracts have different IDs"); - const char* dtypes[3] = {"fp8", "bf8", row->srcs[2].dtype}; - rocke_mma_scale_operand_t scales[3]; - for(int j = 0; j < 3; ++j) - scales[j] = {row->srcs[j].scale_dtype, row->srcs[j].scale_block_size}; - // Exercise the scale-format alias through every query operation. - if(i == 1) - scales[0].dtype = "fp8e4m3"; - const rocke_mma_op_t* out[10]; - CHECK(rocke_mma_catalog_enumerate_indexed( - &catalog, row->family, dtypes, row->dst.dtype, scales, 16, 16, out, 10) - == 1 - && out[0] == row, - "enumeration distinguishes every operand contract"); - CHECK(rocke_mma_catalog_has_shape_indexed( - &catalog, row->family, dtypes, row->dst.dtype, scales, 16, 16, 128), - "full contract exists"); - CHECK(rocke_mma_catalog_op_for_shape_indexed( - &catalog, row->family, dtypes, row->dst.dtype, scales, 16, 16, 128) - == row, - "exact contract selection"); - CHECK(rocke_mma_catalog_select_largest_k_indexed( - &catalog, row->family, dtypes, row->dst.dtype, scales, 16, 16, -1) - == row, - "largest K contract selection"); - } - const char* dtypes[3] = {"fp8", "bf8", "fp32"}; - CHECK(rocke_mma_catalog_enumerate_indexed( - &catalog, "wmma_scaled", dtypes, "fp32", NULL, 16, 16, NULL, 0) - == 8, - "omitted scales are wildcard"); - for(bool largest : {false, true}) - { - bool rejected = false; - try - { - if(largest) - rocke_mma_catalog_select_largest_k_indexed( - &catalog, "wmma_scaled", dtypes, "fp32", NULL, 16, 16, -1); - else - rocke_mma_catalog_op_for_shape_indexed( - &catalog, "wmma_scaled", dtypes, "fp32", NULL, 16, 16, 128); - } - catch(const ckc::Error&) - { - rejected = true; - } - CHECK(rejected, "underspecified selection rejects ambiguity"); - } - rocke_mma_scale_operand_t scales[3] = {{"e5m3", 32}, {"e5m3", 32}, {NULL, 0}}; - CHECK(rocke_mma_catalog_op_for_shape_indexed( - &catalog, "wmma_scaled", dtypes, "fp32", scales, 16, 16, 128) - == NULL, - "unsupported contract absent"); + CHECK(op->a_frag_len == 16, op_id); + CHECK(op->b_frag_len == 16, op_id); + CHECK(op->c_frag_len == 8, op_id); } int main(void) { - check_contract_queries(); - const char* cpu_targets[] = {"gfx950", "gfx1250"}; - for(const char* gfx : cpu_targets) - { - const rocke_arch_target_t* target = rocke_arch_target_from_gfx(gfx); - CHECK(target != NULL, "CPU catalog target exists"); - if(!target) - { - continue; - } - for(int i = 0; i < target->mma.num_ops; ++i) - { - const rocke_mma_op_t* op = &target->mma.ops[i]; - const char* src_dtypes[3] = {op->srcs[0].dtype, op->srcs[1].dtype, op->srcs[2].dtype}; - rocke_mma_scale_operand_t scales[3]; - for(int j = 0; j < 3; ++j) - scales[j] = {op->srcs[j].scale_dtype, op->srcs[j].scale_block_size}; - if(strcmp(op->family, "wmma_scaled") == 0) - { - char id[256]; - CHECK(rocke_mma_op_semantic_id(op, id, sizeof(id)) != NULL, "semantic ID fits"); - CHECK(strcmp(id, op->op_id) == 0, "semantic ID agrees with catalog"); - } - CHECK(rocke_mma_catalog_op_for_shape_indexed(&target->mma, - op->family, - src_dtypes, - op->dst.dtype, - scales, - op->m, - op->n, - op->k) - != NULL, - "indexed query resolves CPU catalog key"); - } - } - - { - rocke_mma_op_t distinct = {}; - const rocke_layout_map_t src2_layout = {ROCKE_MMA_ROLE_SRC2, 3, 32, NULL}; - const rocke_layout_map_t dst_layout = {ROCKE_MMA_ROLE_DST, 7, 32, NULL}; - distinct.family = "mma"; - distinct.srcs[0].dtype = "xf32"; - distinct.srcs[1].dtype = "xf32"; - distinct.srcs[0].scale_dtype = "e8m0"; - distinct.srcs[0].scale_block_size = 32; - distinct.srcs[1].scale_dtype = "e4m3"; - distinct.srcs[1].scale_block_size = 16; - distinct.srcs[2].dtype = "fp32"; - distinct.srcs[2].frag_len = 3; - distinct.srcs[2].layout = &src2_layout; - distinct.dst.dtype = "i32"; - distinct.dst.frag_len = 7; - distinct.dst.layout = &dst_layout; - distinct.m = 16; - distinct.n = 16; - distinct.k = 8; - distinct.op_id = "synthetic_distinct_dst"; - const rocke_mma_catalog_t catalog = {&distinct, 1}; - const char* src_dtypes[3] = {"xf32", "xf32", "fp32"}; - CHECK(rocke_mma_catalog_op_for_shape_indexed( - &catalog, "mma", src_dtypes, "i32", NULL, 16, 16, 8) - == &distinct, - "indexed query distinguishes src2 and dst"); - CHECK(rocke_mma_catalog_op_for_shape(&catalog, "mma", "xf32", "xf32", "fp32", 16, 16, 8) - == NULL, - "legacy query defaults dst to src2"); - CHECK(rocke_mma_op_src_layout(&distinct, 2, NULL) == &src2_layout, - "indexed src2 layout remains independent"); - CHECK(rocke_mma_op_dst_layout(&distinct, NULL) == &dst_layout, - "dst layout remains independent"); - CHECK(rocke_mma_op_c_layout(&distinct, NULL) == &dst_layout, - "historical C layout projects dst"); - const rocke_mma_op_t* selected - = rocke_mma_catalog_by_op_id(&catalog, "synthetic_distinct_dst"); - CHECK(selected != NULL, "catalog preserves scaled operand metadata"); - if(selected) - { - CHECK(strcmp(selected->srcs[0].scale_dtype, "e8m0") == 0 - && selected->srcs[0].scale_block_size == 32, - "src0 scale retains its value format and block size"); - CHECK(strcmp(selected->srcs[1].scale_dtype, "e4m3") == 0 - && selected->srcs[1].scale_block_size == 16, - "src1 scale is independent of src0"); - CHECK(selected->srcs[2].scale_dtype == NULL && selected->srcs[2].scale_block_size == 0, - "src2 remains unscaled"); - } - } - rocke_ir_builder_t b; if(rocke_ir_builder_init(&b, "rocke_mma_frag_ssot") != ROCKE_OK) { @@ -295,12 +101,10 @@ int main(void) { check_catalog_fragments( &gfx1250->mma, - "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32", - 32); + "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32"); check_catalog_fragments( &gfx1250->mma, - "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32", - 16); + "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32"); } /* Unknown op_id must be rejected. The engine's error path either returns diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py index edce7ca18915..58641452b7d1 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py @@ -1,265 +1,193 @@ # Copyright (c) Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT -"""SSOT guards for indexed MMA ``srcs`` and ``dst`` metadata. +"""SSOT guards for the bare-op_id MMA accumulator-dtype lookup. -``IRBuilder.mma`` uses ``dst`` metadata to size a ``tile.mma`` result. These -tests pin the ``dst`` lookup, indexed catalog query, and optional per-source -scale parsing without assuming ``src2`` and ``dst`` are always identical. +``IRBuilder.mma`` uses ``target._op_id_c_dtype()`` to size a ``tile.mma`` result +vector's accumulator element without an ``ArchTarget`` in hand. These tests pin +the first-wins / raise-on-drift contract of that lookup so it stays deterministic +across the arches that list a given op_id. """ from __future__ import annotations import unittest +from dataclasses import replace from unittest import mock +import pytest + +from rocke.core.arch import ArchTarget, MmaCatalog, MmaOp, MmaScaleOperand +from rocke.core.arch.wmma_scale import gfx1250_scaled_wmma + from rocke.core.arch.target import ( - ArchTarget, - MmaCatalog, - MmaOp, - MmaDst, - MmaScaleOperand, - MmaSrc, - _FragInfo, - _FragOperand, - _MMA_FRAGMENT_INFO, - _build_mma_op, _load_specs, - _op_id_dst_dtype, + _op_id_c_dtype, normalize_dtype, ) -from rocke.core.ir import IRBuilder -class TestOpIdDstDtype(unittest.TestCase): +class TestOpIdCDtype(unittest.TestCase): def test_matches_catalog_first_hit(self): - # Every op_id in the catalog resolves to its normalized dst dtype, + # Every op_id in the catalog resolves to its normalized accumulator dtype, # taking the first arch that lists it (dict preserves catalog order). expected: dict = {} for row in _load_specs().values(): for o in row["mma"]: - expected.setdefault(o["op_id"], normalize_dtype(o["dst"]["dtype"])) - self.assertEqual(_op_id_dst_dtype(), expected) + expected.setdefault(o["op_id"], normalize_dtype(o["c"])) + self.assertEqual(_op_id_c_dtype(), expected) - def test_dst_dtype_invariant_across_arches(self): - # The whole premise of the bare-op_id lookup: an op_id's dst dtype + def test_c_dtype_invariant_across_arches(self): + # The whole premise of the bare-op_id lookup: an op_id's accumulator dtype # is invariant across the arches that list it, so building the map must not # raise on the real catalog. (The raise path is exercised below.) try: - _op_id_dst_dtype() + _op_id_c_dtype() except ValueError as exc: # pragma: no cover - only hit on real drift - self.fail(f"_op_id_dst_dtype() raised on the shipped catalog: {exc}") + self.fail(f"_op_id_c_dtype() raised on the shipped catalog: {exc}") def test_raises_on_cross_arch_disagreement(self): specs = _load_specs() - # Find an op_id and clone its row into a fake arch with a different dst dtype. + # Find an op_id and clone its row into a fake arch with a different c dtype. sample = next(o for row in specs.values() for o in row["mma"]) - original_dst = normalize_dtype(sample["dst"]["dtype"]) - other_dst = "i32" if original_dst != "i32" else "f32" + original_c = normalize_dtype(sample["c"]) + other_c = "i32" if original_c != "i32" else "f32" clash = dict(sample) - clash["dst"] = {"dtype": other_dst} + clash["c"] = other_c drifted = dict(specs) drifted["_synthetic_drift"] = {"mma": [clash]} - _op_id_dst_dtype.cache_clear() + _op_id_c_dtype.cache_clear() try: with mock.patch("rocke.core.arch.target._load_specs", return_value=drifted): with self.assertRaises(ValueError): - _op_id_dst_dtype() + _op_id_c_dtype() finally: - _op_id_dst_dtype.cache_clear() - - -class TestIndexedMmaOperands(unittest.TestCase): - def test_cpu_catalog_indexed_queries_cover_gfx950_and_gfx1250(self): - for gfx in ("gfx950", "gfx1250"): - catalog = ArchTarget.from_gfx(gfx).mma - for op in catalog.ops: - with self.subTest(gfx=gfx, op_id=op.op_id): - self.assertEqual(len(op.srcs), 3) - self.assertIn( - op, - catalog.enumerate( - family=op.family, - src_dtypes=tuple(src.dtype for src in op.srcs), - dst_dtype=op.dst.dtype, - m=op.m, - n=op.n, - ), - ) - - def test_indexed_query_distinguishes_src2_and_dst(self): - distinct = MmaOp( - family="mma", - srcs=(MmaSrc("xf32"), MmaSrc("xf32"), MmaSrc("fp32")), - dst=MmaDst("i32", frag_len=7), - m=16, - n=16, - k=8, - op_id="synthetic_distinct_dst", - ) - catalog = MmaCatalog([distinct]) - self.assertEqual( - catalog.enumerate(src_dtypes=("xf32", "xf32", "fp32"), dst_dtype="i32"), - [distinct], + _op_id_c_dtype.cache_clear() + + +def _contracts(): + base = MmaOp( + family="wmma_scaled", + a_dtype="fp8e4m3", + b_dtype="bf8e5m2", + c_dtype="fp32", + a_scale=MmaScaleOperand("e8m0", 32), + b_scale=MmaScaleOperand("e8m0", 32), + m=16, + n=16, + k=128, + op_id="fixture", + ) + rows = [base] + for field in ("a_scale", "b_scale"): + for scale in (MmaScaleOperand("e4m3", 32), MmaScaleOperand("e8m0", 16), None): + rows.append(replace(base, **{field: scale})) + rows.append(replace(base, a_scale=None, b_scale=None)) + return [replace(row, op_id=row.semantic_id()) for row in rows] + + +def test_full_contract_queries_distinguish_each_input_scale(): + rows = _contracts() + assert len({row.op_id for row in rows}) == len(rows) + catalog = MmaCatalog(rows) + for row in rows: + query = dict( + family=row.family, + a_dtype=row.a_dtype, + b_dtype=row.b_dtype, + c_dtype=row.c_dtype, + scales=(row.a_scale, row.b_scale), + m=row.m, + n=row.n, ) - self.assertEqual( - catalog.enumerate(a_dtype="xf32", b_dtype="xf32", c_dtype="fp32"), - [], - ) - - builder = IRBuilder("distinct_dst") - value = builder.const_i32(0) - result = builder.mma(distinct, value, value, value) - self.assertEqual(result.type.count, distinct.dst.frag_len) - self.assertEqual(result.type.elem.name, "i32") - - def test_fragment_metadata_distinguishes_src2_and_dst(self): - def src2_coord(builder, lane, slot): - return builder.const_i32(slot), lane - - def dst_coord(builder, lane, slot): - return lane, builder.const_i32(slot) - - op_id = "synthetic_distinct_fragment_metadata" - info = _FragInfo( - srcs=( - _FragOperand(1), - _FragOperand(1), - _FragOperand(3, src2_coord), - ), - dst=_FragOperand(7, dst_coord), - wave_size=32, - ) - row = { - "family": "mma", - "srcs": [ - {"dtype": "xf32"}, - {"dtype": "xf32"}, - {"dtype": "fp32"}, - ], - "dst": {"dtype": "i32"}, - "m": 16, - "n": 16, - "k": 8, - "op_id": op_id, - } - - with mock.patch.dict(_MMA_FRAGMENT_INFO, {op_id: info}): - op = _build_mma_op(row) - - self.assertEqual(op.srcs[2].frag_len, 3) - self.assertEqual(op.dst.frag_len, 7) - self.assertIs(op.src_layout(2).fn, src2_coord) - self.assertIs(op.dst_layout().fn, dst_coord) - self.assertEqual(op.c_frag_len, op.dst.frag_len) - self.assertIs(op.c_layout(), op.dst_layout()) - - builder = IRBuilder("distinct_fragment_metadata") - value = builder.const_i32(0) - result = builder.mma(op, value, value, value) - self.assertEqual(result.type.count, op.dst.frag_len) - self.assertEqual(result.type.elem.name, "i32") - - def test_scaled_wmma_catalog_preserves_scale_contracts(self): - catalog = ArchTarget.from_gfx("gfx1250").mma - for family, block_size in (("wmma_scale", 32), ("wmma_scale16", 16)): - with self.subTest(family=family): - scale = MmaScaleOperand("e8m0", block_size) - op = catalog.op_for_shape( - family="wmma_scaled", - src_dtypes=("fp8", "fp8", "fp32"), - dst_dtype="fp32", - src_scales=(scale, scale, None), - m=16, - n=16, - k=128, - ) - self.assertIsNotNone(op) - self.assertEqual(op.family, "wmma_scaled") - self.assertEqual(op.shape, (16, 16, 128)) - for src in op.srcs[:2]: - self.assertEqual(src.dtype, "fp8e4m3") - self.assertEqual(src.frag_len, 16) - self.assertEqual(src.scale, MmaScaleOperand("e8m0", block_size)) - self.assertIsNone(op.srcs[2].scale) - self.assertEqual(op.srcs[2].frag_len, 8) - self.assertEqual(op.dst.frag_len, 8) - self.assertEqual(op.src_layout(2).role, "src2") - self.assertEqual(op.dst_layout().role, "dst") - - def test_scale_format_and_block_size_are_independent_per_source(self): - # Synthetic rows exercise the descriptor without admitting new hardware - # operations into the shipped catalog. - for dtype, block_size in ( - ("e8m0", 32), - ("e8m0", 16), - ("e4m3", 16), - ("fp8e4m3", 16), - ("e5m3", 16), - ): - with self.subTest(dtype=dtype, block_size=block_size): - op = _build_mma_op( - { - "family": "mma", - "srcs": [ - { - "dtype": "fp4", - "scale": {"dtype": dtype, "block_size": block_size}, - }, - { - "dtype": "fp4", - "scale": {"dtype": "e8m0", "block_size": 32}, - }, - {"dtype": "fp32"}, - ], - "dst": {"dtype": "fp32"}, - "m": 16, - "n": 16, - "k": 128, - "op_id": "synthetic_scaled_mma", - } - ) - expected_dtype = "e4m3" if dtype == "fp8e4m3" else dtype - self.assertEqual( - op.srcs[0].scale, MmaScaleOperand(expected_dtype, block_size) - ) - self.assertEqual(op.srcs[1].scale, MmaScaleOperand("e8m0", 32)) - self.assertIsNone(op.srcs[2].scale) - self.assertFalse(hasattr(op.dst, "scale")) - - def test_scale_dtype_must_be_a_supported_value_format(self): - for dtype in ("i32", "fp4", "e5m2", "", None): - with self.subTest(dtype=dtype): - with self.assertRaisesRegex(ValueError, "e8m0, e4m3, or e5m3"): - MmaScaleOperand(dtype, 16) - - def test_scale_dtype_alias_is_canonicalized(self): - for block_size in (16, 32): - with self.subTest(block_size=block_size): - alias = MmaScaleOperand("fp8e4m3", block_size) - canonical = MmaScaleOperand("e4m3", block_size) - self.assertEqual(alias.dtype, "e4m3") - self.assertEqual(alias, canonical) - self.assertEqual(hash(alias), hash(canonical)) - - def test_scale_block_size_must_be_16_or_32(self): - for block_size in (0, -16, 1, 8, 64, 16.0, 32.0, 16.5, "16", True, None): - with self.subTest(block_size=block_size): - with self.assertRaisesRegex(ValueError, "integer equal to 16 or 32"): - MmaScaleOperand("e8m0", block_size) - - def test_scale_identity_includes_format_and_block_size(self): - self.assertEqual( - len( - { - MmaScaleOperand("e8m0", 32), - MmaScaleOperand("e8m0", 16), - MmaScaleOperand("e4m3", 16), - } - ), - 3, + assert catalog.enumerate(**query) == [row] + assert catalog.has_shape(**query, k=row.k) + assert catalog.op_for_shape(**query, k=row.k) is row + assert catalog.select_largest_k(**query) is row + assert catalog.select_largest_k(**query, k_max=64) is None + + +def test_partial_query_rejects_ambiguity_but_enumeration_and_existence_are_valid(): + catalog = MmaCatalog(_contracts()) + query = dict( + family="wmma_scaled", + a_dtype="fp8", + b_dtype="bf8", + c_dtype="fp32", + m=16, + n=16, + ) + assert len(catalog.enumerate(**query)) == 8 + assert catalog.has_shape(**query, k=128) + with pytest.raises(ValueError, match="ambiguous MMA query"): + catalog.op_for_shape(**query, k=128) + with pytest.raises(ValueError, match="ambiguous MMA query"): + catalog.select_largest_k(**query) + assert len(catalog.enumerate(**query, scales=(None, None))) == 1 + scale = MmaScaleOperand("e5m3", 32) + assert catalog.op_for_shape(**query, scales=(scale, scale), k=128) is None + with pytest.raises(ValueError, match="exactly 2"): + catalog.enumerate(**query, scales=(None,)) + + +def test_scale_alias_selects_the_same_contract(): + catalog = MmaCatalog(_contracts()) + query = dict( + family="wmma_scaled", + a_dtype="fp8", + b_dtype="bf8", + c_dtype="f32", + m=16, + n=16, + k=128, + ) + scales = (MmaScaleOperand("fp8e4m3", 32), MmaScaleOperand("e8m0", 32)) + row = catalog.op_for_shape(**query, scales=scales) + assert row is not None and row.a_scale.dtype == "e4m3" + + +def test_scaled_catalog_identity_and_backend_contract(): + catalog = ArchTarget.from_gfx("gfx1250").mma + rows = [row for row in catalog.ops if row.family == "wmma_scaled"] + assert len(rows) == 4 + for row in rows: + assert row.op_id == row.semantic_id() + packing = gfx1250_scaled_wmma(row.op_id) + assert packing.atom is row + assert packing.matrix_formats == ( + (0, 0) if row.a_dtype == "fp8e4m3" else (1, 1) ) + assert packing.scales.count * packing.scales.block_k == row.k + assert packing.matrix_llvm_types == ("<16 x i32>", "<16 x i32>") + for family in ("wmma_scale", "wmma_scale16"): + old_id = f"{family}_f32_16x16x128_fp8_fp8" + assert catalog.by_op_id(old_id) is None + assert gfx1250_scaled_wmma(old_id) is None + + +@pytest.mark.parametrize("dtype", ["i32", "fp4", "e5m2", "", None]) +def test_invalid_scale_format(dtype): + with pytest.raises(ValueError, match="e8m0, e4m3, or e5m3"): + MmaScaleOperand(dtype, 16) + + +@pytest.mark.parametrize("block", [0, 8, 64, 16.0, True, "32", None]) +def test_invalid_scale_block_size(block): + with pytest.raises(ValueError, match="integer equal to 16 or 32"): + MmaScaleOperand("e8m0", block) + + +def test_largest_k_only_rejects_ties_at_the_selected_k(): + rows = _contracts() + unique = replace(rows[0], k=256) + catalog = MmaCatalog([*rows, unique]) + query = dict( + family="wmma_scaled", a_dtype="fp8", b_dtype="bf8", c_dtype="fp32", m=16, n=16 + ) + assert catalog.select_largest_k(**query) is unique + with pytest.raises(ValueError, match="ambiguous MMA query"): + catalog.select_largest_k(**query, k_max=128) if __name__ == "__main__": # pragma: no cover diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py index 0fbf5d7b6f3f..0717f7f6007d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py @@ -5,8 +5,11 @@ from __future__ import annotations import unittest +from dataclasses import replace +from unittest import mock from rocke.core.arch import ArchTarget, MmaScaleOperand +from rocke.core.arch.wmma_scale import gfx1250_scaled_wmma from rocke.core.ir import F32, I32, I64, IRBuilder, PtrType from rocke.core.ir_serialize import parse, serialize from rocke.core.lower_hip import lower_kernel_to_hip @@ -23,9 +26,10 @@ def _scaled_atom(dtype="fp8", scale16=False): scale = MmaScaleOperand("e8m0", 16 if scale16 else 32) atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( family="wmma_scaled", - src_dtypes=(dtype, dtype, "fp32"), - dst_dtype="fp32", - src_scales=(scale, scale, None), + a_dtype=dtype, + b_dtype=dtype, + c_dtype="fp32", + scales=(scale, scale), m=16, n=16, k=128, @@ -52,6 +56,17 @@ def _build_scaled_atom(*, scale16: bool): class TestGfx1250ScaledWmma(unittest.TestCase): + def test_backend_derives_each_matrix_carrier_and_selector(self): + # Synthetic metadata exercises the lowering contract without adding atoms. + atom = replace(_scaled_atom(), b_dtype="bf8e5m2", b_frag_len=8) + catalog = ArchTarget.from_gfx("gfx1250").mma + with mock.patch.object(catalog, "by_op_id", return_value=atom): + spec = gfx1250_scaled_wmma(atom.op_id) + self.assertEqual(spec.matrix_formats, (0, 1)) + self.assertEqual(spec.matrix_llvm_types, ("<16 x i32>", "<8 x i32>")) + self.assertTrue(spec.intrinsic.endswith("v16i32.v8i32")) + self.assertIn("wmma.scale.block32", spec.declaration_key) + def test_matrix_formats_share_intrinsic_declarations(self): b = IRBuilder("shared_scaled_wmma_declarations") matrix = b.param("matrix", PtrType(I32, "global"), readonly=True) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py index 3936546309b4..cfdf6cde84f7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py @@ -32,7 +32,7 @@ def test_alias_catalog_lookup(alias, canonical): found = True assert catalog.has_shape( family=op.family, - src_scales=tuple(src.scale for src in op.srcs), + scales=(op.a_scale, op.b_scale), a_dtype=alias, b_dtype=op.b_dtype, c_dtype=op.c_dtype, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp index 0b9cb310a1f5..20a1fc8ca85d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp @@ -3,6 +3,8 @@ /* Exercise public catalog queries against stored keys, without normalizing * away malformed table entries. Alias queries must find the same atom. */ #include "rocke/arch_target.h" +#include "rocke/error.hpp" +#include "rocke/wmma_scale_internal.h" #include #include @@ -23,8 +25,151 @@ static const char* short_dtype(const char* dtype) return dtype; } +#define CHECK(condition) \ + do \ + { \ + if(!(condition)) \ + { \ + fprintf(stderr, "line %d: %s\\n", __LINE__, #condition); \ + return 1; \ + } \ + } while(0) + +template +static bool rejects_query(F query) +{ + try + { + query(); + } + catch(const ckc::Error& e) + { + return e.code() == ROCKE_ERR_VALUE; + } + return false; +} + +static int test_scale_contracts() +{ + const auto* arch = rocke_arch_target_from_gfx("gfx1250"); + const rocke_mma_scale_operand_t e8[2] = {{"e8m0", 32}, {"e8m0", 32}}; + const auto* base = rocke_mma_catalog_op_for_shape( + &arch->mma, "wmma_scaled", "fp8", "fp8", "fp32", 16, 16, 128, e8); + CHECK(base); + rocke_mma_op_t rows[9]; + rows[0] = *base; + rows[0].b_dtype = "bf8e5m2"; + auto packing_atom = rows[0]; + packing_atom.b_frag_len = 8; + const auto packing = rocke_scaled_wmma_contract(&packing_atom); + CHECK(packing.matrix_formats[0] == 0 && packing.matrix_formats[1] == 1); + CHECK(packing.matrix_words[0] == 16 && packing.matrix_words[1] == 8); + CHECK(strstr(packing.intrinsic, "v16i32.v8i32")); + int count = 1; + for(int input = 0; input < 2; ++input) + { + for(const auto scale : {rocke_mma_scale_operand_t{"e4m3", 32}, + rocke_mma_scale_operand_t{"e8m0", 16}, + rocke_mma_scale_operand_t{NULL, 0}}) + { + rows[count] = rows[0]; + (input == 0 ? rows[count].a_scale : rows[count].b_scale) = scale; + ++count; + } + } + rows[count] = rows[0]; + rows[count].a_scale = {NULL, 0}; + rows[count++].b_scale = {NULL, 0}; + rocke_mma_catalog_t cat = {rows, count}; + char ids[8][256]; + for(int i = 0; i < count; ++i) + { + const auto& op = rows[i]; + const rocke_mma_scale_operand_t scales[2] = {op.a_scale, op.b_scale}; + CHECK(rocke_mma_op_semantic_id(&op, ids[i], sizeof(ids[i]))); + for(int j = 0; j < i; ++j) + CHECK(strcmp(ids[i], ids[j]) != 0); + CHECK(rocke_mma_catalog_enumerate( + &cat, "wmma_scaled", "fp8", "bf8", "f32", 16, 16, NULL, 0, scales) + == 1); + CHECK(rocke_mma_catalog_has_shape( + &cat, "wmma_scaled", "fp8", "bf8", "f32", 16, 16, 128, scales)); + CHECK(rocke_mma_catalog_op_for_shape( + &cat, "wmma_scaled", "fp8", "bf8", "f32", 16, 16, 128, scales) + == &op); + CHECK(rocke_mma_catalog_select_largest_k( + &cat, "wmma_scaled", "fp8", "bf8", "f32", 16, 16, -1, scales) + == &op); + CHECK(!rocke_mma_catalog_select_largest_k( + &cat, "wmma_scaled", "fp8", "bf8", "f32", 16, 16, 64, scales)); + } + CHECK(rocke_mma_catalog_enumerate(&cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, NULL, 0) + == count); + CHECK(rocke_mma_catalog_has_shape(&cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128)); + CHECK(rejects_query([&] { + rocke_mma_catalog_op_for_shape(&cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128); + })); + CHECK(rejects_query([&] { + rocke_mma_catalog_select_largest_k(&cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, -1); + })); + const rocke_mma_scale_operand_t unscaled[2] = {{NULL, 0}, {NULL, 0}}; + CHECK(rocke_mma_catalog_op_for_shape( + &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128, unscaled) + == &rows[7]); + const rocke_mma_scale_operand_t alias[2] = {{"fp8e4m3", 32}, {"e8m0", 32}}; + CHECK(rocke_mma_catalog_op_for_shape( + &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128, alias) + == &rows[1]); + rows[8] = rows[0]; + rows[8].k = 256; + cat.num_ops = 9; + CHECK(rocke_mma_catalog_select_largest_k(&cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, -1) + == &rows[8]); + CHECK(rejects_query([&] { + rocke_mma_catalog_select_largest_k(&cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128); + })); + for(const auto invalid : {rocke_mma_scale_operand_t{"i32", 32}, + rocke_mma_scale_operand_t{"e8m0", 8}, + rocke_mma_scale_operand_t{NULL, 32}}) + { + const rocke_mma_scale_operand_t scales[2] = {invalid, e8[1]}; + CHECK(rejects_query([&] { + rocke_mma_catalog_enumerate( + &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, NULL, 0, scales); + })); + } + const rocke_mma_scale_operand_t valid_missing[2] = {{"e5m3", 32}, {"e8m0", 32}}; + CHECK(!rocke_mma_catalog_op_for_shape( + &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128, valid_missing)); + char id[256]; + CHECK(rocke_mma_op_semantic_id(base, id, sizeof(id))); + CHECK(strcmp(id, base->op_id) == 0); + CHECK(!rocke_mma_op_semantic_id(base, id, strlen(base->op_id))); + CHECK(rocke_mma_op_semantic_id(base, id, strlen(base->op_id) + 1)); + int packed_rows = 0, scaled_rows = 0; + for(int i = 0; i < arch->mma.num_ops; ++i) + { + const auto& op = arch->mma.ops[i]; + if(strncmp(op.op_id, "wmma_gfx1250_f32_16x16x64_", 25) == 0) + { + CHECK(op.a_frag_len == 8 && op.b_frag_len == 8 && op.c_frag_len == 8); + ++packed_rows; + } + if(strcmp(op.family, "wmma_scaled") == 0) + { + CHECK(rocke_mma_op_semantic_id(&op, id, sizeof(id))); + CHECK(strcmp(id, op.op_id) == 0); + ++scaled_rows; + } + } + CHECK(packed_rows == 4 && scaled_rows == 4); + return 0; +} + int main() { + if(test_scale_contracts()) + return 1; int checked = 0; for(const char* gfx : {"gfx950", "gfx1250"}) { @@ -34,15 +179,7 @@ int main() for(int i = 0; i < arch->mma.num_ops; ++i) { const rocke_mma_op_t* op = &arch->mma.ops[i]; - // K=64 low-bit WMMA packs 32 bytes into eight i32 registers per source. - if(strcmp(gfx, "gfx1250") == 0 && op->k == 64 && strcmp(op->family, "wmma") == 0 - && (op->srcs[0].frag_len != 8 || op->srcs[1].frag_len != 8)) - { - fprintf(stderr, "%s: fragment length must count packed i32 elements\n", op->op_id); - return 1; - } - for(const char* dtype : - {op->srcs[0].dtype, op->srcs[1].dtype, op->srcs[2].dtype, op->dst.dtype}) + for(const char* dtype : {op->a_dtype, op->b_dtype, op->c_dtype}) { char scratch[64]; if(strcmp(dtype, rocke_normalize_dtype(dtype, scratch, sizeof(scratch))) != 0) @@ -51,49 +188,43 @@ int main() return 1; } } - for(const char* a : {op->srcs[0].dtype, short_dtype(op->srcs[0].dtype)}) + const rocke_mma_scale_operand_t scales[2] = {op->a_scale, op->b_scale}; + for(const char* a : {op->a_dtype, short_dtype(op->a_dtype)}) { - for(const char* b : {op->srcs[1].dtype, short_dtype(op->srcs[1].dtype)}) + for(const char* b : {op->b_dtype, short_dtype(op->b_dtype)}) { - const char* src_dtypes[3] = {a, b, op->srcs[2].dtype}; - rocke_mma_scale_operand_t src_scales[3]; - for(int j = 0; j < 3; ++j) - src_scales[j] = {op->srcs[j].scale_dtype, op->srcs[j].scale_block_size}; - if(!rocke_mma_catalog_has_shape_indexed(&arch->mma, - op->family, - src_dtypes, - op->dst.dtype, - src_scales, - op->m, - op->n, - op->k) - || rocke_mma_catalog_op_for_shape_indexed(&arch->mma, - op->family, - src_dtypes, - op->dst.dtype, - src_scales, - op->m, - op->n, - op->k) + if(!rocke_mma_catalog_has_shape( + &arch->mma, op->family, a, b, op->c_dtype, op->m, op->n, op->k, scales) + || rocke_mma_catalog_op_for_shape(&arch->mma, + op->family, + a, + b, + op->c_dtype, + op->m, + op->n, + op->k, + scales) != op - || rocke_mma_catalog_select_largest_k_indexed(&arch->mma, - op->family, - src_dtypes, - op->dst.dtype, - src_scales, - op->m, - op->n, - op->k) + || rocke_mma_catalog_select_largest_k(&arch->mma, + op->family, + a, + b, + op->c_dtype, + op->m, + op->n, + op->k, + scales) != op - || rocke_mma_catalog_enumerate_indexed(&arch->mma, - op->family, - src_dtypes, - op->dst.dtype, - src_scales, - op->m, - op->n, - NULL, - 0) + || rocke_mma_catalog_enumerate(&arch->mma, + op->family, + a, + b, + op->c_dtype, + op->m, + op->n, + NULL, + 0, + scales) < 1) { fprintf(stderr, "%s: catalog query missed %s/%s\n", op->op_id, a, b); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_contracts.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_contracts.py deleted file mode 100644 index daf5b0f487a8..000000000000 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_contracts.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -# SPDX-License-Identifier: MIT -"""Selection must distinguish logical operands even when carriers agree.""" - -from dataclasses import replace - -import pytest - -from rocke.core.arch import ( - ArchTarget, - MmaCatalog, - MmaDst, - MmaOp, - MmaScaleOperand, - MmaSrc, -) -from rocke.core.arch.wmma_scale import gfx1250_scaled_wmma - - -def _contracts(): - base = MmaOp( - family="wmma_scaled", - srcs=( - MmaSrc("fp8", scale=MmaScaleOperand("e8m0", 32)), - MmaSrc("bf8", scale=MmaScaleOperand("e8m0", 32)), - MmaSrc("fp32"), - ), - dst=MmaDst("fp32"), - m=16, - n=16, - k=128, - op_id="fixture", - ) - rows = [base] - for index in (0, 1): - for scale in (MmaScaleOperand("e4m3", 32), MmaScaleOperand("e8m0", 16), None): - srcs = list(base.srcs) - srcs[index] = replace(srcs[index], scale=scale) - rows.append(replace(base, srcs=tuple(srcs))) - rows.append(replace(base, srcs=(*base.srcs[:2], MmaSrc("fp16")))) - rows.append(replace(base, dst=MmaDst("fp16"))) - rows.append( - replace(base, srcs=tuple(replace(src, scale=None) for src in base.srcs)) - ) - return [replace(row, op_id=row.semantic_id()) for row in rows] - - -def test_full_contract_queries_distinguish_each_source_scale_and_result(): - rows = _contracts() - assert len({row.op_id for row in rows}) == len(rows) - catalog = MmaCatalog(rows) - for row in rows: - query = dict( - family=row.family, - src_dtypes=tuple(src.dtype for src in row.srcs), - dst_dtype=row.dst.dtype, - src_scales=tuple(src.scale for src in row.srcs), - m=row.m, - n=row.n, - ) - assert catalog.enumerate(**query) == [row] - assert catalog.has_shape(**query, k=row.k) - assert catalog.op_for_shape(**query, k=row.k) is row - assert catalog.select_largest_k(**query) is row - assert catalog.select_largest_k(**query, k_max=64) is None - - -def test_partial_query_rejects_ambiguity_but_enumeration_and_existence_are_valid(): - catalog = MmaCatalog(_contracts()) - query = dict( - family="wmma_scaled", - src_dtypes=("fp8", "bf8", "fp32"), - dst_dtype="fp32", - m=16, - n=16, - ) - assert len(catalog.enumerate(**query)) == 8 - assert catalog.has_shape(**query, k=128) - with pytest.raises(ValueError, match="ambiguous MMA query"): - catalog.op_for_shape(**query, k=128) - with pytest.raises(ValueError, match="ambiguous MMA query"): - catalog.select_largest_k(**query) - assert len(catalog.enumerate(**query, src_scales=(None, None, None))) == 1 - scale = MmaScaleOperand("e5m3", 32) - assert catalog.op_for_shape(**query, src_scales=(scale, scale, None), k=128) is None - with pytest.raises(ValueError, match="exactly 3"): - catalog.enumerate(**query, src_scales=(None, None)) - - -def test_scale_alias_selects_the_same_contract(): - catalog = MmaCatalog(_contracts()) - query = dict( - family="wmma_scaled", - src_dtypes=("fp8", "bf8", "f32"), - dst_dtype="fp32", - m=16, - n=16, - k=128, - ) - scales = (MmaScaleOperand("fp8e4m3", 32), MmaScaleOperand("e8m0", 32), None) - row = catalog.op_for_shape(**query, src_scales=scales) - assert row is not None and row.srcs[0].scale.dtype == "e4m3" - - -def test_scaled_catalog_identity_and_backend_contract(): - catalog = ArchTarget.from_gfx("gfx1250").mma - rows = [row for row in catalog.ops if row.family == "wmma_scaled"] - assert len(rows) == 4 - for row in rows: - assert row.op_id == row.semantic_id() - packing = gfx1250_scaled_wmma(row.op_id) - assert packing.atom is row - assert packing.matrix_formats == ( - (0, 0) if row.srcs[0].dtype == "fp8e4m3" else (1, 1) - ) - assert packing.scales.count * packing.scales.block_k == row.k - assert packing.matrix_llvm_types == ("<16 x i32>", "<16 x i32>") - for family in ("wmma_scale", "wmma_scale16"): - old_id = f"{family}_f32_16x16x128_fp8_fp8" - assert catalog.by_op_id(old_id) is None - assert gfx1250_scaled_wmma(old_id) is None diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py index 3758d86ebf01..eeb6fa6b1bf4 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_frag_tables.py @@ -7,10 +7,10 @@ int-accumulator tables. ``IRBuilder.mma`` sizes a ``tile.mma`` result vector from a bare ``op_id`` string via three pieces: -* ``_mma_dst_frag_len`` — ``dst`` fragment length, resolved from the arch SSOT +* ``_mma_c_frag_len`` — accumulator fragment length, resolved from the arch SSOT ``core.arch.target._MMA_FRAGMENT_INFO`` (no ir-side copy); -* ``_mma_dst_is_int`` — whether the atom produces i32, resolved from the JSON - catalog ``dst`` dtype (``target._op_id_dst_dtype``); +* ``_mma_c_is_int`` — whether the atom accumulates in i32, resolved from the JSON + catalog accumulator dtype (``target._op_id_c_dtype``); * ``_MMA_RESULT_HINT`` — the ir-side SSA result-name hints (naming only, kept in ir.py to preserve byte-identical value numbering). @@ -26,20 +26,20 @@ import unittest -from rocke.core.arch.target import _MMA_FRAGMENT_INFO, _op_id_dst_dtype +from rocke.core.arch.target import _MMA_FRAGMENT_INFO, _op_id_c_dtype from rocke.core.ir import ( _MMA_RESULT_HINT, - _mma_dst_frag_len, - _mma_dst_is_int, + _mma_c_frag_len, + _mma_c_is_int, ) def _sizable_op_ids(): - """Op_ids the SSOT can actually size (positive ``dst`` frag length).""" + """Op_ids the SSOT can actually size (positive accumulator frag length).""" return { - op_id: info.dst.frag_len + op_id: info.c_frag_len for op_id, info in _MMA_FRAGMENT_INFO.items() - if info.dst.frag_len > 0 + if info.c_frag_len > 0 } @@ -50,27 +50,27 @@ def test_frag_lengths_are_positive(self): for op_id, frag in sizable.items(): self.assertIsInstance(frag, int) self.assertGreater( - frag, 0, msg=f"dst.frag_len for {op_id!r} must be positive" + frag, 0, msg=f"c_frag_len for {op_id!r} must be positive" ) def test_accessor_matches_ssot_and_raises_on_unknown(self): for op_id, frag in _sizable_op_ids().items(): - self.assertEqual(_mma_dst_frag_len(op_id), frag) + self.assertEqual(_mma_c_frag_len(op_id), frag) with self.assertRaises(ValueError): - _mma_dst_frag_len("not_a_real_op_id") + _mma_c_frag_len("not_a_real_op_id") def test_int_acc_op_ids_have_frag_lengths(self): - int_op_ids = [op for op, dtype in _op_id_dst_dtype().items() if dtype == "i32"] + int_op_ids = [op for op, dtype in _op_id_c_dtype().items() if dtype == "i32"] self.assertTrue( int_op_ids, "expected at least one i32-accumulator atom in the catalog" ) for op_id in int_op_ids: self.assertTrue( - _mma_dst_is_int(op_id), - msg=f"{op_id!r} is i32 in the catalog but _mma_dst_is_int disagrees", + _mma_c_is_int(op_id), + msg=f"{op_id!r} is i32 in the catalog but _mma_c_is_int disagrees", ) self.assertGreater( - _mma_dst_frag_len(op_id), + _mma_c_frag_len(op_id), 0, msg=f"int-accumulator op_id {op_id!r} has no SSOT frag length, so " f"IRBuilder.mma would raise for it", @@ -79,16 +79,16 @@ def test_int_acc_op_ids_have_frag_lengths(self): def test_result_hint_op_ids_have_frag_lengths(self): for op_id in _MMA_RESULT_HINT: self.assertGreater( - _mma_dst_frag_len(op_id), + _mma_c_frag_len(op_id), 0, msg=f"result-hint op_id {op_id!r} has no SSOT frag length", ) def test_known_frag_lengths(self): # Spot-check representative 16x16 vs 32x32 accumulator widths. - self.assertEqual(_mma_dst_frag_len("mfma_f32_16x16x16_f16"), 4) - self.assertEqual(_mma_dst_frag_len("mfma_f32_32x32x8_f16"), 16) - self.assertEqual(_mma_dst_frag_len("wmma_f32_16x16x16_f16"), 8) + self.assertEqual(_mma_c_frag_len("mfma_f32_16x16x16_f16"), 4) + self.assertEqual(_mma_c_frag_len("mfma_f32_32x32x8_f16"), 16) + self.assertEqual(_mma_c_frag_len("wmma_f32_16x16x16_f16"), 8) if __name__ == "__main__": # pragma: no cover diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_wmma_gfx12_acc_layout.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_wmma_gfx12_acc_layout.py index 4390380282fe..32283cd99c9f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_wmma_gfx12_acc_layout.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_wmma_gfx12_acc_layout.py @@ -1,13 +1,12 @@ # Copyright (c) Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT -"""Validation tests: gfx1201 and gfx1250 share the same ``dst`` fragment layout. +"""Validation tests: gfx1201 and gfx1250 share the same accumulator fragment layout. Three layers of validation: -1. **Metadata identity** — every gfx1250 WMMA op carries the same ``dst`` - fragment length, ``wave_size``, and coordinate function as its gfx1201 - counterparts. +1. **Metadata identity** — every gfx1250 WMMA op carries the same ``c_frag_len``, + ``wave_size``, and ``c_fn`` object as its gfx1201 counterparts. 2. **Coordinate exhaustion** — the coordinate function produces identical (row, col) pairs for all 256 (lane, slot) inputs on both sets of op_ids, using the same @@ -24,7 +23,7 @@ import unittest from typing import Callable, Tuple -from rocke.core.arch.target import _MMA_FRAGMENT_INFO, _wmma_gfx12_row_col_16x16 +from rocke.core.arch.target import _MMA_FRAGMENT_INFO, _wmma_gfx12_acc_16x16 # --------------------------------------------------------------------------- @@ -52,7 +51,7 @@ def mul(self, a: int, b: int) -> int: return a * b -def _eval_row_col(fn: Callable, lane: int, slot: int) -> Tuple[int, int]: +def _eval_acc(fn: Callable, lane: int, slot: int) -> Tuple[int, int]: b = _ConstBuilder() return fn(b, lane, slot) @@ -60,7 +59,7 @@ def _eval_row_col(fn: Callable, lane: int, slot: int) -> Tuple[int, int]: def _full_layout(fn: Callable, frag_len: int, wave_size: int): """Return the complete (lane, slot) -> (row, col) mapping as a dict.""" return { - (lane, slot): _eval_row_col(fn, lane, slot) + (lane, slot): _eval_acc(fn, lane, slot) for lane in range(wave_size) for slot in range(frag_len) } @@ -85,13 +84,13 @@ def _full_layout(fn: Callable, frag_len: int, wave_size: int): "wmma_gfx1250_f32_16x16x64_bf8_bf8", } -# ``dst`` params shared by both arches: 16x16 output tile, wave32, +# Accumulator params shared by both arches: 16x16 output tile, wave32, # <8 x float> per lane. -_EXPECTED_DST_FRAG_LEN = 8 +_EXPECTED_C_FRAG_LEN = 8 _EXPECTED_WAVE_SIZE = 32 -class TestWmmaGfx12DstLayoutCoverage(unittest.TestCase): +class TestWmmaGfx12AccLayoutCoverage(unittest.TestCase): """Guard that the op-id sets are non-empty and fully registered.""" def test_gfx1201_ops_in_ssot(self): @@ -111,15 +110,15 @@ def test_gfx1250_ops_in_ssot(self): ) -class TestWmmaGfx12DstFragMetadata(unittest.TestCase): - """gfx1201 and gfx1250 WMMA ops must carry identical ``dst`` metadata.""" +class TestWmmaGfx12AccFragMetadata(unittest.TestCase): + """gfx1201 and gfx1250 WMMA ops must carry identical accumulator metadata.""" - def _check_dst_metadata(self, op_id: str): + def _check_acc_metadata(self, op_id: str): info = _MMA_FRAGMENT_INFO[op_id] self.assertEqual( - info.dst.frag_len, - _EXPECTED_DST_FRAG_LEN, - msg=f"{op_id!r}: dst.frag_len {info.dst.frag_len} != {_EXPECTED_DST_FRAG_LEN}", + info.c_frag_len, + _EXPECTED_C_FRAG_LEN, + msg=f"{op_id!r}: c_frag_len {info.c_frag_len} != {_EXPECTED_C_FRAG_LEN}", ) self.assertEqual( info.wave_size, @@ -127,24 +126,24 @@ def _check_dst_metadata(self, op_id: str): msg=f"{op_id!r}: wave_size {info.wave_size} != {_EXPECTED_WAVE_SIZE}", ) self.assertIs( - info.dst.fn, - _wmma_gfx12_row_col_16x16, - msg=f"{op_id!r}: dst coordinate function has been forked " - "from the shared gfx12 map", + info.c_fn, + _wmma_gfx12_acc_16x16, + msg=f"{op_id!r}: c_fn is not _wmma_gfx12_acc_16x16 — " + f"accumulator layout has been forked from the shared gfx12 map", ) - def test_gfx1201_dst_metadata(self): + def test_gfx1201_acc_metadata(self): for op_id in _GFX1201_WMMA_OPS: with self.subTest(op_id=op_id): - self._check_dst_metadata(op_id) + self._check_acc_metadata(op_id) - def test_gfx1250_dst_metadata(self): + def test_gfx1250_acc_metadata(self): for op_id in _GFX1250_WMMA_OPS: with self.subTest(op_id=op_id): - self._check_dst_metadata(op_id) + self._check_acc_metadata(op_id) -class TestWmmaGfx12DstCoordinateFormula(unittest.TestCase): +class TestWmmaGfx12AccCoordinateFormula(unittest.TestCase): """The shared coordinate function must encode the expected formula for all inputs. Expected: slot ``i`` of lane ``l`` (0..31) maps to @@ -160,13 +159,13 @@ def _expected_coord(self, lane: int, slot: int) -> Tuple[int, int]: def test_coordinate_formula(self): builder = _ConstBuilder() for lane in range(_EXPECTED_WAVE_SIZE): - for slot in range(_EXPECTED_DST_FRAG_LEN): - got = _wmma_gfx12_row_col_16x16(builder, lane, slot) + for slot in range(_EXPECTED_C_FRAG_LEN): + got = _wmma_gfx12_acc_16x16(builder, lane, slot) want = self._expected_coord(lane, slot) self.assertEqual( got, want, - msg=f"_wmma_gfx12_row_col_16x16(lane={lane}, slot={slot}): " + msg=f"_wmma_gfx12_acc_16x16(lane={lane}, slot={slot}): " f"got {got}, expected {want}", ) @@ -175,20 +174,20 @@ def test_output_tile_is_fully_covered(self): coords = { self._expected_coord(lane, slot) for lane in range(_EXPECTED_WAVE_SIZE) - for slot in range(_EXPECTED_DST_FRAG_LEN) + for slot in range(_EXPECTED_C_FRAG_LEN) } expected = {(r, c) for r in range(16) for c in range(16)} self.assertEqual( coords, expected, - msg="dst layout does not cover the full 16x16 output tile", + msg="Accumulator layout does not cover the full 16x16 output tile", ) def test_no_coordinate_collisions(self): """Each (lane, slot) pair must map to a unique matrix element.""" seen: dict = {} for lane in range(_EXPECTED_WAVE_SIZE): - for slot in range(_EXPECTED_DST_FRAG_LEN): + for slot in range(_EXPECTED_C_FRAG_LEN): coord = self._expected_coord(lane, slot) if coord in seen: prev_lane, prev_slot = seen[coord] @@ -200,15 +199,15 @@ def test_no_coordinate_collisions(self): seen[coord] = (lane, slot) -class TestWmmaGfx12DstLayoutIdentical(unittest.TestCase): +class TestWmmaGfx12AccLayoutIdentical(unittest.TestCase): """The concrete (lane,slot)->(row,col) mapping must be bit-for-bit identical - across every gfx1201 op and every gfx1250 op that shares the gfx12 map.""" + across every gfx1201 op and every gfx1250 op that shares ``lm_wmma_gfx12_c``.""" def _layout_for(self, op_id: str) -> dict: info = _MMA_FRAGMENT_INFO[op_id] - if info.dst.fn is None: - self.skipTest(f"{op_id!r} has no verified dst lane map") - return _full_layout(info.dst.fn, info.dst.frag_len, info.wave_size) + if info.c_fn is None: + self.skipTest(f"{op_id!r} has no verified accumulator lane map (c_fn=None)") + return _full_layout(info.c_fn, info.c_frag_len, info.wave_size) def test_gfx1250_layout_matches_gfx1201_reference(self): # Use wmma_gfx12_f32_16x16x16_f16 as the canonical gfx1201 reference. @@ -218,15 +217,15 @@ def test_gfx1250_layout_matches_gfx1201_reference(self): for op_id in _GFX1250_WMMA_OPS: with self.subTest(op_id=op_id): info = _MMA_FRAGMENT_INFO[op_id] - if info.dst.fn is None: + if info.c_fn is None: # op has no verified map; identity is enforced via the - # metadata test (same dst metadata object values). + # metadata test (same c_frag_len / wave_size / c_fn object). continue - candidate = _full_layout(info.dst.fn, info.dst.frag_len, info.wave_size) + candidate = _full_layout(info.c_fn, info.c_frag_len, info.wave_size) self.assertEqual( candidate, ref_layout, - msg=f"{op_id!r} dst layout differs from {reference_op!r}", + msg=f"{op_id!r} accumulator layout differs from {reference_op!r}", ) def test_gfx1201_ops_have_identical_layouts(self): @@ -236,7 +235,7 @@ def test_gfx1201_ops_have_identical_layouts(self): self.assertEqual( layouts[ops[i]], layouts[ops[0]], - msg=f"gfx1201 dst layout mismatch: {ops[i]!r} vs {ops[0]!r}", + msg=f"gfx1201 accumulator layout mismatch: {ops[i]!r} vs {ops[0]!r}", ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c index e86befe8ca25..f532e7bff623 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c @@ -12,6 +12,7 @@ #include #include +#include "rocke/arch_target.h" #include "rocke/ir.h" #include "rocke/ir_serialize.h" #include "rocke/lower_llvm.h" @@ -119,8 +120,14 @@ static void build_wmma_k64_bf8_bf8(rocke_ir_builder_t* b) /* K=128 FP8 SCALE/SCALE16 WMMA. Matrix fragments are <16 x i32>; packed * E8M0 scale operands are i32 for SCALE and i64 for SCALE16. */ -static void wmma_scaled(rocke_ir_builder_t* b, bool scale16, const char* op_id) +static void wmma_scaled(rocke_ir_builder_t* b, bool scale16, const char* dtype) { + const int block = scale16 ? 16 : 32; + const rocke_mma_scale_operand_t query[2] = {{"e8m0", block}, {"e8m0", block}}; + const rocke_arch_target_t* target = rocke_arch_target_from_gfx("gfx1250"); + const rocke_mma_op_t* atom = rocke_mma_catalog_op_for_shape( + &target->mma, "wmma_scaled", dtype, dtype, "fp32", 16, 16, 128, query); + const char* op_id = atom->op_id; const rocke_type_t* scale_ty = scale16 ? rocke_i64() : rocke_i32(); rocke_value_t* a_ptr = frag_param(b, "A", rocke_i32(), true); rocke_value_t* b_ptr = frag_param(b, "B", rocke_i32(), true); @@ -145,34 +152,22 @@ static void wmma_scaled(rocke_ir_builder_t* b, bool scale16, const char* op_id) static void build_wmma_scale(rocke_ir_builder_t* b) { - wmma_scaled( - b, - false, - "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32"); + wmma_scaled(b, false, "fp8e4m3"); } static void build_wmma_scale_bf8(rocke_ir_builder_t* b) { - wmma_scaled( - b, - false, - "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32"); + wmma_scaled(b, false, "bf8e5m2"); } static void build_wmma_scale16(rocke_ir_builder_t* b) { - wmma_scaled( - b, - true, - "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32"); + wmma_scaled(b, true, "fp8e4m3"); } static void build_wmma_scale16_bf8(rocke_ir_builder_t* b) { - wmma_scaled( - b, - true, - "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32"); + wmma_scaled(b, true, "bf8e5m2"); } /* ds_read_b128_tr_b16. gfx950 has one type-agnostic opcode returning diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py index 0e02e99e270b..a9ec2a40f2b9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py @@ -81,9 +81,10 @@ def _wmma_scaled(a_kind, b_kind, scale_mode): scale = MmaScaleOperand("e8m0", 16 if scale_mode == "scale16" else 32) atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( family="wmma_scaled", - src_dtypes=(a_kind, b_kind, "fp32"), - dst_dtype="fp32", - src_scales=(scale, scale, None), + a_dtype=a_kind, + b_dtype=b_kind, + c_dtype="fp32", + scales=(scale, scale), m=16, n=16, k=128, From cf3616de7e212a161c0f9274dfd2b79ea46afbaa Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:09:03 -0400 Subject: [PATCH 07/15] refactor(rocke): share scaled WMMA K-group metadata --- .../rocke/platform/cpp/core/arch/data.cpp | 36 +++--- .../rocke/platform/cpp/core/arch/query.cpp | 86 ++++--------- .../rocke/platform/cpp/core/ir/ir_tile.cpp | 4 +- .../platform/cpp/include/rocke/arch_target.h | 96 ++++++++------- .../cpp/include/rocke/wmma_scale_internal.h | 11 +- .../dsl_docs/architecture/authoring_model.md | 38 +++--- .../python/rocke/core/arch/__init__.py | 4 +- .../rocke/core/arch/data/arch_specs.json | 8 +- .../platform/python/rocke/core/arch/target.py | 111 ++++++++--------- .../python/rocke/core/arch/wmma_scale.py | 10 +- .../rocke/platform/python/rocke/core/ir.py | 5 +- .../platform/python/rocke/core/lower_hip.py | 5 +- .../rocke/examples/gfx1250/gemm/README.md | 2 +- .../instances/gfx1250/block_scaled_gemm.py | 8 +- .../tests/core/future_intrinsic_lowering.cpp | 6 +- .../platform/tests/core/mma_frag_ssot.cpp | 10 +- .../platform/tests/core/test_arch_mma_ssot.py | 114 ++++++++++++++---- .../tests/core/test_gfx1250_scaled_wmma.py | 28 +++-- .../tests/core/test_lowbit_dtype_aliases.py | 2 +- .../tests/core/test_mma_catalog_lookup.cpp | 107 +++++++++------- .../rocke_representative_ir_sha256.json | 16 +-- .../instances/parity/gfx1250_lowering_emit.c | 6 +- .../instances/parity/gfx1250_lowering_emit.py | 6 +- 23 files changed, 398 insertions(+), 321 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp index 81e1465cbe6f..a71c5c20dac9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp @@ -887,10 +887,10 @@ static const rocke_ati_mma_frag_row_t rocke_ati_mma_frag[] = { {"wmma_gfx1250_f32_16x16x64_fp8_bf8", 8}, {"wmma_gfx1250_f32_16x16x64_bf8_fp8", 8}, {"wmma_gfx1250_f32_16x16x64_bf8_bf8", 8}, - {"wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32", 8}, - {"wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32", 8}, - {"wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32", 8}, - {"wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k32", 8}, + {"wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16", 8}, + {"wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16", 8}, }; int rocke_arch_mma_c_frag_len(const char* op_id) @@ -1679,7 +1679,7 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { 16, 16, 128, - "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32", + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32", 16, 16, 8, @@ -1687,8 +1687,9 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { NULL, NULL, &lm_wmma_gfx12_c, - {"e8m0", 32}, - {"e8m0", 32}}, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32}, {"wmma_scaled", "bf8e5m2", "bf8e5m2", @@ -1696,7 +1697,7 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { 16, 16, 128, - "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32", + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k32", 16, 16, 8, @@ -1704,8 +1705,9 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { NULL, NULL, &lm_wmma_gfx12_c, - {"e8m0", 32}, - {"e8m0", 32}}, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K32}, {"wmma_scaled", "fp8e4m3", "fp8e4m3", @@ -1713,7 +1715,7 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { 16, 16, 128, - "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32", + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16", 16, 16, 8, @@ -1721,8 +1723,9 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { NULL, NULL, &lm_wmma_gfx12_c, - {"e8m0", 16}, - {"e8m0", 16}}, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16}, {"wmma_scaled", "bf8e5m2", "bf8e5m2", @@ -1730,7 +1733,7 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { 16, 16, 128, - "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32", + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16", 16, 16, 8, @@ -1738,8 +1741,9 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { NULL, NULL, &lm_wmma_gfx12_c, - {"e8m0", 16}, - {"e8m0", 16}}, + "e8m0", + "e8m0", + ROCKE_MMA_SCALE_K16}, }; /* ========================================================================= diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp index 3d7fda13359c..faa1fe52fd9c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp @@ -53,82 +53,44 @@ static const char* rocke_ati_scale_dtype(const char* dtype) return dtype && strcmp(dtype, "fp8e4m3") == 0 ? "e4m3" : dtype; } -static void rocke_ati_validate_scales(const rocke_mma_scale_operand_t* scales) +static void rocke_ati_validate_scales(const rocke_mma_scale_filter_t* scales) { if(!scales) return; + if(!scales->a_scale_dtype && !scales->b_scale_dtype && scales->scale_block_k == 0) + return; + const char* dtypes[2] = {scales->a_scale_dtype, scales->b_scale_dtype}; for(int i = 0; i < 2; ++i) { - const char* dtype = rocke_ati_scale_dtype(scales[i].dtype); - if(dtype && strcmp(dtype, "e8m0") != 0 && strcmp(dtype, "e4m3") != 0 - && strcmp(dtype, "e5m3") != 0) + const char* dtype = rocke_ati_scale_dtype(dtypes[i]); + if(!dtype + || (strcmp(dtype, "e8m0") != 0 && strcmp(dtype, "e4m3") != 0 + && strcmp(dtype, "e5m3") != 0)) ckc::raise_status(ROCKE_ERR_VALUE, "MMA scale dtype must be e8m0, e4m3, or e5m3"); - if((dtype && scales[i].block_size != 16 && scales[i].block_size != 32) - || (!dtype && scales[i].block_size != 0)) - ckc::raise_status(ROCKE_ERR_VALUE, - "MMA scale block_size must be an integer equal to 16 or 32"); } + if(scales->scale_block_k != 16 && scales->scale_block_k != 32) + ckc::raise_status(ROCKE_ERR_VALUE, + "MMA scale_block_k must be an integer equal to 16 or 32"); } -static bool rocke_ati_scales_match(const rocke_mma_op_t* op, - const rocke_mma_scale_operand_t* scales) +static bool rocke_ati_scales_match(const rocke_mma_op_t* op, const rocke_mma_scale_filter_t* scales) { if(!scales) return true; - const rocke_mma_scale_operand_t actual[2] = {op->a_scale, op->b_scale}; + if(scales->scale_block_k != op->scale_block_k) + return false; + const char* actual[2] = {op->a_scale_dtype, op->b_scale_dtype}; + const char* queried[2] = {scales->a_scale_dtype, scales->b_scale_dtype}; for(int i = 0; i < 2; ++i) { - const char* expected = rocke_ati_scale_dtype(scales[i].dtype); - const char* dtype = rocke_ati_scale_dtype(actual[i].dtype); - if(bool(expected) != bool(dtype) || (dtype && strcmp(expected, dtype) != 0) - || scales[i].block_size != actual[i].block_size) + const char* expected = rocke_ati_scale_dtype(queried[i]); + const char* dtype = rocke_ati_scale_dtype(actual[i]); + if(bool(expected) != bool(dtype) || (dtype && strcmp(expected, dtype) != 0)) return false; } return true; } -const char* rocke_mma_op_semantic_id(const rocke_mma_op_t* op, char* out, size_t cap) -{ - if(!op || !out || !cap) - return NULL; - const rocke_mma_scale_operand_t scales[2] = {op->a_scale, op->b_scale}; - rocke_ati_validate_scales(scales); - int n = snprintf(out, cap, "%s.%dx%dx%d", op->family, op->m, op->n, op->k); - if(n < 0 || (size_t)n >= cap) - return NULL; - for(size_t i = 0; i < strlen(op->family); ++i) - if(out[i] == '_') - out[i] = '.'; - size_t used = (size_t)n; - const char* dtypes[2] = {op->a_dtype, op->b_dtype}; - char buf[64]; - for(int i = 0; i < 2; ++i) - { - n = snprintf(out + used, - cap - used, - ".src%d_%s", - i, - rocke_normalize_dtype(dtypes[i], buf, sizeof(buf))); - if(n < 0 || (size_t)n >= cap - used) - return NULL; - used += (size_t)n; - if(scales[i].dtype) - { - n = snprintf(out + used, - cap - used, - "_%s_b%d", - rocke_ati_scale_dtype(scales[i].dtype), - scales[i].block_size); - if(n < 0 || (size_t)n >= cap - used) - return NULL; - used += (size_t)n; - } - } - const char* acc = rocke_normalize_dtype(op->c_dtype, buf, sizeof(buf)); - n = snprintf(out + used, cap - used, ".src2_%s.dst_%s", acc, acc); - return n < 0 || (size_t)n >= cap - used ? NULL : out; -} - /* ============================== MMA atom getters ====================== */ void rocke_mma_op_shape(const rocke_mma_op_t* op, int* m, int* n, int* k) @@ -221,7 +183,7 @@ static bool rocke_ati_op_matches(const rocke_mma_op_t* op, const char* c, int m, int n, - const rocke_mma_scale_operand_t* scales) + const rocke_mma_scale_filter_t* scales) { if(strcmp(op->family, family) != 0) return false; @@ -247,7 +209,7 @@ int rocke_mma_catalog_enumerate(const rocke_mma_catalog_t* cat, int n, const rocke_mma_op_t** out, int cap, - const rocke_mma_scale_operand_t scales[2]) + const rocke_mma_scale_filter_t* scales) { char abuf[64], bbuf[64], cbuf[64]; const char *a, *bd, *c, *fam; @@ -281,7 +243,7 @@ bool rocke_mma_catalog_has_shape(const rocke_mma_catalog_t* cat, int m, int n, int k, - const rocke_mma_scale_operand_t scales[2]) + const rocke_mma_scale_filter_t* scales) { char abuf[64], bbuf[64], cbuf[64]; const char *a, *bd, *c, *fam; @@ -315,7 +277,7 @@ const rocke_mma_op_t* rocke_mma_catalog_select_largest_k(const rocke_mma_catalog int m, int n, int k_max, - const rocke_mma_scale_operand_t scales[2]) + const rocke_mma_scale_filter_t* scales) { char abuf[64], bbuf[64], cbuf[64]; const char *a, *bd, *c, *fam; @@ -374,7 +336,7 @@ const rocke_mma_op_t* rocke_mma_catalog_op_for_shape(const rocke_mma_catalog_t* int m, int n, int k, - const rocke_mma_scale_operand_t scales[2]) + const rocke_mma_scale_filter_t* scales) { char abuf[64], bbuf[64], cbuf[64]; const char *a, *bd, *c, *fam; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp index a77f7656c774..384430fb9ff0 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp @@ -69,7 +69,9 @@ static bool rocke_mma_is_int_acc(const char* op_id) static const char* rocke_mma_result_hint(const char* op_id) { - if(op_id && strncmp(op_id, "wmma.scaled.", 12) == 0) + const rocke_arch_target_t* target = rocke_arch_target_from_gfx("gfx1250"); + const rocke_mma_op_t* atom = op_id ? rocke_mma_catalog_by_op_id(&target->mma, op_id) : NULL; + if(atom && strcmp(atom->family, "wmma_scaled") == 0) return "mxacc"; size_t i; if(op_id) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h index b8fd90966070..1c965ae2b045 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h @@ -103,18 +103,27 @@ bool rocke_layout_map_coord(const rocke_layout_map_t* m, /* ============================== MMA atom =============================== */ -/* Logical scale format and source elements along K per scale. - * {NULL, 0} means unscaled. */ -typedef struct rocke_mma_scale_operand +/* Shared K-group size. NONE is only the unscaled sentinel. */ +typedef enum rocke_mma_scale_block_k { - const char* dtype; - int block_size; -} rocke_mma_scale_operand_t; + ROCKE_MMA_SCALE_NONE = 0, + ROCKE_MMA_SCALE_K16 = 16, + ROCKE_MMA_SCALE_K32 = 32 +} rocke_mma_scale_block_k_t; + +/* Complete scale query: independent A/B types, shared K-group size. + * {NULL, NULL, 0} selects unscaled atoms. */ +typedef struct rocke_mma_scale_filter +{ + const char* a_scale_dtype; + const char* b_scale_dtype; + rocke_mma_scale_block_k_t scale_block_k; +} rocke_mma_scale_filter_t; /* A supported matrix-multiply-accumulate atom. Frozen, in static storage. * Mirrors MmaOp: family is "mma", "wmma", or "wmma_scaled"; matrix dtypes * are canonical catalog keys. C describes both accumulator input and result. - * op_id is an opaque backend handle; scaled WMMA uses semantic contract IDs. + * op_id is an opaque backend handle; lowering reads metadata, not its spelling. * Layout pointers are NULL when no verified lane map is registered. */ typedef struct rocke_mma_op { @@ -133,13 +142,11 @@ typedef struct rocke_mma_op const rocke_layout_map_t* a_layout; /* may be NULL */ const rocke_layout_map_t* b_layout; /* may be NULL */ const rocke_layout_map_t* c_layout; /* may be NULL */ - rocke_mma_scale_operand_t a_scale; - rocke_mma_scale_operand_t b_scale; + const char* a_scale_dtype; /* NULL when unscaled */ + const char* b_scale_dtype; /* NULL when unscaled */ + rocke_mma_scale_block_k_t scale_block_k; /* shared by A/B; NONE when unscaled */ } rocke_mma_op_t; -/* Write the semantic contract ID into caller-owned out; NULL on truncation. */ -const char* rocke_mma_op_semantic_id(const rocke_mma_op_t* op, char* out, size_t cap); - /* MmaOp.shape -> (m, n, k) via out params. */ void rocke_mma_op_shape(const rocke_mma_op_t* op, int* m, int* n, int* k); @@ -184,8 +191,8 @@ typedef struct rocke_mma_catalog /* MmaCatalog.ops accessor (count + pointer). */ const rocke_mma_op_t* rocke_mma_catalog_ops(const rocke_mma_catalog_t* cat, int* num_out); -/* Optional A/B scale pair: NULL leaves scales unconstrained; {NULL, 0} - * selects an unscaled input. C callers supply the pair explicitly. +/* Optional scale filter: NULL leaves scales unconstrained; {NULL, NULL, 0} + * selects unscaled atoms. C callers supply the filter pointer explicitly. * Exact selection and largest-K ties raise a query error on ambiguity. * As with layout queries, C++ callers catch ckc::Error at their entry boundary. */ #ifdef __cplusplus @@ -208,48 +215,47 @@ int rocke_mma_catalog_enumerate(const rocke_mma_catalog_t* cat, int n, const rocke_mma_op_t** out, int cap, - const rocke_mma_scale_operand_t scales[2] ROCKE_MMA_DEFAULT_SCALES); + const rocke_mma_scale_filter_t* scales ROCKE_MMA_DEFAULT_SCALES); /* MmaCatalog.has_shape. family NULL => "mma". */ -bool rocke_mma_catalog_has_shape( - const rocke_mma_catalog_t* cat, - const char* family, - const char* a_dtype, - const char* b_dtype, - const char* c_dtype, - int m, - int n, - int k, - const rocke_mma_scale_operand_t scales[2] ROCKE_MMA_DEFAULT_SCALES); +bool rocke_mma_catalog_has_shape(const rocke_mma_catalog_t* cat, + const char* family, + const char* a_dtype, + const char* b_dtype, + const char* c_dtype, + int m, + int n, + int k, + const rocke_mma_scale_filter_t* scales ROCKE_MMA_DEFAULT_SCALES); /* MmaCatalog.select_largest_k: the matching atom with the largest k (k <= k_max * if k_max >= 0; pass k_max<0 for Python None). Returns NULL if none match. */ -const rocke_mma_op_t* rocke_mma_catalog_select_largest_k( - const rocke_mma_catalog_t* cat, - const char* family, - const char* a_dtype, - const char* b_dtype, - const char* c_dtype, - int m, - int n, - int k_max, - const rocke_mma_scale_operand_t scales[2] ROCKE_MMA_DEFAULT_SCALES); +const rocke_mma_op_t* rocke_mma_catalog_select_largest_k(const rocke_mma_catalog_t* cat, + const char* family, + const char* a_dtype, + const char* b_dtype, + const char* c_dtype, + int m, + int n, + int k_max, + const rocke_mma_scale_filter_t* scales + ROCKE_MMA_DEFAULT_SCALES); /* MmaCatalog.by_op_id. Returns NULL if absent. */ const rocke_mma_op_t* rocke_mma_catalog_by_op_id(const rocke_mma_catalog_t* cat, const char* op_id); /* MmaCatalog.op_for_shape: the atom with exactly (m, n, k). NULL if absent. * family NULL => "mma". */ -const rocke_mma_op_t* rocke_mma_catalog_op_for_shape( - const rocke_mma_catalog_t* cat, - const char* family, - const char* a_dtype, - const char* b_dtype, - const char* c_dtype, - int m, - int n, - int k, - const rocke_mma_scale_operand_t scales[2] ROCKE_MMA_DEFAULT_SCALES); +const rocke_mma_op_t* + rocke_mma_catalog_op_for_shape(const rocke_mma_catalog_t* cat, + const char* family, + const char* a_dtype, + const char* b_dtype, + const char* c_dtype, + int m, + int n, + int k, + const rocke_mma_scale_filter_t* scales ROCKE_MMA_DEFAULT_SCALES); /* ===================== bare-op_id SSOT lookups ======================== */ /* These resolve a bare op_id string (no MmaOp / ArchTarget in hand) against the diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h index 8d3be8d5ff1d..01041b53fa24 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h @@ -50,7 +50,7 @@ static inline rocke_scaled_wmma_op_t rocke_scaled_wmma_contract(const rocke_mma_ spec.op_id = atom->op_id; const char* dtypes[2] = {atom->a_dtype, atom->b_dtype}; const int words[2] = {atom->a_frag_len, atom->b_frag_len}; - const rocke_mma_scale_operand_t scales[2] = {atom->a_scale, atom->b_scale}; + const char* scale_dtypes[2] = {atom->a_scale_dtype, atom->b_scale_dtype}; for(int i = 0; i < 2; ++i) { if(strcmp(dtypes[i], "fp8e4m3") == 0) @@ -59,16 +59,15 @@ static inline rocke_scaled_wmma_op_t rocke_scaled_wmma_contract(const rocke_mma_ spec.matrix_formats[i] = 1; else ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA matrix format"); - if(!scales[i].dtype || strcmp(scales[i].dtype, "e8m0") != 0 - || (scales[i].block_size != 16 && scales[i].block_size != 32)) + if(!scale_dtypes[i] || strcmp(scale_dtypes[i], "e8m0") != 0) ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA scale format"); spec.scale_formats[i] = 0; // E8M0. spec.matrix_words[i] = words[i]; } - if(atom->a_scale.block_size != atom->b_scale.block_size || strcmp(atom->c_dtype, "fp32") != 0 - || atom->m != 16 || atom->n != 16 || atom->k != 128) + if((atom->scale_block_k != 16 && atom->scale_block_k != 32) + || strcmp(atom->c_dtype, "fp32") != 0 || atom->m != 16 || atom->n != 16 || atom->k != 128) ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA backend contract"); - spec.scales.block_k = atom->a_scale.block_size; + spec.scales.block_k = atom->scale_block_k; spec.scales.count = atom->k / spec.scales.block_k; char suffix[96]; snprintf(suffix, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md index acfb38ef041c..3c328f9c084e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md @@ -32,43 +32,49 @@ and validate the operation's wave and layout contract. The exact catalog is listed in [`kernel_taxonomy.md`](./kernel_taxonomy.md). MMA metadata uses `a_dtype`, `b_dtype`, and `c_dtype`, with C describing both -the accumulator input and result. Optional `a_scale` and `b_scale` descriptors -(`MmaScaleOperand(dtype, block_size)`) describe scale values independently of -the matrix dtype and packed register type. Fragment and layout accessors -retain their A/B/C roles. +the accumulator input and result. Optional `a_scale_dtype`, `b_scale_dtype`, +and `scale_block_k` fields describe independent scale types and a shared K-group +size. `MmaScaleBlockK.K16` and `MmaScaleBlockK.K32` are the only scaled sizes; +all three fields are `None` for unscaled atoms. Integer 16/32 values from JSON +are normalized to the enum. Scale types are independent of matrix dtypes and +packed register types. Fragment and layout accessors retain their A/B/C roles. For scaled operations, query the complete contract and pass the selected atom to `IRBuilder.mma`: ```python -from rocke.core.arch import ArchTarget, MmaScaleOperand +from rocke.core.arch import ArchTarget, MmaScaleBlockK -scale = MmaScaleOperand("e8m0", 32) atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( family="wmma_scaled", a_dtype="fp8", b_dtype="fp8", c_dtype="fp32", - scales=(scale, scale), + scales=("e8m0", "e8m0", MmaScaleBlockK.K32), m=16, n=16, k=128, ) assert atom is not None # result = builder.mma(atom, a, b, c, scale_a, scale_b) ``` -Omitting `scales` leaves scales unconstrained. Passing an A/B pair matches both -inputs exactly; `None` in that pair means unscaled. Enumeration +Omitting `scales` leaves scales unconstrained. Passing `(a_type, b_type, block_k)` +matches the scale contract exactly; `(None, None, None)` selects unscaled atoms. +Partially specified scale contracts are invalid. Enumeration and existence queries may match several records, while `op_for_shape` and `select_largest_k` reject ambiguous exact matches or largest-K ties. The C queries -use the same rules: a NULL scale array is unconstrained, and `{NULL, 0}` requests -an unscaled source. Scale value formats are `e8m0`, `e4m3`, and `e5m3`, with +use the same rules: a NULL `rocke_mma_scale_filter_t` pointer is unconstrained, +and `{NULL, NULL, ROCKE_MMA_SCALE_NONE}` requests an unscaled atom. The C block +enum has `ROCKE_MMA_SCALE_K16` and `ROCKE_MMA_SCALE_K32`, plus the unscaled +sentinel. Scale value formats are `e8m0`, `e4m3`, and `e5m3`, with `fp8e4m3` accepted as an alias for `e4m3`; actual target support comes from the catalog. Scaled-WMMA IDs have the form -`wmma.scaled..src0___b.src1___b.src2_.dst_`. -They describe a semantic contract independently of LLVM intrinsic names and -packed carriers. The `src2` and `dst` fields both use the shared C dtype; -they do not imply independent result-type selection. Existing `wmma_scale*_f32_*` IDs and their dedicated builder -wrappers are retired; serialized IR using those IDs must be regenerated. +`wmma______scale___k`. +For example, `wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32` +has atom K=128 and one scale per 32 K elements for both inputs. Both scale +types are written even when equal. Lowering reads the catalog fields, never +parses the ID, and selects LLVM intrinsic names and packed carriers separately. +Existing `wmma_scale*_f32_*` and dotted `wmma.scaled.*` IDs are retired, as are +the dedicated scaled builder wrappers; serialized IR using those IDs must be regenerated. Use `tile.mma` with a resolved catalog atom. Other MMA operation IDs retain their existing spelling. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py index fdada1852122..6b1576f5163c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py @@ -14,7 +14,7 @@ MemoryCapabilities, MmaCatalog, MmaOp, - MmaScaleOperand, + MmaScaleBlockK, ResourceLimits, arch_from_isa, base_arch_from_target_id, @@ -30,7 +30,7 @@ "MemoryCapabilities", "MmaCatalog", "MmaOp", - "MmaScaleOperand", + "MmaScaleBlockK", "ResourceLimits", "arch_from_isa", "base_arch_from_target_id", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json index 1663e919166d..674428134f06 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json @@ -177,10 +177,10 @@ {"family": "wmma", "a": "fp8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_fp8_bf8"}, {"family": "wmma", "a": "bf8", "b": "fp8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_fp8"}, {"family": "wmma", "a": "bf8", "b": "bf8", "c": "fp32", "m": 16, "n": 16, "k": 64, "op_id": "wmma_gfx1250_f32_16x16x64_bf8_bf8"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale": {"dtype": "e8m0", "block_size": 32}, "b_scale": {"dtype": "e8m0", "block_size": 32}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale": {"dtype": "e8m0", "block_size": 32}, "b_scale": {"dtype": "e8m0", "block_size": 32}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32"}, - {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale": {"dtype": "e8m0", "block_size": 16}, "b_scale": {"dtype": "e8m0", "block_size": 16}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32"}, - {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale": {"dtype": "e8m0", "block_size": 16}, "b_scale": {"dtype": "e8m0", "block_size": 16}, "m": 16, "n": 16, "k": 128, "op_id": "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32"} + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 32, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k32"}, + {"family": "wmma_scaled", "a": "fp8", "b": "fp8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16"}, + {"family": "wmma_scaled", "a": "bf8", "b": "bf8", "c": "fp32", "a_scale_dtype": "e8m0", "b_scale_dtype": "e8m0", "scale_block_k": 16, "m": 16, "n": 16, "k": 128, "op_id": "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16"} ] }, "gfx11-generic": { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py index 0ff4916ea99d..f47dc724c5ee 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py @@ -20,6 +20,7 @@ import json import re from dataclasses import dataclass, field +from enum import IntEnum from functools import lru_cache from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -98,38 +99,35 @@ def coord(self, builder: Any, lane: Any, slot: int) -> Tuple[Any, Any]: return self.fn(builder, lane, slot) -@dataclass(frozen=True) -class MmaScaleOperand: - """Scale value format and granularity for one matrix source. - - ``dtype`` is one of ``e8m0``, ``e4m3``, or ``e5m3``; ``fp8e4m3`` is an - alias for ``e4m3``. It describes the scale values independently of the - source dtype and the backend's packed register carrier. ``block_size`` - is the number of source elements along K sharing one scale value. Only - 16 and 32 are valid: SCALE and SCALE16 use 32 and 16, respectively. - """ +class MmaScaleBlockK(IntEnum): + """Number of K elements sharing one scale, common to both matrix inputs.""" - dtype: str - block_size: int + K16 = 16 + K32 = 32 - def __post_init__(self) -> None: - if self.dtype == "fp8e4m3": - object.__setattr__(self, "dtype", "e4m3") - if self.dtype not in ("e8m0", "e4m3", "e5m3"): - raise ValueError("MMA scale dtype must be e8m0, e4m3, or e5m3") - if type(self.block_size) is not int or self.block_size not in (16, 32): - raise ValueError( - "MMA scale block_size must be an integer equal to 16 or 32" - ) + +def _normalize_mma_scales( + a_dtype: str | None, b_dtype: str | None, block_k: int | None +) -> tuple[str | None, str | None, MmaScaleBlockK | None]: + """Validate the complete scale contract, independently of backend support.""" + if a_dtype is None and b_dtype is None and block_k is None: + return None, None, None + a = "e4m3" if a_dtype == "fp8e4m3" else a_dtype + b = "e4m3" if b_dtype == "fp8e4m3" else b_dtype + if a not in ("e8m0", "e4m3", "e5m3") or b not in ("e8m0", "e4m3", "e5m3"): + raise ValueError("MMA scale dtype must be e8m0, e4m3, or e5m3") + if type(block_k) not in (int, MmaScaleBlockK) or block_k not in (16, 32): + raise ValueError("MMA scale_block_k must be an integer equal to 16 or 32") + return a, b, MmaScaleBlockK(block_k) @dataclass(frozen=True) class MmaOp: """A single supported matrix-multiply-accumulate atom on a target. - ``op_id`` identifies a concrete operand contract. Scaled WMMA uses semantic - IDs including matrix dtypes and per-input scales. The backend maps this - identity to an intrinsic and format selectors. + ``op_id`` identifies a concrete operand contract. Scaled WMMA IDs extend + the target/accumulator/shape/input convention with A/B scale types and a + shared K-group size. The backend reads metadata, not the ID spelling. The ``*_frag_len`` fields and the layout-map accessors describe the *physical* register fragmentation of the atom: how many values of each @@ -160,21 +158,19 @@ class MmaOp: _b_layout: Optional[LayoutMap] = field(default=None, repr=False, compare=False) _c_layout: Optional[LayoutMap] = field(default=None, repr=False, compare=False) - a_scale: MmaScaleOperand | None = None - b_scale: MmaScaleOperand | None = None - - def semantic_id(self) -> str: - """Logical identity; C describes both accumulator input and result.""" - fields = [self.family.replace("_", "."), f"{self.m}x{self.n}x{self.k}"] - for i, (dtype, scale) in enumerate( - ((self.a_dtype, self.a_scale), (self.b_dtype, self.b_scale)) - ): - field = f"src{i}_{normalize_dtype(dtype)}" - if scale is not None: - field += f"_{scale.dtype}_b{scale.block_size}" - fields.append(field) - acc = normalize_dtype(self.c_dtype) - return ".".join(fields + [f"src2_{acc}", f"dst_{acc}"]) + # All absent means unscaled. The K-group size applies to both inputs; + # scale types describe values, not the backend's packed register carrier. + a_scale_dtype: str | None = None + b_scale_dtype: str | None = None + scale_block_k: MmaScaleBlockK | None = None + + def __post_init__(self) -> None: + a, b, block_k = _normalize_mma_scales( + self.a_scale_dtype, self.b_scale_dtype, self.scale_block_k + ) + object.__setattr__(self, "a_scale_dtype", a) + object.__setattr__(self, "b_scale_dtype", b) + object.__setattr__(self, "scale_block_k", block_k) @property def shape(self) -> Tuple[int, int, int]: @@ -762,7 +758,7 @@ class _FragInfo: # Native gfx1250 scaled WMMA. FP8/BF8 use 64 bytes per lane as <16 x i32>. # SCALE packs four K=32 E8M0 bytes in i32 and SCALE16 packs eight K=16 # bytes in i64. Both share the gfx12 column-distributed accumulator. - "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32": _FragInfo( + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32": _FragInfo( 16, 16, 8, @@ -771,7 +767,7 @@ class _FragInfo: None, _wmma_gfx12_acc_16x16, ), - "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32": _FragInfo( + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k32": _FragInfo( 16, 16, 8, @@ -780,7 +776,7 @@ class _FragInfo: None, _wmma_gfx12_acc_16x16, ), - "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32": _FragInfo( + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16": _FragInfo( 16, 16, 8, @@ -789,7 +785,7 @@ class _FragInfo: None, _wmma_gfx12_acc_16x16, ), - "wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32": _FragInfo( + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16": _FragInfo( 16, 16, 8, @@ -839,8 +835,9 @@ class ResourceLimits: class MmaCatalog: """The arch-selected MMA atoms, with optional exact A/B scale filtering. - ``scales=None`` leaves scales unconstrained; ``(None, None)`` selects - unscaled inputs. Exact selection and largest-K ties must be unambiguous. + ``scales=(a_dtype, b_dtype, block_k)`` selects the complete scale contract. + ``None`` leaves scales unconstrained; ``(None, None, None)`` selects unscaled + atoms. Exact selection and largest-K ties must be unambiguous. """ def __init__(self, ops: List[MmaOp]) -> None: @@ -857,12 +854,14 @@ def enumerate( a_dtype: str, b_dtype: str, c_dtype: str, - scales: tuple[MmaScaleOperand | None, MmaScaleOperand | None] | None = None, + scales: tuple[str | None, str | None, int | None] | None = None, m: Optional[int] = None, n: Optional[int] = None, ) -> List[MmaOp]: - if scales is not None and len(scales) != 2: - raise ValueError("scales must contain exactly 2 entries") + if scales is not None: + if len(scales) != 3: + raise ValueError("scales must contain exactly 3 entries") + scales = _normalize_mma_scales(*scales) a, b, c = ( normalize_dtype(a_dtype), normalize_dtype(b_dtype), @@ -874,7 +873,10 @@ def enumerate( continue if (op.a_dtype, op.b_dtype, op.c_dtype) != (a, b, c): continue - if scales is not None and (op.a_scale, op.b_scale) != tuple(scales): + if ( + scales is not None + and (op.a_scale_dtype, op.b_scale_dtype, op.scale_block_k) != scales + ): continue if m is not None and op.m != m: continue @@ -890,7 +892,7 @@ def has_shape( a_dtype: str, b_dtype: str, c_dtype: str, - scales: tuple[MmaScaleOperand | None, MmaScaleOperand | None] | None = None, + scales: tuple[str | None, str | None, int | None] | None = None, m: int, n: int, k: int, @@ -915,7 +917,7 @@ def select_largest_k( a_dtype: str, b_dtype: str, c_dtype: str, - scales: tuple[MmaScaleOperand | None, MmaScaleOperand | None] | None = None, + scales: tuple[str | None, str | None, int | None] | None = None, m: int, n: int, k_max: Optional[int] = None, @@ -958,7 +960,7 @@ def op_for_shape( a_dtype: str, b_dtype: str, c_dtype: str, - scales: tuple[MmaScaleOperand | None, MmaScaleOperand | None] | None = None, + scales: tuple[str | None, str | None, int | None] | None = None, m: int, n: int, k: int, @@ -1107,8 +1109,9 @@ def _mk(role: str, frag_len: int, fn: _LaneCoordFn) -> Optional[LayoutMap]: _a_layout=_mk("a", info.a_frag_len, info.a_fn), _b_layout=_mk("b", info.b_frag_len, info.b_fn), _c_layout=_mk("c", info.c_frag_len, info.c_fn), - a_scale=MmaScaleOperand(**o["a_scale"]) if o.get("a_scale") else None, - b_scale=MmaScaleOperand(**o["b_scale"]) if o.get("b_scale") else None, + a_scale_dtype=o.get("a_scale_dtype"), + b_scale_dtype=o.get("b_scale_dtype"), + scale_block_k=o.get("scale_block_k"), ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py index c4aeb359eafd..848f4ceac1d7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py @@ -80,14 +80,14 @@ def gfx1250_scaled_wmma(op_id: str) -> ScaledWmmaOp | None: if atom is None or atom.family != "wmma_scaled": return None formats = {"fp8e4m3": 0, "bf8e5m2": 1} - # The current backend signatures require matching E8M0 scale packing. + # The current backend supports E8M0 for both inputs and a shared K-group size. # Keep these restrictions here, independently of the catalog query model. if ( atom.a_dtype not in formats or atom.b_dtype not in formats - or atom.a_scale is None - or atom.a_scale != atom.b_scale - or atom.a_scale.dtype != "e8m0" + or atom.a_scale_dtype != "e8m0" + or atom.b_scale_dtype != "e8m0" + or atom.scale_block_k not in (16, 32) or atom.c_dtype != "fp32" or atom.shape != (16, 16, 128) ): @@ -97,6 +97,6 @@ def gfx1250_scaled_wmma(op_id: str) -> ScaledWmmaOp | None: matrix_formats=(formats[atom.a_dtype], formats[atom.b_dtype]), scale_formats=(0, 0), # E8M0 for each source. scales=E8M0ScalePacking( - count=atom.k // atom.a_scale.block_size, block_k=atom.a_scale.block_size + count=atom.k // atom.scale_block_k, block_k=atom.scale_block_k ), ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py index 7b6b47918382..55c74be31d00 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py @@ -1857,9 +1857,12 @@ def mma( c_dtype = getattr(op, "c_dtype", None) is_int_acc = c_dtype == "i32" if c_dtype is not None else _mma_c_is_int(op_id) c_elem = I32 if is_int_acc else F32 + from .arch import ArchTarget + + atom = ArchTarget.from_gfx("gfx1250").mma.by_op_id(op_id) hint = ( "mxacc" - if op_id.startswith("wmma.scaled.") + if atom is not None and atom.family == "wmma_scaled" else _MMA_RESULT_HINT.get(op_id, "acc") ) return self._op( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py index 8996d6aeacc9..44c2ddec18e9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py @@ -309,10 +309,7 @@ def _pop_indent(self) -> None: self._indent -= 1 def lower_op(self, op: Op) -> None: - if ( - op.name.startswith("tile.wmma.") - and gfx1250_scaled_wmma(op.name) is not None - ): + if op.name.startswith("tile.") and gfx1250_scaled_wmma(op.name) is not None: self._emit_wmma_gfx1250_scaled(op) return method = getattr(self, f"_op_{op.name.replace('.', '_')}", None) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md index 6c085c1ad0db..30d8c347b06d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md @@ -26,5 +26,5 @@ its accepted matrix and scale formats. `e8m0` identifies a scale format carried in bytes and packed integer operands. It is not a general scalar IR type or conversion API. Native scaled WMMA uses one `wmma_scaled` catalog family. Its operation IDs encode the matrix source -and result dtypes and each source's scale format and block size; the backend +and accumulator dtype, each source's scale format, and a shared K-group size; the backend selects the LLVM intrinsic and packed operand types from that contract. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py index 4c73642e4204..a5073169eacf 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py @@ -14,7 +14,7 @@ from typing import List, Tuple from ...core.dtypes import normalize_dtype -from ...core.arch import ArchTarget, MmaScaleOperand +from ...core.arch import ArchTarget from ...core.arch.wmma_scale import gfx1250_scaled_wmma from ...core.ir import ( BF16, @@ -123,15 +123,13 @@ def resolved_matrix_path(self) -> str: def _native_scaled_atom(spec: BlockScaledGemmSpec, target: ArchTarget): - scale = MmaScaleOperand( - "e8m0" if spec.scale_dtype == "i8" else spec.scale_dtype, spec.block_k - ) + scale_dtype = "e8m0" if spec.scale_dtype == "i8" else spec.scale_dtype return target.mma.op_for_shape( family="wmma_scaled", a_dtype=spec.dtype_a, b_dtype=spec.dtype_b, c_dtype=spec.dtype_acc, - scales=(scale, scale), + scales=(scale_dtype, scale_dtype, spec.block_k), m=_BLOCK_M, n=_BLOCK_N, k=_WMMA_SCALE_K, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp index e5834c8d24b6..89ad55845ded 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp @@ -794,11 +794,11 @@ void case_gfx1250_scaled_wmma() rocke_value_t* fragment = rocke_b_vec_concat(b, lo, hi); rocke_value_t* c = rocke_b_global_load_vN(b, accum, lane, rocke_f32(), 8, 0); rocke_value_t* scale = rocke_b_global_load(b, scales, lane, v.scale_type(), 1); - const int block = v.scale16 ? 16 : 32; - const rocke_mma_scale_operand_t scales_query[2] = {{"e8m0", block}, {"e8m0", block}}; + const auto block = v.scale16 ? ROCKE_MMA_SCALE_K16 : ROCKE_MMA_SCALE_K32; + const rocke_mma_scale_filter_t scales_query = {"e8m0", "e8m0", block}; const rocke_arch_target_t* target = rocke_arch_target_from_gfx("gfx1250"); const rocke_mma_op_t* atom = rocke_mma_catalog_op_for_shape( - &target->mma, "wmma_scaled", "fp8", "fp8", "fp32", 16, 16, 128, scales_query); + &target->mma, "wmma_scaled", "fp8", "fp8", "fp32", 16, 16, 128, &scales_query); rocke_value_t* extra[2] = {scale, scale}; rocke_value_t* d = rocke_b_mma(b, atom->op_id, fragment, fragment, c, extra, 2); rocke_b_global_store(b, accum, lane, rocke_b_vec_extract(b, d, 0), 1); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp index 7ce214a1e235..9cad6977a3f5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/mma_frag_ssot.cpp @@ -99,12 +99,10 @@ int main(void) CHECK(gfx1250 != NULL, "gfx1250 target"); if(gfx1250) { - check_catalog_fragments( - &gfx1250->mma, - "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32"); - check_catalog_fragments( - &gfx1250->mma, - "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32"); + check_catalog_fragments(&gfx1250->mma, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32"); + check_catalog_fragments(&gfx1250->mma, + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16"); } /* Unknown op_id must be rejected. The engine's error path either returns diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py index 58641452b7d1..9e1a5eb4d013 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py @@ -17,7 +17,7 @@ import pytest -from rocke.core.arch import ArchTarget, MmaCatalog, MmaOp, MmaScaleOperand +from rocke.core.arch import ArchTarget, MmaCatalog, MmaOp, MmaScaleBlockK from rocke.core.arch.wmma_scale import gfx1250_scaled_wmma from rocke.core.arch.target import ( @@ -72,22 +72,26 @@ def _contracts(): a_dtype="fp8e4m3", b_dtype="bf8e5m2", c_dtype="fp32", - a_scale=MmaScaleOperand("e8m0", 32), - b_scale=MmaScaleOperand("e8m0", 32), + a_scale_dtype="e8m0", + b_scale_dtype="e8m0", + scale_block_k=32, m=16, n=16, k=128, op_id="fixture", ) rows = [base] - for field in ("a_scale", "b_scale"): - for scale in (MmaScaleOperand("e4m3", 32), MmaScaleOperand("e8m0", 16), None): - rows.append(replace(base, **{field: scale})) - rows.append(replace(base, a_scale=None, b_scale=None)) - return [replace(row, op_id=row.semantic_id()) for row in rows] + for field in ("a_scale_dtype", "b_scale_dtype"): + for dtype in ("e4m3", "e5m3"): + rows.append(replace(base, **{field: dtype})) + rows.append(replace(base, scale_block_k=16)) + rows.append( + replace(base, a_scale_dtype=None, b_scale_dtype=None, scale_block_k=None) + ) + return [replace(row, op_id=f"fixture_{i}") for i, row in enumerate(rows)] -def test_full_contract_queries_distinguish_each_input_scale(): +def test_full_contract_queries_distinguish_each_scale_type_and_shared_block(): rows = _contracts() assert len({row.op_id for row in rows}) == len(rows) catalog = MmaCatalog(rows) @@ -97,7 +101,7 @@ def test_full_contract_queries_distinguish_each_input_scale(): a_dtype=row.a_dtype, b_dtype=row.b_dtype, c_dtype=row.c_dtype, - scales=(row.a_scale, row.b_scale), + scales=(row.a_scale_dtype, row.b_scale_dtype, row.scale_block_k), m=row.m, n=row.n, ) @@ -118,16 +122,15 @@ def test_partial_query_rejects_ambiguity_but_enumeration_and_existence_are_valid m=16, n=16, ) - assert len(catalog.enumerate(**query)) == 8 + assert len(catalog.enumerate(**query)) == 7 assert catalog.has_shape(**query, k=128) with pytest.raises(ValueError, match="ambiguous MMA query"): catalog.op_for_shape(**query, k=128) with pytest.raises(ValueError, match="ambiguous MMA query"): catalog.select_largest_k(**query) - assert len(catalog.enumerate(**query, scales=(None, None))) == 1 - scale = MmaScaleOperand("e5m3", 32) - assert catalog.op_for_shape(**query, scales=(scale, scale), k=128) is None - with pytest.raises(ValueError, match="exactly 2"): + assert len(catalog.enumerate(**query, scales=(None, None, None))) == 1 + assert catalog.op_for_shape(**query, scales=("e5m3", "e5m3", 32), k=128) is None + with pytest.raises(ValueError, match="exactly 3"): catalog.enumerate(**query, scales=(None,)) @@ -142,17 +145,28 @@ def test_scale_alias_selects_the_same_contract(): n=16, k=128, ) - scales = (MmaScaleOperand("fp8e4m3", 32), MmaScaleOperand("e8m0", 32)) - row = catalog.op_for_shape(**query, scales=scales) - assert row is not None and row.a_scale.dtype == "e4m3" + for scales, field in ( + (("fp8e4m3", "e8m0", 32), "a_scale_dtype"), + (("e8m0", "fp8e4m3", 32), "b_scale_dtype"), + ): + row = catalog.op_for_shape(**query, scales=scales) + assert row is not None and getattr(row, field) == "e4m3" + assert getattr(replace(row, **{field: "fp8e4m3"}), field) == "e4m3" def test_scaled_catalog_identity_and_backend_contract(): catalog = ArchTarget.from_gfx("gfx1250").mma rows = [row for row in catalog.ops if row.family == "wmma_scaled"] assert len(rows) == 4 + assert len({row.op_id for row in rows}) == 4 for row in rows: - assert row.op_id == row.semantic_id() + dtype = {"fp8e4m3": "fp8", "bf8e5m2": "bf8"}[row.a_dtype] + assert row.op_id == ( + f"wmma_gfx1250_f32_16x16x128_{dtype}_{dtype}" + f"_scale_e8m0_e8m0_k{row.scale_block_k}" + ) + assert row.a_scale_dtype == row.b_scale_dtype == "e8m0" + assert isinstance(row.scale_block_k, MmaScaleBlockK) packing = gfx1250_scaled_wmma(row.op_id) assert packing.atom is row assert packing.matrix_formats == ( @@ -166,16 +180,72 @@ def test_scaled_catalog_identity_and_backend_contract(): assert gfx1250_scaled_wmma(old_id) is None +def test_scale_block_k_is_a_two_value_enum(): + assert list(MmaScaleBlockK) == [MmaScaleBlockK.K16, MmaScaleBlockK.K32] + for block in MmaScaleBlockK: + assert replace(_contracts()[0], scale_block_k=block).scale_block_k is block + assert replace(_contracts()[0], scale_block_k=int(block)).scale_block_k is block + with pytest.raises(ValueError): + MmaScaleBlockK(64) + + +@pytest.mark.parametrize("field", ["a_scale_dtype", "b_scale_dtype"]) @pytest.mark.parametrize("dtype", ["i32", "fp4", "e5m2", "", None]) -def test_invalid_scale_format(dtype): +def test_invalid_scale_format(field, dtype): with pytest.raises(ValueError, match="e8m0, e4m3, or e5m3"): - MmaScaleOperand(dtype, 16) + replace(_contracts()[0], **{field: dtype}) @pytest.mark.parametrize("block", [0, 8, 64, 16.0, True, "32", None]) def test_invalid_scale_block_size(block): with pytest.raises(ValueError, match="integer equal to 16 or 32"): - MmaScaleOperand("e8m0", block) + replace(_contracts()[0], scale_block_k=block) + + +@pytest.mark.parametrize( + "scales", + [ + ("e8m0", None, 32), + (None, "e8m0", 32), + (None, None, 32), + ("e8m0", "e8m0", None), + ("i32", "e8m0", 32), + ("e8m0", "e8m0", 0), + ("e8m0", "e8m0", 16.0), + ("e8m0", "e8m0", True), + ], +) +def test_invalid_scale_query_even_for_empty_catalog(scales): + with pytest.raises(ValueError, match="MMA scale"): + MmaCatalog([]).enumerate( + family="wmma_scaled", + a_dtype="fp8", + b_dtype="fp8", + c_dtype="fp32", + scales=scales, + ) + + +def test_unscaled_defaults_have_no_scale_metadata(): + row = MmaOp("wmma", "fp8e4m3", "fp8e4m3", "fp32", 16, 16, 64, "fixture") + assert (row.a_scale_dtype, row.b_scale_dtype, row.scale_block_k) == ( + None, + None, + None, + ) + assert ( + MmaCatalog([row]).op_for_shape( + family="wmma", + a_dtype="fp8", + b_dtype="fp8", + c_dtype="fp32", + m=16, + n=16, + k=64, + scales=(None, None, None), + ) + is row + ) def test_largest_k_only_rejects_ties_at_the_selected_k(): diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py index 0717f7f6007d..31cb77ec09c5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py @@ -8,7 +8,7 @@ from dataclasses import replace from unittest import mock -from rocke.core.arch import ArchTarget, MmaScaleOperand +from rocke.core.arch import ArchTarget from rocke.core.arch.wmma_scale import gfx1250_scaled_wmma from rocke.core.ir import F32, I32, I64, IRBuilder, PtrType from rocke.core.ir_serialize import parse, serialize @@ -23,13 +23,13 @@ def _scaled_atom(dtype="fp8", scale16=False): - scale = MmaScaleOperand("e8m0", 16 if scale16 else 32) + block_k = 16 if scale16 else 32 atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( family="wmma_scaled", a_dtype=dtype, b_dtype=dtype, c_dtype="fp32", - scales=(scale, scale), + scales=("e8m0", "e8m0", block_k), m=16, n=16, k=128, @@ -56,6 +56,17 @@ def _build_scaled_atom(*, scale16: bool): class TestGfx1250ScaledWmma(unittest.TestCase): + def test_backend_rejects_unsupported_scale_type_on_either_input(self): + catalog = ArchTarget.from_gfx("gfx1250").mma + for field in ("a_scale_dtype", "b_scale_dtype"): + atom = replace(_scaled_atom(), **{field: "e4m3"}) + with ( + self.subTest(field=field), + mock.patch.object(catalog, "by_op_id", return_value=atom), + self.assertRaisesRegex(ValueError, "unsupported scaled WMMA"), + ): + gfx1250_scaled_wmma(atom.op_id) + def test_backend_derives_each_matrix_carrier_and_selector(self): # Synthetic metadata exercises the lowering contract without adding atoms. atom = replace(_scaled_atom(), b_dtype="bf8e5m2", b_frag_len=8) @@ -108,7 +119,7 @@ def test_matrix_formats_share_intrinsic_declarations(self): self.assertEqual(call.count(f"i32 {selector}, <16 x i32>"), 2) self.assertEqual(call.count(f", {scale_ty} %"), 2) - def test_semantic_ids_survive_serialization(self): + def test_catalog_ids_survive_serialization(self): for scale16 in (False, True): kernel = _build_scaled_atom(scale16=scale16) text = serialize(kernel) @@ -126,8 +137,8 @@ def test_semantic_ids_survive_serialization(self): def test_catalog_fragment_lengths(self): target = ArchTarget.from_gfx("gfx1250") for op_id in ( - "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32", - "wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32", + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32", + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16", ): with self.subTest(op_id=op_id): op = target.mma.by_op_id(op_id) @@ -172,8 +183,9 @@ def test_scale_and_scale16_exact_llvm23_abi(self): def test_scaled_wmma_rejects_pre_llvm23_flavors(self): for flavor in ("llvm20", "llvm22"): - with self.subTest(flavor=flavor), self.assertRaisesRegex( - NotImplementedError, "requires llvm23" + with ( + self.subTest(flavor=flavor), + self.assertRaisesRegex(NotImplementedError, "requires llvm23"), ): lower_kernel_to_llvm( _build_scaled_atom(scale16=False), diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py index cfdf6cde84f7..fe590a754013 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_lowbit_dtype_aliases.py @@ -32,7 +32,7 @@ def test_alias_catalog_lookup(alias, canonical): found = True assert catalog.has_shape( family=op.family, - scales=(op.a_scale, op.b_scale), + scales=(op.a_scale_dtype, op.b_scale_dtype, op.scale_block_k), a_dtype=alias, b_dtype=op.b_dtype, c_dtype=op.c_dtype, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp index 20a1fc8ca85d..3d4763db4c29 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp @@ -52,11 +52,11 @@ static bool rejects_query(F query) static int test_scale_contracts() { const auto* arch = rocke_arch_target_from_gfx("gfx1250"); - const rocke_mma_scale_operand_t e8[2] = {{"e8m0", 32}, {"e8m0", 32}}; + const rocke_mma_scale_filter_t e8 = {"e8m0", "e8m0", ROCKE_MMA_SCALE_K32}; const auto* base = rocke_mma_catalog_op_for_shape( - &arch->mma, "wmma_scaled", "fp8", "fp8", "fp32", 16, 16, 128, e8); + &arch->mma, "wmma_scaled", "fp8", "fp8", "fp32", 16, 16, 128, &e8); CHECK(base); - rocke_mma_op_t rows[9]; + rocke_mma_op_t rows[8]; rows[0] = *base; rows[0].b_dtype = "bf8e5m2"; auto packing_atom = rows[0]; @@ -65,43 +65,47 @@ static int test_scale_contracts() CHECK(packing.matrix_formats[0] == 0 && packing.matrix_formats[1] == 1); CHECK(packing.matrix_words[0] == 16 && packing.matrix_words[1] == 8); CHECK(strstr(packing.intrinsic, "v16i32.v8i32")); + for(int input = 0; input < 2; ++input) + { + auto unsupported = *base; + (input == 0 ? unsupported.a_scale_dtype : unsupported.b_scale_dtype) = "e4m3"; + CHECK(rejects_query([&] { rocke_scaled_wmma_contract(&unsupported); })); + } int count = 1; for(int input = 0; input < 2; ++input) { - for(const auto scale : {rocke_mma_scale_operand_t{"e4m3", 32}, - rocke_mma_scale_operand_t{"e8m0", 16}, - rocke_mma_scale_operand_t{NULL, 0}}) + for(const char* dtype : {"e4m3", "e5m3"}) { rows[count] = rows[0]; - (input == 0 ? rows[count].a_scale : rows[count].b_scale) = scale; + (input == 0 ? rows[count].a_scale_dtype : rows[count].b_scale_dtype) = dtype; ++count; } } rows[count] = rows[0]; - rows[count].a_scale = {NULL, 0}; - rows[count++].b_scale = {NULL, 0}; + rows[count++].scale_block_k = ROCKE_MMA_SCALE_K16; + rows[count] = rows[0]; + rows[count].a_scale_dtype = NULL; + rows[count].b_scale_dtype = NULL; + rows[count++].scale_block_k = ROCKE_MMA_SCALE_NONE; rocke_mma_catalog_t cat = {rows, count}; - char ids[8][256]; for(int i = 0; i < count; ++i) { const auto& op = rows[i]; - const rocke_mma_scale_operand_t scales[2] = {op.a_scale, op.b_scale}; - CHECK(rocke_mma_op_semantic_id(&op, ids[i], sizeof(ids[i]))); - for(int j = 0; j < i; ++j) - CHECK(strcmp(ids[i], ids[j]) != 0); + const rocke_mma_scale_filter_t scales + = {op.a_scale_dtype, op.b_scale_dtype, op.scale_block_k}; CHECK(rocke_mma_catalog_enumerate( - &cat, "wmma_scaled", "fp8", "bf8", "f32", 16, 16, NULL, 0, scales) + &cat, "wmma_scaled", "fp8", "bf8", "f32", 16, 16, NULL, 0, &scales) == 1); CHECK(rocke_mma_catalog_has_shape( - &cat, "wmma_scaled", "fp8", "bf8", "f32", 16, 16, 128, scales)); + &cat, "wmma_scaled", "fp8", "bf8", "f32", 16, 16, 128, &scales)); CHECK(rocke_mma_catalog_op_for_shape( - &cat, "wmma_scaled", "fp8", "bf8", "f32", 16, 16, 128, scales) + &cat, "wmma_scaled", "fp8", "bf8", "f32", 16, 16, 128, &scales) == &op); CHECK(rocke_mma_catalog_select_largest_k( - &cat, "wmma_scaled", "fp8", "bf8", "f32", 16, 16, -1, scales) + &cat, "wmma_scaled", "fp8", "bf8", "f32", 16, 16, -1, &scales) == &op); CHECK(!rocke_mma_catalog_select_largest_k( - &cat, "wmma_scaled", "fp8", "bf8", "f32", 16, 16, 64, scales)); + &cat, "wmma_scaled", "fp8", "bf8", "f32", 16, 16, 64, &scales)); } CHECK(rocke_mma_catalog_enumerate(&cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, NULL, 0) == count); @@ -112,40 +116,44 @@ static int test_scale_contracts() CHECK(rejects_query([&] { rocke_mma_catalog_select_largest_k(&cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, -1); })); - const rocke_mma_scale_operand_t unscaled[2] = {{NULL, 0}, {NULL, 0}}; + const rocke_mma_scale_filter_t unscaled = {NULL, NULL, ROCKE_MMA_SCALE_NONE}; CHECK(rocke_mma_catalog_op_for_shape( - &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128, unscaled) - == &rows[7]); - const rocke_mma_scale_operand_t alias[2] = {{"fp8e4m3", 32}, {"e8m0", 32}}; + &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128, &unscaled) + == &rows[6]); + const rocke_mma_scale_filter_t alias = {"fp8e4m3", "e8m0", ROCKE_MMA_SCALE_K32}; CHECK(rocke_mma_catalog_op_for_shape( - &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128, alias) + &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128, &alias) == &rows[1]); - rows[8] = rows[0]; - rows[8].k = 256; - cat.num_ops = 9; + const rocke_mma_scale_filter_t alias_b = {"e8m0", "fp8e4m3", ROCKE_MMA_SCALE_K32}; + CHECK(rocke_mma_catalog_op_for_shape( + &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128, &alias_b) + == &rows[3]); + rows[7] = rows[0]; + rows[7].k = 256; + cat.num_ops = 8; CHECK(rocke_mma_catalog_select_largest_k(&cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, -1) - == &rows[8]); + == &rows[7]); CHECK(rejects_query([&] { rocke_mma_catalog_select_largest_k(&cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128); })); - for(const auto invalid : {rocke_mma_scale_operand_t{"i32", 32}, - rocke_mma_scale_operand_t{"e8m0", 8}, - rocke_mma_scale_operand_t{NULL, 32}}) + for(const auto scales : + {rocke_mma_scale_filter_t{"i32", "e8m0", ROCKE_MMA_SCALE_K32}, + rocke_mma_scale_filter_t{"e8m0", "i32", ROCKE_MMA_SCALE_K32}, + rocke_mma_scale_filter_t{"e8m0", "e8m0", static_cast(8)}, + rocke_mma_scale_filter_t{NULL, "e8m0", ROCKE_MMA_SCALE_K32}, + rocke_mma_scale_filter_t{"e8m0", NULL, ROCKE_MMA_SCALE_K32}, + rocke_mma_scale_filter_t{NULL, NULL, ROCKE_MMA_SCALE_K32}, + rocke_mma_scale_filter_t{"e8m0", "e8m0", ROCKE_MMA_SCALE_NONE}}) { - const rocke_mma_scale_operand_t scales[2] = {invalid, e8[1]}; CHECK(rejects_query([&] { rocke_mma_catalog_enumerate( - &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, NULL, 0, scales); + &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, NULL, 0, &scales); })); } - const rocke_mma_scale_operand_t valid_missing[2] = {{"e5m3", 32}, {"e8m0", 32}}; + const rocke_mma_scale_filter_t valid_missing = {"e5m3", "e5m3", ROCKE_MMA_SCALE_K32}; CHECK(!rocke_mma_catalog_op_for_shape( - &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128, valid_missing)); + &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128, &valid_missing)); char id[256]; - CHECK(rocke_mma_op_semantic_id(base, id, sizeof(id))); - CHECK(strcmp(id, base->op_id) == 0); - CHECK(!rocke_mma_op_semantic_id(base, id, strlen(base->op_id))); - CHECK(rocke_mma_op_semantic_id(base, id, strlen(base->op_id) + 1)); int packed_rows = 0, scaled_rows = 0; for(int i = 0; i < arch->mma.num_ops; ++i) { @@ -157,8 +165,16 @@ static int test_scale_contracts() } if(strcmp(op.family, "wmma_scaled") == 0) { - CHECK(rocke_mma_op_semantic_id(&op, id, sizeof(id))); + snprintf(id, + sizeof(id), + "wmma_gfx1250_f32_16x16x128_%s_%s_scale_%s_%s_k%d", + short_dtype(op.a_dtype), + short_dtype(op.b_dtype), + op.a_scale_dtype, + op.b_scale_dtype, + op.scale_block_k); CHECK(strcmp(id, op.op_id) == 0); + CHECK(rocke_mma_catalog_by_op_id(&arch->mma, id) == &op); ++scaled_rows; } } @@ -188,13 +204,14 @@ int main() return 1; } } - const rocke_mma_scale_operand_t scales[2] = {op->a_scale, op->b_scale}; + const rocke_mma_scale_filter_t scales + = {op->a_scale_dtype, op->b_scale_dtype, op->scale_block_k}; for(const char* a : {op->a_dtype, short_dtype(op->a_dtype)}) { for(const char* b : {op->b_dtype, short_dtype(op->b_dtype)}) { if(!rocke_mma_catalog_has_shape( - &arch->mma, op->family, a, b, op->c_dtype, op->m, op->n, op->k, scales) + &arch->mma, op->family, a, b, op->c_dtype, op->m, op->n, op->k, &scales) || rocke_mma_catalog_op_for_shape(&arch->mma, op->family, a, @@ -203,7 +220,7 @@ int main() op->m, op->n, op->k, - scales) + &scales) != op || rocke_mma_catalog_select_largest_k(&arch->mma, op->family, @@ -213,7 +230,7 @@ int main() op->m, op->n, op->k, - scales) + &scales) != op || rocke_mma_catalog_enumerate(&arch->mma, op->family, @@ -224,7 +241,7 @@ int main() op->n, NULL, 0, - scales) + &scales) < 1) { fprintf(stderr, "%s: catalog query missed %s/%s\n", op->op_id, a, b); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json index ca8f4b2799bf..5e2c928fa856 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json @@ -1117,19 +1117,19 @@ }, "expected_failures": { "gemm/gfx1250/mxfp8/wmma_scale/bf8e5m2": { - "message": "tile.wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm20", + "message": "tile.wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k32 requires llvm23 (ROCm 7.13+), got llvm20", "type": "NotImplementedError" }, "gemm/gfx1250/mxfp8/wmma_scale/fp8e4m3": { - "message": "tile.wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm20", + "message": "tile.wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32 requires llvm23 (ROCm 7.13+), got llvm20", "type": "NotImplementedError" }, "gemm/gfx1250/mxfp8/wmma_scale16/bf8e5m2": { - "message": "tile.wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm20", + "message": "tile.wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16 requires llvm23 (ROCm 7.13+), got llvm20", "type": "NotImplementedError" }, "gemm/gfx1250/mxfp8/wmma_scale16/fp8e4m3": { - "message": "tile.wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm20", + "message": "tile.wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16 requires llvm23 (ROCm 7.13+), got llvm20", "type": "NotImplementedError" } }, @@ -2281,19 +2281,19 @@ }, "expected_failures": { "gemm/gfx1250/mxfp8/wmma_scale/bf8e5m2": { - "message": "tile.wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b32.src1_bf8e5m2_e8m0_b32.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm22", + "message": "tile.wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k32 requires llvm23 (ROCm 7.13+), got llvm22", "type": "NotImplementedError" }, "gemm/gfx1250/mxfp8/wmma_scale/fp8e4m3": { - "message": "tile.wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b32.src1_fp8e4m3_e8m0_b32.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm22", + "message": "tile.wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32 requires llvm23 (ROCm 7.13+), got llvm22", "type": "NotImplementedError" }, "gemm/gfx1250/mxfp8/wmma_scale16/bf8e5m2": { - "message": "tile.wmma.scaled.16x16x128.src0_bf8e5m2_e8m0_b16.src1_bf8e5m2_e8m0_b16.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm22", + "message": "tile.wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16 requires llvm23 (ROCm 7.13+), got llvm22", "type": "NotImplementedError" }, "gemm/gfx1250/mxfp8/wmma_scale16/fp8e4m3": { - "message": "tile.wmma.scaled.16x16x128.src0_fp8e4m3_e8m0_b16.src1_fp8e4m3_e8m0_b16.src2_fp32.dst_fp32 requires llvm23 (ROCm 7.13+), got llvm22", + "message": "tile.wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16 requires llvm23 (ROCm 7.13+), got llvm22", "type": "NotImplementedError" } }, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c index f532e7bff623..66a947f45ecf 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c @@ -122,11 +122,11 @@ static void build_wmma_k64_bf8_bf8(rocke_ir_builder_t* b) * E8M0 scale operands are i32 for SCALE and i64 for SCALE16. */ static void wmma_scaled(rocke_ir_builder_t* b, bool scale16, const char* dtype) { - const int block = scale16 ? 16 : 32; - const rocke_mma_scale_operand_t query[2] = {{"e8m0", block}, {"e8m0", block}}; + const rocke_mma_scale_block_k_t block = scale16 ? ROCKE_MMA_SCALE_K16 : ROCKE_MMA_SCALE_K32; + const rocke_mma_scale_filter_t query = {"e8m0", "e8m0", block}; const rocke_arch_target_t* target = rocke_arch_target_from_gfx("gfx1250"); const rocke_mma_op_t* atom = rocke_mma_catalog_op_for_shape( - &target->mma, "wmma_scaled", dtype, dtype, "fp32", 16, 16, 128, query); + &target->mma, "wmma_scaled", dtype, dtype, "fp32", 16, 16, 128, &query); const char* op_id = atom->op_id; const rocke_type_t* scale_ty = scale16 ? rocke_i64() : rocke_i32(); rocke_value_t* a_ptr = frag_param(b, "A", rocke_i32(), true); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py index a9ec2a40f2b9..eb7dffc65f1b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py @@ -19,7 +19,7 @@ # each with its gfx950 counterpart pins both sides of the choice. # # arch is per-config (see _spec), llvm_flavor = AUTO, matching the C side. -from rocke.core.arch import ArchTarget, MmaScaleOperand +from rocke.core.arch import ArchTarget from rocke.core.ir import BF16, F16, F32, I16, I32, I64, IRBuilder, KernelDef, PtrType from _emit_common import run_emit @@ -78,13 +78,13 @@ def build(b: IRBuilder) -> None: def _wmma_scaled(a_kind, b_kind, scale_mode): """K=128 scaled WMMA, parameterized by operand dtypes and scale mode.""" scale_ty = {"scale": I32, "scale16": I64}[scale_mode] - scale = MmaScaleOperand("e8m0", 16 if scale_mode == "scale16" else 32) + block_k = 16 if scale_mode == "scale16" else 32 atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( family="wmma_scaled", a_dtype=a_kind, b_dtype=b_kind, c_dtype="fp32", - scales=(scale, scale), + scales=("e8m0", "e8m0", block_k), m=16, n=16, k=128, From 856e2452c2e9a63f2af5750d1a4440a629b8d6ef Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:26:54 -0400 Subject: [PATCH 08/15] docs(rocke): correct scaled WMMA catalog support comment --- .../rocke/platform/python/rocke/core/arch/data/arch_specs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json index 674428134f06..3d12a94fae34 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/arch_specs.json @@ -142,7 +142,7 @@ ] }, "gfx1250": { - "_comment": "gfx1250-class CDNA multi-chip device on the GFX12 programming model (wave32, WMMA, no MFMA). Primary fp16/bf16 WMMA atom is 16x16x32 (NOT gfx1201's 16x16x16). ABI empirically confirmed on ROCm 7.13 clang 23 (gfx1250 device target): __builtin_amdgcn_wmma_f32_16x16x32_f16 -> llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1,<16 x half>,i1,<16 x half>,i16,<8 x float>,i1,i1); bf16 uses v16bf16 (<16 x bfloat>) directly, not the gfx11 <16 x i16> bitcast. Datalayout/triple byte-identical to gfx950/gfx1201. This catalog ships validated fp16/bf16 and fp8/bf8 WMMA atoms plus the FP8xFP8 SCALE/SCALE16 prerequisite slice; FP4xFP4 SCALE/SCALE16 accepts prepacked E2M1 bytes; FP6/BF6 forms remain omitted. Lane maps are a HYPOTHESIS verified by examples/gfx1250/wmma_probe.py. MEMORY: gfx1250 HAS async global->LDS DMA (its own GFX12 family: global_load_async_to_lds_b128 + s_wait_asynccnt -- NOT the gfx9 buffer_load_lds/global_load_lds, which do not select), so has_async_global_lds=true (GPU-validated via the DTLA path). It also HAS a hardware transpose-LDS read, ds_load_tr16_b128 (GPU-validated via compute_pv_dstr). The has_async_lds and has_ds_read_tr flags stay false on purpose: in this catalog they denote the gfx9/gfx950 ds_read_*_tr_* / async-LDS *ABI* used to admit the gfx950 MFMA tiled builder (attention_arch.py), which gfx1250 does NOT have -- its async/transpose use distinct GFX12 opcodes gated directly in the ISA backend, not via these MFMA-pipeline bits. has_tdm=false: TDM not verified on this part.", + "_comment": "gfx1250-class CDNA multi-chip device on the GFX12 programming model (wave32, WMMA, no MFMA). Primary fp16/bf16 WMMA atom is 16x16x32 (NOT gfx1201's 16x16x16). ABI empirically confirmed on ROCm 7.13 clang 23 (gfx1250 device target): __builtin_amdgcn_wmma_f32_16x16x32_f16 -> llvm.amdgcn.wmma.f32.16x16x32.f16.v8f32.v16f16(i1,<16 x half>,i1,<16 x half>,i16,<8 x float>,i1,i1); bf16 uses v16bf16 (<16 x bfloat>) directly, not the gfx11 <16 x i16> bitcast. Datalayout/triple byte-identical to gfx950/gfx1201. This catalog ships validated fp16/bf16 and fp8/bf8 WMMA atoms plus homogeneous FP8xFP8 and BF8xBF8 SCALE/SCALE16 with E8M0 scales; FP4/FP6/BF6 block-scaled forms remain intentionally omitted. Lane maps are a HYPOTHESIS verified by examples/gfx1250/wmma_probe.py. MEMORY: gfx1250 HAS async global->LDS DMA (its own GFX12 family: global_load_async_to_lds_b128 + s_wait_asynccnt -- NOT the gfx9 buffer_load_lds/global_load_lds, which do not select), so has_async_global_lds=true (GPU-validated via the DTLA path). It also HAS a hardware transpose-LDS read, ds_load_tr16_b128 (GPU-validated via compute_pv_dstr). The has_async_lds and has_ds_read_tr flags stay false on purpose: in this catalog they denote the gfx9/gfx950 ds_read_*_tr_* / async-LDS *ABI* used to admit the gfx950 MFMA tiled builder (attention_arch.py), which gfx1250 does NOT have -- its async/transpose use distinct GFX12 opcodes gated directly in the ISA backend, not via these MFMA-pipeline bits. has_tdm=false: TDM not verified on this part.", "family": "cdna", "target_family": "gfx12_cdna", "matrix_path": "wmma", From 082651964df5de0090e65135dcb2067b2ddd38ce Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:00:53 -0400 Subject: [PATCH 09/15] fix(rocke): expose scale layouts and restore unscaled MMA lowering --- .../rocke/platform/cpp/core/arch/data.cpp | 74 +++++++- .../rocke/platform/cpp/core/arch/query.cpp | 36 ++++ .../rocke/platform/cpp/core/ir/ir_tile.cpp | 11 +- .../platform/cpp/core/lower_llvm/mma.cpp | 63 ++----- .../platform/cpp/include/rocke/arch_target.h | 34 +++- .../cpp/include/rocke/wmma_scale_internal.h | 5 +- .../dsl_docs/architecture/authoring_model.md | 18 +- .../platform/python/rocke/core/arch/target.py | 85 ++++++++- .../python/rocke/core/arch/wmma_scale.py | 5 +- .../rocke/platform/python/rocke/core/ir.py | 5 +- .../platform/python/rocke/core/lower_hip.py | 1 + .../platform/python/rocke/core/lower_llvm.py | 1 + .../rocke/examples/gfx1250/gemm/README.md | 12 ++ .../gfx1250/gemm/block_scaled_gemm_verify.py | 11 +- .../instances/gfx1250/block_scaled_gemm.py | 28 +-- .../tests/core/future_intrinsic_lowering.cpp | 54 ++++++ .../platform/tests/core/test_arch_mma_ssot.py | 63 +++++++ .../tests/core/test_gfx1250_scaled_wmma.py | 44 +++++ .../tests/core/test_mma_catalog_lookup.cpp | 71 ++++++- .../tests/core/test_mma_scale_layouts.py | 174 ++++++++++++++++++ .../rocke_representative_ir_sha256.json | 16 +- .../instances/parity/gfx1250_lowering_emit.c | 35 ++++ .../instances/parity/gfx1250_lowering_emit.py | 28 +++ 23 files changed, 771 insertions(+), 103 deletions(-) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_scale_layouts.py diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp index a71c5c20dac9..3774425da6ac 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/data.cpp @@ -606,9 +606,11 @@ bool rocke_layout_map_coord(const rocke_layout_map_t* m, { if(b != NULL) { - const char* role_txt = (m->role == ROCKE_MMA_ROLE_ACC) ? "acc" - : (m->role == ROCKE_MMA_ROLE_A) ? "a" - : "b"; + const char* role_txt = (m->role == ROCKE_MMA_ROLE_ACC) ? "acc" + : (m->role == ROCKE_MMA_ROLE_A) ? "a" + : (m->role == ROCKE_MMA_ROLE_B) ? "b" + : (m->role == ROCKE_MMA_ROLE_A_SCALE) ? "a_scale" + : "b_scale"; b->status = ROCKE_ERR_VALUE; snprintf(b->err, ROCKE_ERR_MSG_CAP, @@ -627,6 +629,39 @@ bool rocke_layout_map_coord(const rocke_layout_map_t* m, return true; } +/* Scale slots describe logical K groups; both half-waves duplicate them. */ +static void _wmma_gfx1250_a_scale(rocke_ir_builder_t* b, + rocke_value_t* lane, + int slot, + rocke_value_t** out0, + rocke_value_t** out1) +{ + ROCKE_ATI_COORD_GUARD(b, out0, out1); + rocke_value_t* c16 = rocke_b_const_i32(b, 16); + rocke_value_t* row = rocke_b_mod(b, lane, c16); + rocke_value_t* group = rocke_b_const_i32(b, slot); + if(out0) + *out0 = row; + if(out1) + *out1 = group; +} + +static void _wmma_gfx1250_b_scale(rocke_ir_builder_t* b, + rocke_value_t* lane, + int slot, + rocke_value_t** out0, + rocke_value_t** out1) +{ + ROCKE_ATI_COORD_GUARD(b, out0, out1); + rocke_value_t* c16 = rocke_b_const_i32(b, 16); + rocke_value_t* col = rocke_b_mod(b, lane, c16); + rocke_value_t* group = rocke_b_const_i32(b, slot); + if(out0) + *out0 = group; + if(out1) + *out1 = col; +} + /* ========================================================================= * Per-op_id LayoutMap statics (the precomputed _build_mma_op output) * ========================================================================= @@ -642,6 +677,15 @@ bool rocke_layout_map_coord(const rocke_layout_map_t* m, * struct; the rest carry NULL layout pointers in the catalog directly. */ +static const rocke_layout_map_t lm_wmma_scale_k32_a + = {ROCKE_MMA_ROLE_A_SCALE, 4, 32, _wmma_gfx1250_a_scale}; +static const rocke_layout_map_t lm_wmma_scale_k32_b + = {ROCKE_MMA_ROLE_B_SCALE, 4, 32, _wmma_gfx1250_b_scale}; +static const rocke_layout_map_t lm_wmma_scale_k16_a + = {ROCKE_MMA_ROLE_A_SCALE, 8, 32, _wmma_gfx1250_a_scale}; +static const rocke_layout_map_t lm_wmma_scale_k16_b + = {ROCKE_MMA_ROLE_B_SCALE, 8, 32, _wmma_gfx1250_b_scale}; + /* --- mfma_f32_16x16x16_f16 / _bf16: a/b/c all present (frag 4/4/4, wave64) --- */ static const rocke_layout_map_t lm_mfma_16x16x16_a = {ROCKE_MMA_ROLE_A, 4, 64, _mfma_a_16x16}; static const rocke_layout_map_t lm_mfma_16x16x16_b = {ROCKE_MMA_ROLE_B, 4, 64, _mfma_b_16x16}; @@ -1689,7 +1733,11 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { &lm_wmma_gfx12_c, "e8m0", "e8m0", - ROCKE_MMA_SCALE_K32}, + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, {"wmma_scaled", "bf8e5m2", "bf8e5m2", @@ -1707,7 +1755,11 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { &lm_wmma_gfx12_c, "e8m0", "e8m0", - ROCKE_MMA_SCALE_K32}, + ROCKE_MMA_SCALE_K32, + 4, + 4, + &lm_wmma_scale_k32_a, + &lm_wmma_scale_k32_b}, {"wmma_scaled", "fp8e4m3", "fp8e4m3", @@ -1725,7 +1777,11 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { &lm_wmma_gfx12_c, "e8m0", "e8m0", - ROCKE_MMA_SCALE_K16}, + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, {"wmma_scaled", "bf8e5m2", "bf8e5m2", @@ -1743,7 +1799,11 @@ static const rocke_mma_op_t k_mma_gfx1250[] = { &lm_wmma_gfx12_c, "e8m0", "e8m0", - ROCKE_MMA_SCALE_K16}, + ROCKE_MMA_SCALE_K16, + 8, + 8, + &lm_wmma_scale_k16_a, + &lm_wmma_scale_k16_b}, }; /* ========================================================================= diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp index faa1fe52fd9c..a33de3abf308 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp @@ -158,6 +158,18 @@ const rocke_layout_map_t* rocke_mma_op_acc_layout(const rocke_mma_op_t* op, rock return rocke_mma_op_c_layout(op, b); } +const rocke_layout_map_t* rocke_mma_op_a_scale_layout(const rocke_mma_op_t* op, + rocke_ir_builder_t* b) +{ + return rocke_ati_require_layout(op, op ? op->a_scale_layout : NULL, "a_scale", b); +} + +const rocke_layout_map_t* rocke_mma_op_b_scale_layout(const rocke_mma_op_t* op, + rocke_ir_builder_t* b) +{ + return rocke_ati_require_layout(op, op ? op->b_scale_layout : NULL, "b_scale", b); +} + /* ============================== MMA catalog =========================== */ const rocke_mma_op_t* rocke_mma_catalog_ops(const rocke_mma_catalog_t* cat, int* num_out) @@ -370,6 +382,30 @@ const rocke_mma_op_t* rocke_mma_catalog_op_for_shape(const rocke_mma_catalog_t* /* ===================== bare-op_id SSOT lookups ======================== */ +const char* rocke_arch_mma_op_id_family(const char* op_id) +{ + if(!op_id) + return NULL; + const char* family = NULL; + for(int i = 0; i < rocke_ati_arch_registry_len; ++i) + { + const auto* target = rocke_ati_arch_registry[i].target; + if(!target) + continue; + for(int j = 0; j < target->mma.num_ops; ++j) + { + const auto& op = target->mma.ops[j]; + if(strcmp(op.op_id, op_id) != 0) + continue; + if(family && strcmp(family, op.family) != 0) + ckc::raise_status(ROCKE_ERR_VALUE, + "arch SSOT drift: op_id has inconsistent family across arches"); + family = op.family; + } + } + return family; +} + const char* rocke_arch_mma_op_id_c_dtype(const char* op_id) { int i; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp index 384430fb9ff0..8ebb8bb6191e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp @@ -26,8 +26,8 @@ * rocke_b_mma emits a single tile.mma op keyed by op_id; the ISA backend lowers * that op_id to the matching MFMA/WMMA call. To size the result vector it needs * the accumulator fragment length and dtype for the atom. Both come from the - * arch SSOT (core/arch/target): frag lengths from rocke_arch_mma_c_frag_len - * (the _MMA_FRAGMENT_INFO projection) and the accumulator dtype from + * arch SSOT (core/arch/target): frag lengths from rocke_arch_mma_c_frag_len (the + * _MMA_FRAGMENT_INFO projection) and the accumulator dtype from * rocke_arch_mma_op_id_c_dtype (the JSON catalog aggregation). This bucket keeps * NO private copy of that data, mirroring ir.py after it dropped _MMA_C_FRAG_LEN * / _MMA_C_INT_OP_IDS. @@ -51,7 +51,7 @@ static const rocke_mma_hint_row_t ROCKE_MMA_RESULT_HINT[] = { {"mfma_scale_f32_16x16x128_f8f6f4", "mxacc"}, }; -/* accumulator fragment length for op_id, from the arch SSOT +/* Accumulator fragment length for op_id, from the arch SSOT * (target._frag_info(op_id).c_frag_len). Returns <= 0 for an unknown atom (the * zero-length _frag_info fallback), which rocke_b_mma reports as an error. */ static int rocke_mma_c_frag_len(const char* op_id) @@ -69,9 +69,8 @@ static bool rocke_mma_is_int_acc(const char* op_id) static const char* rocke_mma_result_hint(const char* op_id) { - const rocke_arch_target_t* target = rocke_arch_target_from_gfx("gfx1250"); - const rocke_mma_op_t* atom = op_id ? rocke_mma_catalog_by_op_id(&target->mma, op_id) : NULL; - if(atom && strcmp(atom->family, "wmma_scaled") == 0) + const char* family = rocke_arch_mma_op_id_family(op_id); + if(family && strcmp(family, "wmma_scaled") == 0) return "mxacc"; size_t i; if(op_id) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp index e3c9c980f346..48cb714160be 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp @@ -441,26 +441,23 @@ static const int WMMA_INT_SPECS_N = (int)(sizeof(WMMA_INT_SPECS) / sizeof(WMMA_I static void _emit_wmma_int(rocke_lower_t* L, const rocke_op_t* op, const _wmma_int_spec_t* spec) { const rocke_value_t *a, *b, *c; - const char *src2_ty, *dst_ty; a = op->operands[0]; b = op->operands[1]; c = op->operands[2]; - src2_ty = rocke_ll_llvm_type(L, c->type); - dst_ty = rocke_ll_llvm_type(L, op->results[0]->type); rocke_ll_need(L, spec->decl_key); rocke_ll_emitf(L, - " %s = call %s @%s(" + " %s = call <%d x i32> @%s(" "i1 1, <%d x i32> %s, " "i1 1, <%d x i32> %s, " - "%s %s, i1 0)", + "<%d x i32> %s, i1 0)", mma_result_name(L, op), - dst_ty, + spec->acc_vec, spec->intrinsic, spec->op_vec, rocke_ll_operand(L, a), spec->op_vec, rocke_ll_operand(L, b), - src2_ty, + spec->acc_vec, rocke_ll_operand(L, c)); } @@ -468,7 +465,7 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id { const _wmma_spec_t* spec = NULL; const rocke_value_t *a, *b, *c; - const char *a_arg, *b_arg, *src2_ty, *dst_ty; + const char *a_arg, *b_arg; int w, i; if(!rocke_ll_live(L)) @@ -516,8 +513,6 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id a = op->operands[0]; b = op->operands[1]; c = op->operands[2]; - src2_ty = rocke_ll_llvm_type(L, c->type); - dst_ty = rocke_ll_llvm_type(L, op->results[0]->type); w = spec->frag_width; rocke_ll_need(L, spec->decl_key); @@ -556,13 +551,12 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id * i1 reuseA, i1 reuseB). The negate / format / reuse immediates are * pinned to the plain unscaled MMA. */ rocke_ll_emitf(L, - " %s = call %s @%s(" + " %s = call <8 x float> @%s(" "i1 false, <%d x %s> %s, " "i1 false, <%d x %s> %s, " - "i16 0, %s %s, " + "i16 0, <8 x float> %s, " "i1 false, i1 false)", mma_result_name(L, op), - dst_ty, spec->intrinsic, w, spec->call_elt, @@ -570,7 +564,6 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id w, spec->call_elt, b_arg, - src2_ty, rocke_ll_operand(L, c)); return; } @@ -579,12 +572,11 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id /* gfx1250 K=64 6-operand form: (A, B, i16 fmt, C, i1, i1), format and * reuse immediates pinned to 0 (plain unscaled MMA). */ rocke_ll_emitf(L, - " %s = call %s @%s(" + " %s = call <8 x float> @%s(" "<%d x %s> %s, <%d x %s> %s, " - "i16 0, %s %s, " + "i16 0, <8 x float> %s, " "i1 false, i1 false)", mma_result_name(L, op), - dst_ty, spec->intrinsic, w, spec->call_elt, @@ -592,15 +584,13 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id w, spec->call_elt, b_arg, - src2_ty, rocke_ll_operand(L, c)); return; } rocke_ll_emitf(L, - " %s = call %s @%s(" - "<%d x %s> %s, <%d x %s> %s, %s %s)", + " %s = call <8 x float> @%s(" + "<%d x %s> %s, <%d x %s> %s, <8 x float> %s)", mma_result_name(L, op), - dst_ty, spec->intrinsic, w, spec->call_elt, @@ -608,7 +598,6 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id w, spec->call_elt, b_arg, - src2_ty, rocke_ll_operand(L, c)); } @@ -620,17 +609,16 @@ static void _emit_wmma(rocke_lower_t* L, const rocke_op_t* op, const char* op_id * guards (live + exactly 3 operands), tracks one intrinsic decl key, optionally * prepends a fixed two-line operand bitcast (the bf16 `_1k` atoms widen * <4 x bfloat> -> <4 x i16>), then emits a single MFMA call. They differ only - * by five literal strings (decl key, intrinsic, src0/src1 element-vector - * spelling, src2 spelling, dst spelling) plus the optional bitcast target. This + * by four literal strings (decl key, intrinsic, A/B element-vector spelling, + * accumulator/result vector spelling) plus the optional bitcast target. This * mirrors the WMMA_SPECS / _emit_wmma idiom already used in this file. */ typedef struct _mfma_spec { const char* op_id; /* the tile.mma op_id (no "tile." prefix) */ const char* decl_key; /* _need() key */ const char* intrinsic; /* fully-mangled @llvm.amdgcn.mfma.... */ - const char* src01_ty; /* src0/src1 SSA operand type spelling */ - const char* src2_ty; /* src2 vector spelling */ - const char* dst_ty; /* dst vector spelling */ + const char* ab_ty; /* A/B SSA operand type spelling */ + const char* acc_ty; /* accumulator/result vector spelling */ const char* bitcast_to; /* operand bitcast target (NULL = no bitcast) */ } _mfma_spec_t; @@ -640,14 +628,12 @@ static const _mfma_spec_t MFMA_SPECS[] = { "llvm.amdgcn.mfma.f32.16x16x16f16", "<4 x half>", "<4 x float>", - "<4 x float>", NULL}, {"mfma_f32_16x16x32_f16", "mfma.f32.16x16x32.f16", "llvm.amdgcn.mfma.f32.16x16x32.f16", "<8 x half>", "<4 x float>", - "<4 x float>", NULL}, /* bf16 `_1k`: bitcast <4 x bfloat> -> <4 x i16> before the call. */ {"mfma_f32_16x16x16_bf16", @@ -655,21 +641,18 @@ static const _mfma_spec_t MFMA_SPECS[] = { "llvm.amdgcn.mfma.f32.16x16x16bf16.1k", "<4 x bfloat>", "<4 x float>", - "<4 x float>", "<4 x i16>"}, {"mfma_f32_16x16x32_bf16", "mfma.f32.16x16x32.bf16", "llvm.amdgcn.mfma.f32.16x16x32.bf16", "<8 x bfloat>", "<4 x float>", - "<4 x float>", NULL}, {"mfma_f32_32x32x8_f16", "mfma.f32.32x32x8f16", "llvm.amdgcn.mfma.f32.32x32x8f16", "<4 x half>", "<16 x float>", - "<16 x float>", NULL}, /* bf16 `_1k`: bitcast <4 x bfloat> -> <4 x i16> exactly like 16x16x16. */ {"mfma_f32_32x32x8_bf16", @@ -677,7 +660,6 @@ static const _mfma_spec_t MFMA_SPECS[] = { "llvm.amdgcn.mfma.f32.32x32x8bf16.1k", "<4 x bfloat>", "<16 x float>", - "<16 x float>", "<4 x i16>"}, /* fp32 (TF32-class) scalar atoms: A/B are single floats per lane. */ {"mfma_f32_16x16x4_f32", @@ -685,35 +667,30 @@ static const _mfma_spec_t MFMA_SPECS[] = { "llvm.amdgcn.mfma.f32.16x16x4f32", "float", "<4 x float>", - "<4 x float>", NULL}, {"mfma_f32_32x32x2_f32", "mfma.f32.32x32x2f32", "llvm.amdgcn.mfma.f32.32x32x2f32", "float", "<16 x float>", - "<16 x float>", NULL}, {"mfma_f32_32x32x16_f16", "mfma.f32.32x32x16.f16", "llvm.amdgcn.mfma.f32.32x32x16.f16", "<8 x half>", "<16 x float>", - "<16 x float>", NULL}, {"mfma_f32_32x32x16_bf16", "mfma.f32.32x32x16.bf16", "llvm.amdgcn.mfma.f32.32x32x16.bf16", "<8 x bfloat>", "<16 x float>", - "<16 x float>", NULL}, {"mfma_f32_4x4x4_f16", "mfma.f32.4x4x4f16", "llvm.amdgcn.mfma.f32.4x4x4f16", "<4 x half>", "<4 x float>", - "<4 x float>", NULL}, }; static const int MFMA_SPECS_N = (int)(sizeof(MFMA_SPECS) / sizeof(MFMA_SPECS[0])); @@ -752,13 +729,13 @@ static void _emit_mfma(rocke_lower_t* L, const rocke_op_t* op, const _mfma_spec_ rocke_ll_emitf(L, " %s = bitcast %s %s to %s", a_cast, - spec->src01_ty, + spec->ab_ty, rocke_ll_operand(L, a), spec->bitcast_to); rocke_ll_emitf(L, " %s = bitcast %s %s to %s", b_cast, - spec->src01_ty, + spec->ab_ty, rocke_ll_operand(L, b), spec->bitcast_to); a_arg = a_cast; @@ -769,20 +746,20 @@ static void _emit_mfma(rocke_lower_t* L, const rocke_op_t* op, const _mfma_spec_ { a_arg = rocke_ll_operand(L, a); b_arg = rocke_ll_operand(L, b); - call_ty = spec->src01_ty; + call_ty = spec->ab_ty; } rocke_ll_emitf(L, " %s = call %s @%s(" "%s %s, %s %s, %s %s, i32 0, i32 0, i32 0)", mma_result_name(L, op), - spec->dst_ty, + spec->acc_ty, spec->intrinsic, call_ty, a_arg, call_ty, b_arg, - spec->src2_ty, + spec->acc_ty, rocke_ll_operand(L, c)); } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h index 1c965ae2b045..12c52d012c2b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/arch_target.h @@ -58,14 +58,15 @@ extern "C" { /* ============================== layout map ============================== */ -/* MMA fragment role. Mirrors LayoutMap.role ("acc"/"a"/"b"). The accumulator is - * spelled "c" in the C op_id maps (MmaOp.c_layout) but role text is "acc"/"a"/ - * "b" exactly as Python LayoutMap.role. */ +/* MMA fragment role. Matrix roles mirror LayoutMap.role ("acc"/"a"/"b"); + * scale roles are "a_scale"/"b_scale" and use logical K-group coordinates. */ typedef enum rocke_mma_role { ROCKE_MMA_ROLE_ACC = 0, /* accumulator C/D: coords (row, col) */ ROCKE_MMA_ROLE_A, /* A operand: coords (row, k) */ - ROCKE_MMA_ROLE_B /* B operand: coords (k, col) */ + ROCKE_MMA_ROLE_B, /* B operand: coords (k, col) */ + ROCKE_MMA_ROLE_A_SCALE, /* A scale: coords (row, K-group) */ + ROCKE_MMA_ROLE_B_SCALE /* B scale: coords (K-group, col) */ } rocke_mma_role_t; /* The lane/slot -> tile-coordinate emitter. Given the builder, a runtime i32 @@ -145,21 +146,30 @@ typedef struct rocke_mma_op const char* a_scale_dtype; /* NULL when unscaled */ const char* b_scale_dtype; /* NULL when unscaled */ rocke_mma_scale_block_k_t scale_block_k; /* shared by A/B; NONE when unscaled */ + /* Logical scale elements per lane, independent of the packed carrier. + * Appended ABI fields: native consumers must rebuild. */ + int a_scale_frag_len; + int b_scale_frag_len; + const rocke_layout_map_t* a_scale_layout; /* may be NULL */ + const rocke_layout_map_t* b_scale_layout; /* may be NULL */ } rocke_mma_op_t; /* MmaOp.shape -> (m, n, k) via out params. */ void rocke_mma_op_shape(const rocke_mma_op_t* op, int* m, int* n, int* k); -/* Physical-layout accessors. Each returns the verified map for the role, or NULL - * when none is registered. Unlike Python (which raises NotImplementedError), the - * C getters return NULL and -- when `b` is non-NULL -- set the builder's sticky - * error (ROCKE_ERR_NOTIMPL) with the same message text, so callers can either - * check NULL or rely on the sticky-fail builder. Pass b=NULL for a pure lookup. - * Mirrors MmaOp.a_layout / b_layout / c_layout / acc_layout. */ +/* Physical-layout accessors. Missing maps raise ckc::Error with + * ROCKE_ERR_NOTIMPL, like Python's NotImplementedError. The enclosing public + * builder/lowering boundary translates that exception into status + message; + * direct C++ callers must catch it, including when b is NULL. A NULL op returns + * NULL. Mirrors MmaOp's matrix and scale layout accessors. */ const rocke_layout_map_t* rocke_mma_op_a_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b); const rocke_layout_map_t* rocke_mma_op_b_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b); const rocke_layout_map_t* rocke_mma_op_c_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b); const rocke_layout_map_t* rocke_mma_op_acc_layout(const rocke_mma_op_t* op, rocke_ir_builder_t* b); +const rocke_layout_map_t* rocke_mma_op_a_scale_layout(const rocke_mma_op_t* op, + rocke_ir_builder_t* b); +const rocke_layout_map_t* rocke_mma_op_b_scale_layout(const rocke_mma_op_t* op, + rocke_ir_builder_t* b); /* ====================== memory caps / resource limits ================== */ @@ -278,6 +288,10 @@ int rocke_arch_mma_c_frag_len(const char* op_id); * interned canonical key (e.g. ROCKE_DTYPE_I32 / "fp32"); do not free. */ const char* rocke_arch_mma_op_id_c_dtype(const char* op_id); +/* Target-independent family projection. Unknown/NULL IDs return NULL; + * conflicting families for one ID raise ROCKE_ERR_VALUE. */ +const char* rocke_arch_mma_op_id_family(const char* op_id); + /* ============================== arch target =========================== */ /* Hardware-facts surface for one gfx target. Frozen; cheap to pass around. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h index 01041b53fa24..cfaaa125c78e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h @@ -68,7 +68,10 @@ static inline rocke_scaled_wmma_op_t rocke_scaled_wmma_contract(const rocke_mma_ || strcmp(atom->c_dtype, "fp32") != 0 || atom->m != 16 || atom->n != 16 || atom->k != 128) ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA backend contract"); spec.scales.block_k = atom->scale_block_k; - spec.scales.count = atom->k / spec.scales.block_k; + const int count = atom->k / spec.scales.block_k; + if(atom->a_scale_frag_len != count || atom->b_scale_frag_len != count) + ckc::raise_status(ROCKE_ERR_VALUE, "unsupported scaled WMMA scale fragment lengths"); + spec.scales.count = atom->a_scale_frag_len; char suffix[96]; snprintf(suffix, sizeof(suffix), diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md index 3c328f9c084e..3cd67081d8c7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md @@ -37,7 +37,23 @@ and `scale_block_k` fields describe independent scale types and a shared K-group size. `MmaScaleBlockK.K16` and `MmaScaleBlockK.K32` are the only scaled sizes; all three fields are `None` for unscaled atoms. Integer 16/32 values from JSON are normalized to the enum. Scale types are independent of matrix dtypes and -packed register types. Fragment and layout accessors retain their A/B/C roles. +packed register types. Matrix fragment and layout accessors retain their A/B/C roles. + +`a_scale_frag_len` and `b_scale_frag_len` count logical scale elements per lane. +`a_scale_layout()` maps `(lane, slot)` to `(row, K-group)`; +`b_scale_layout()` maps it to `(K-group, column)`. These instruction-local maps +are separate from register byte packing and global tensor strides. Unscaled +atoms default to zero scale fragments and absent maps. An unavailable map raises +`NotImplementedError`, including for a scaled atom whose layout is not verified. +The native record appends corresponding counts and map pointers, so native +consumers must rebuild; existing role enum values remain unchanged. + +The four gfx1250 FP8/BF8 scaled atoms provide these scale maps: lane `l` and +slot `j` map to `(l % 16, j)` for A and `(j, l % 16)` for B. Both half-waves +duplicate the scales. K32 uses four E8M0 elements per lane packed into i32; +K16 uses eight packed into i64, with slot `j` at bit offset `8*j`. Matrix A/B +maps for these atoms remain unavailable. The gfx1250 loader applies tile and +instruction-step offsets to the scale coordinates before applying tensor strides. For scaled operations, query the complete contract and pass the selected atom to `IRBuilder.mma`: diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py index f47dc724c5ee..20543dc19159 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py @@ -63,7 +63,9 @@ class LayoutMap: role ``"acc"`` (accumulator C/D, coords are ``(row, col)``), ``"a"`` (A operand, coords are ``(row, k)``) or ``"b"`` (B operand, coords are - ``(k, col)``). + ``(k, col)``), ``"a_scale"`` (``(row, K-group)``), or + ``"b_scale"`` (``(K-group, col)``). Scale slots count logical elements, + independently of the backend register carrier. frag_len Number of fragment slots per lane for this role (the per-lane vector length: e.g. 4 for an MFMA 16x16x16 accumulator, 8 for the WMMA @@ -164,6 +166,15 @@ class MmaOp: b_scale_dtype: str | None = None scale_block_k: MmaScaleBlockK | None = None + a_scale_frag_len: int = 0 + b_scale_frag_len: int = 0 + _a_scale_layout: Optional[LayoutMap] = field( + default=None, repr=False, compare=False + ) + _b_scale_layout: Optional[LayoutMap] = field( + default=None, repr=False, compare=False + ) + def __post_init__(self) -> None: a, b, block_k = _normalize_mma_scales( self.a_scale_dtype, self.b_scale_dtype, self.scale_block_k @@ -171,6 +182,18 @@ def __post_init__(self) -> None: object.__setattr__(self, "a_scale_dtype", a) object.__setattr__(self, "b_scale_dtype", b) object.__setattr__(self, "scale_block_k", block_k) + for role in ("a_scale", "b_scale"): + count = getattr(self, f"{role}_frag_len") + layout = getattr(self, f"_{role}_layout") + if type(count) is not int or count < 0 or (block_k is None and count): + raise ValueError(f"invalid {role} fragment length: {count!r}") + if layout is not None and ( + not count + or layout.role != role + or layout.frag_len != count + or layout.wave_size != self.wave_size + ): + raise ValueError(f"{role} layout does not match its fragment metadata") @property def shape(self) -> Tuple[int, int, int]: @@ -189,6 +212,14 @@ def c_layout(self) -> LayoutMap: """The accumulator (C/D) ``(row, col)`` lane/slot -> coordinate map.""" return self._require_layout(self._c_layout, "c") + def a_scale_layout(self) -> LayoutMap: + """A scale ``(row, K-group)`` coordinates for each logical scale slot.""" + return self._require_layout(self._a_scale_layout, "a_scale") + + def b_scale_layout(self) -> LayoutMap: + """B scale ``(K-group, col)`` coordinates for each logical scale slot.""" + return self._require_layout(self._b_scale_layout, "b_scale") + # Convenience aliases for the accumulator (the most-used map). def acc_layout(self) -> LayoutMap: return self.c_layout() @@ -574,10 +605,20 @@ def _wmma_gfx1250_b_16x16x32(builder, lane, slot): return k, col +def _wmma_gfx1250_a_scale(builder, lane, slot): + row = builder.mod(lane, builder.const_i32(16)) + return row, builder.const_i32(slot) + + +def _wmma_gfx1250_b_scale(builder, lane, slot): + col = builder.mod(lane, builder.const_i32(16)) + return builder.const_i32(slot), col + + @dataclass(frozen=True) class _FragInfo: """Per-op_id fragment metadata: per-lane vector lengths, wave size, and the - (optional) lane/slot coordinate functions for A / B / accumulator.""" + (optional) lane/slot coordinate functions for matrices and scales.""" a_frag_len: int b_frag_len: int @@ -586,6 +627,10 @@ class _FragInfo: a_fn: _LaneCoordFn = None b_fn: _LaneCoordFn = None c_fn: _LaneCoordFn = None + a_scale_frag_len: int = 0 + b_scale_frag_len: int = 0 + a_scale_fn: _LaneCoordFn = None + b_scale_fn: _LaneCoordFn = None # op_id -> physical fragment metadata. Frag lengths are populated for every @@ -766,6 +811,10 @@ class _FragInfo: None, None, _wmma_gfx12_acc_16x16, + a_scale_frag_len=4, + b_scale_frag_len=4, + a_scale_fn=_wmma_gfx1250_a_scale, + b_scale_fn=_wmma_gfx1250_b_scale, ), "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k32": _FragInfo( 16, @@ -775,6 +824,10 @@ class _FragInfo: None, None, _wmma_gfx12_acc_16x16, + a_scale_frag_len=4, + b_scale_frag_len=4, + a_scale_fn=_wmma_gfx1250_a_scale, + b_scale_fn=_wmma_gfx1250_b_scale, ), "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16": _FragInfo( 16, @@ -784,6 +837,10 @@ class _FragInfo: None, None, _wmma_gfx12_acc_16x16, + a_scale_frag_len=8, + b_scale_frag_len=8, + a_scale_fn=_wmma_gfx1250_a_scale, + b_scale_fn=_wmma_gfx1250_b_scale, ), "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16": _FragInfo( 16, @@ -793,6 +850,10 @@ class _FragInfo: None, None, _wmma_gfx12_acc_16x16, + a_scale_frag_len=8, + b_scale_frag_len=8, + a_scale_fn=_wmma_gfx1250_a_scale, + b_scale_fn=_wmma_gfx1250_b_scale, ), "wmma_gfx1250_f32_16x16x32_bf16": _FragInfo( 16, @@ -1082,6 +1143,22 @@ def _op_id_c_dtype() -> Dict[str, str]: return out +@lru_cache(maxsize=1) +def _op_id_family() -> Dict[str, str]: + """Resolve operation families without selecting a target or parsing IDs.""" + out: Dict[str, str] = {} + for row in _load_specs().values(): + for op in row["mma"]: + op_id, family = op["op_id"], op["family"] + previous = out.setdefault(op_id, family) + if previous != family: + raise ValueError( + f"arch SSOT drift: op_id {op_id!r} has inconsistent family " + f"across arches ({previous!r} vs {family!r})" + ) + return out + + def _build_mma_op(o: dict) -> MmaOp: """Construct an :class:`MmaOp` from one catalog JSON row, attaching the physical fragment lengths and layout maps registered for its op_id.""" @@ -1112,6 +1189,10 @@ def _mk(role: str, frag_len: int, fn: _LaneCoordFn) -> Optional[LayoutMap]: a_scale_dtype=o.get("a_scale_dtype"), b_scale_dtype=o.get("b_scale_dtype"), scale_block_k=o.get("scale_block_k"), + a_scale_frag_len=info.a_scale_frag_len, + b_scale_frag_len=info.b_scale_frag_len, + _a_scale_layout=_mk("a_scale", info.a_scale_frag_len, info.a_scale_fn), + _b_scale_layout=_mk("b_scale", info.b_scale_frag_len, info.b_scale_fn), ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py index 848f4ceac1d7..db247a74c0a8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py @@ -92,11 +92,14 @@ def gfx1250_scaled_wmma(op_id: str) -> ScaledWmmaOp | None: or atom.shape != (16, 16, 128) ): raise ValueError(f"unsupported scaled WMMA backend contract: {atom.op_id}") + counts = (atom.a_scale_frag_len, atom.b_scale_frag_len) + if any(count != atom.k // atom.scale_block_k for count in counts): + raise ValueError(f"unsupported scaled WMMA scale fragment lengths: {counts}") return ScaledWmmaOp( atom=atom, matrix_formats=(formats[atom.a_dtype], formats[atom.b_dtype]), scale_formats=(0, 0), # E8M0 for each source. scales=E8M0ScalePacking( - count=atom.k // atom.scale_block_k, block_k=atom.scale_block_k + count=atom.a_scale_frag_len, block_k=atom.scale_block_k ), ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py index 55c74be31d00..8789d022f978 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py @@ -1857,12 +1857,11 @@ def mma( c_dtype = getattr(op, "c_dtype", None) is_int_acc = c_dtype == "i32" if c_dtype is not None else _mma_c_is_int(op_id) c_elem = I32 if is_int_acc else F32 - from .arch import ArchTarget + from .arch.target import _op_id_family - atom = ArchTarget.from_gfx("gfx1250").mma.by_op_id(op_id) hint = ( "mxacc" - if atom is not None and atom.family == "wmma_scaled" + if _op_id_family().get(op_id) == "wmma_scaled" else _MMA_RESULT_HINT.get(op_id, "acc") ) return self._op( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py index 44c2ddec18e9..f0628a9fdee8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py @@ -28,6 +28,7 @@ VectorType, ) + _HIP_TYPE = { "i1": "bool", "i8": "int8_t", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py index e3249b1c64e3..4743a1335ea8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py @@ -56,6 +56,7 @@ split_loc, ) + # Datalayout / triple. Copied verbatim from clang's output for the same # target on this box: clang -target amdgcn-amd-amdhsa -mcpu=gfx950 # -emit-llvm -S. The string is LLVM-version-keyed, not gfx-keyed: every diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md index 30d8c347b06d..1be444bf4917 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/README.md @@ -28,3 +28,15 @@ It is not a general scalar IR type or conversion API. Native scaled WMMA uses one `wmma_scaled` catalog family. Its operation IDs encode the matrix source and accumulator dtype, each source's scale format, and a shared K-group size; the backend selects the LLVM intrinsic and packed operand types from that contract. + +The native loader uses the atom's `a_scale_layout()` and `b_scale_layout()` for +lane ownership. Global scale tensors have shapes `[M, K/block_k]` and +`[K/block_k, N]`, respectively. K32 packs four E8M0 bytes per lane into i32; +K16 packs eight into i64, first K group in the low byte. Both half-waves carry +the same scales. Matrix A/B lane maps for these scaled atoms are not yet exposed. + +CPU tests independently check every scale coordinate, multi-tile addresses and +packed byte order. The opt-in numerical suite exercises both encodings and +scale group sizes, one-sided/group-isolated inputs, multiple tiles and K steps, +and HIP/COMGR compilation. These cases bound the validation; dtype recognition +alone does not establish support for other operand contracts. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py index 36f4d9da818e..86252f0c3c9f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/gfx1250/gemm/block_scaled_gemm_verify.py @@ -67,10 +67,8 @@ def reference_result( sa = decode_e8m0(a_scale) if native else a_scale.astype(np.float64) sb = decode_e8m0(b_scale) if native else b_scale.astype(np.float64) - a_values = a.astype(np.float64) - b_values = b.astype(np.float64) - scaled_a = a_values * np.repeat(sa, block_k, axis=1) - scaled_b = b_values * np.repeat(sb.T, block_k, axis=1) + scaled_a = a.astype(np.float64) * np.repeat(sa, block_k, axis=1) + scaled_b = b.astype(np.float64) * np.repeat(sb.T, block_k, axis=1) ref = scaled_a @ scaled_b.T if native: ref = ref.astype(ml_dtypes.bfloat16) @@ -102,9 +100,8 @@ def make_case_inputs( lowbit_types[spec.dtype_b] ) if native: - max_exponent = 131 - sa = rng.integers(125, max_exponent, size=(spec.M, groups), dtype=np.uint8) - sb = rng.integers(125, max_exponent, size=(groups, spec.N), dtype=np.uint8) + sa = rng.integers(125, 131, size=(spec.M, groups), dtype=np.uint8) + sb = rng.integers(125, 131, size=(groups, spec.N), dtype=np.uint8) neutral = 127 else: sa = rng.uniform(0.5, 1.5, size=(spec.M, groups)).astype(np.float32) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py index a5073169eacf..9ee88cbdeb4a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/gfx1250/block_scaled_gemm.py @@ -349,9 +349,8 @@ def build_block_scaled_gemm( n0 = ir.mul(ir.block_id_x(), c16) a_row = ir.add(m0, frag) # this lane's A row b_row = ir.add(n0, frag) # this lane's B row (= output col n) - storage_k = cK - a_base = ir.mul(a_row, storage_k) - b_base = ir.mul(b_row, storage_k) + a_base = ir.mul(a_row, cK) + b_base = ir.mul(b_row, cK) def _load_frag(ptr, base, storage_ty, k0): if not native_scale: @@ -385,8 +384,9 @@ def _load_frag(ptr, base, storage_ty, k0): packed = ir.vec_concat(packed, chunk) return ir.bitcast(packed, a_frag_ty) - def _pack_strided_scales(ptr, row_or_col, call_idx, *, for_b): - assert scale_op is not None + def _pack_strided_scales(ptr, call_idx, *, for_b): + assert scale_op is not None and atom is not None + layout = atom.b_scale_layout() if for_b else atom.a_scale_layout() packing = scale_op.scales count = packing.count word_ty = I64 if packing.word_bits == 64 else I32 @@ -394,14 +394,16 @@ def _pack_strided_scales(ptr, row_or_col, call_idx, *, for_b): packed = word_const(0) scale_groups = spec.K // spec.block_k for j in range(count): - scale_group = call_idx * count + j + coord0, coord1 = layout.coord(ir, lane, j) + group_offset = ir.const_i32(call_idx * (atom.k // spec.block_k)) if for_b: - idx = ir.add(ir.mul(ir.const_i32(scale_group), cN), row_or_col) + group = ir.add(group_offset, coord0) + col = ir.add(n0, coord1) + idx = ir.add(ir.mul(group, cN), col) else: - idx = ir.add( - ir.mul(row_or_col, ir.const_i32(scale_groups)), - ir.const_i32(scale_group), - ) + row = ir.add(m0, coord0) + group = ir.add(group_offset, coord1) + idx = ir.add(ir.mul(row, ir.const_i32(scale_groups)), group) byte = ir.global_load(ptr, idx, I8, align=1) widened = ir.zext(byte, word_ty) shift = word_const(j * packing.element_bits) @@ -415,8 +417,8 @@ def _pack_strided_scales(ptr, row_or_col, call_idx, *, for_b): k0 = step * _WMMA_SCALE_K a_frag = _load_frag(A, a_base, a_ty, k0) b_frag = _load_frag(B, b_base, b_ty, k0) - a_scale = _pack_strided_scales(AScale, a_row, step, for_b=False) - b_scale = _pack_strided_scales(BScale, b_row, step, for_b=True) + a_scale = _pack_strided_scales(AScale, step, for_b=False) + b_scale = _pack_strided_scales(BScale, step, for_b=True) acc = ir.mma(op_id, a_frag, b_frag, acc, a_scale, b_scale) out_col = ir.add(n0, frag) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp index 89ad55845ded..4ef24a30488c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/future_intrinsic_lowering.cpp @@ -888,6 +888,59 @@ void case_opcode_names_are_aligned() } } +/* Malformed result lists must reach the public lowering error boundary. */ +void case_wmma_result_count() +{ + for(bool integer : {false, true}) + { + for(int count : {0, 2, 1}) + { + rocke_ir_builder_t b; + if(rocke_ir_builder_init(&b, "wmma_result_count") != ROCKE_OK) + { + fail("rocke_ir_builder_init", __LINE__); + return; + } + auto* a = rocke_b_zero_vec(&b, integer ? rocke_i32() : rocke_f16(), integer ? 4 : 16); + auto* c = rocke_b_zero_vec(&b, integer ? rocke_i32() : rocke_f32(), 8); + auto* d = rocke_b_mma(&b, + integer ? "wmma_i32_16x16x16_iu8" : "wmma_f32_16x16x16_f16", + a, + a, + c, + nullptr, + 0); + rocke_value_t* results[] = {d, d}; + d->op->num_results = count; + d->op->results = count ? results : nullptr; + rocke_b_ret(&b); + char* out = nullptr; + char err[ROCKE_ERR_MSG_CAP] = {}; + const auto status = rocke_lower_kernel_to_llvm_ex( + b.kernel, ROCKE_LLVM_FLAVOR_LLVM23, "gfx1151", &out, err, sizeof(err)); + if(count == 1) + { + if(status != ROCKE_OK || !out) + fail("valid WMMA failed", __LINE__); + else + EXPECT_IR(std::string(out), integer ? "call <8 x i32>" : "call <8 x float>"); + } + else + { + char expected[80]; + snprintf(expected, + sizeof(expected), + "tile.mma: expected exactly one result, got %d", + count); + if(status != ROCKE_ERR_VALUE || out || strcmp(err, expected) != 0) + fail("malformed WMMA did not return its result-count error", __LINE__); + } + std::free(out); + rocke_ir_builder_free(&b); + } + } +} + struct TestCase { const char* name; @@ -895,6 +948,7 @@ struct TestCase }; const TestCase k_cases[] = { + {"wmma_result_count", case_wmma_result_count}, {"ds_swizzle_raw_offset", case_ds_swizzle_raw_offset}, {"quad_perm_hip", case_quad_perm_hip}, {"quad_perm_rejects_invalid_input", case_quad_perm_rejects_invalid_input}, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py index 9e1a5eb4d013..d0894160bf16 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py @@ -23,6 +23,7 @@ from rocke.core.arch.target import ( _load_specs, _op_id_c_dtype, + _op_id_family, normalize_dtype, ) @@ -262,3 +263,65 @@ def test_largest_k_only_rejects_ties_at_the_selected_k(): if __name__ == "__main__": # pragma: no cover unittest.main() + + +def test_op_id_family_matches_all_catalogs_and_rejects_conflicts(): + specs = _load_specs() + for row in specs.values(): + for op in row["mma"]: + assert _op_id_family()[op["op_id"]] == op["family"] + assert _op_id_family().get("unknown") is None + sample = next(op for row in specs.values() for op in row["mma"]) + drifted = {**specs, "synthetic": {"mma": [{**sample, "family": "conflicting"}]}} + _op_id_family.cache_clear() + try: + with mock.patch("rocke.core.arch.target._load_specs", return_value=drifted): + with pytest.raises(ValueError, match="inconsistent family"): + _op_id_family() + finally: + _op_id_family.cache_clear() + + +def test_mma_naming_without_target_lookup(): + from rocke.core.ir import F16, F32, I32, IRBuilder + + atoms = [ + ( + ArchTarget.from_gfx("gfx950").mma.by_op_id("mfma_f32_16x16x16_f16"), + F16, + "acc", + ), + ( + ArchTarget.from_gfx("gfx1151").mma.by_op_id("wmma_i32_16x16x16_iu8"), + I32, + "acc", + ), + ( + next( + op + for op in ArchTarget.from_gfx("gfx1250").mma.ops + if op.family == "wmma_scaled" + ), + I32, + "mxacc", + ), + ] + _op_id_family.cache_clear() + with mock.patch.object( + ArchTarget, + "from_gfx", + side_effect=AssertionError("target lookup in generic MMA"), + ): + for atom, elem, hint in atoms: + assert atom is not None + for arg in (atom, atom.op_id): + b = IRBuilder("neutral_mma") + a = b.zero_vec(elem, atom.a_frag_len) + c = b.zero_vec(I32 if atom.c_dtype == "i32" else F32, atom.c_frag_len) + extra = ( + (b.const_i32(0), b.const_i32(0)) + if atom.family == "wmma_scaled" + else () + ) + result = b.mma(arg, a, a, c, *extra) + assert result.name.lstrip("%").startswith(hint) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py index 31cb77ec09c5..5e66d0728d75 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_gfx1250_scaled_wmma.py @@ -67,6 +67,50 @@ def test_backend_rejects_unsupported_scale_type_on_either_input(self): ): gfx1250_scaled_wmma(atom.op_id) + def test_scale_fragment_counts_are_checked_for_each_source(self): + catalog = ArchTarget.from_gfx("gfx1250").mma + for role in ("a_scale", "b_scale"): + for count in (0, 8): + atom = replace( + _scaled_atom(), + **{ + f"{role}_frag_len": count, + f"_{role}_layout": None, + }, + ) + with ( + self.subTest(role=role, count=count), + mock.patch.object(catalog, "by_op_id", return_value=atom), + self.assertRaisesRegex(ValueError, "scale fragment lengths"), + ): + gfx1250_scaled_wmma(atom.op_id) + + def test_loader_requires_both_scale_maps(self): + for role in ("a_scale", "b_scale"): + atom = replace(_scaled_atom(), **{f"_{role}_layout": None}) + spec = BlockScaledGemmSpec( + "missing_scale_map", + M=16, + N=16, + K=128, + block_k=32, + scale_dtype="e8m0", + matrix_path="wmma_scale", + ) + with ( + mock.patch( + "rocke.instances.gfx1250.block_scaled_gemm._native_scaled_atom", + return_value=atom, + ), + self.assertRaisesRegex(NotImplementedError, role), + ): + build_block_scaled_gemm(spec, arch="gfx1250") + # Lowering prepacked operands needs the carrier contract, not a loader map. + with mock.patch.object( + ArchTarget.from_gfx("gfx1250").mma, "by_op_id", return_value=atom + ): + self.assertEqual(gfx1250_scaled_wmma(atom.op_id).scales.count, 4) + def test_backend_derives_each_matrix_carrier_and_selector(self): # Synthetic metadata exercises the lowering contract without adding atoms. atom = replace(_scaled_atom(), b_dtype="bf8e5m2", b_frag_len=8) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp index 3d4763db4c29..8824bce06d0d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp @@ -182,9 +182,78 @@ static int test_scale_contracts() return 0; } +static int test_scale_layouts_and_families() +{ + CHECK(!rocke_arch_mma_op_id_family(NULL)); + CHECK(!rocke_arch_mma_op_id_family("unknown")); + int num_arches = 0; + const char* const* arches = rocke_known_arches(&num_arches); + int scaled = 0; + for(int i = 0; i < num_arches; ++i) + { + const auto* arch = rocke_arch_target_from_gfx(arches[i]); + for(int j = 0; j < arch->mma.num_ops; ++j) + { + const auto* op = &arch->mma.ops[j]; + CHECK(strcmp(rocke_arch_mma_op_id_family(op->op_id), op->family) == 0); + if(!op->a_scale_dtype) + { + CHECK(op->a_scale_frag_len == 0 && op->b_scale_frag_len == 0); + CHECK(!op->a_scale_layout && !op->b_scale_layout); + for(auto getter : {rocke_mma_op_a_scale_layout, rocke_mma_op_b_scale_layout}) + { + bool rejected = false; + try + { + getter(op, NULL); + } + catch(const ckc::Error& e) + { + rejected = e.code() == ROCKE_ERR_NOTIMPL; + } + CHECK(rejected); + } + continue; + } + ++scaled; + CHECK(strcmp(arches[i], "gfx1250") == 0); + const int count = op->scale_block_k == ROCKE_MMA_SCALE_K16 ? 8 : 4; + CHECK(op->a_scale_frag_len == count && op->b_scale_frag_len == count); + for(int source = 0; source < 2; ++source) + { + const auto* map = source ? rocke_mma_op_b_scale_layout(op, NULL) + : rocke_mma_op_a_scale_layout(op, NULL); + CHECK(map && map->frag_len == count && map->wave_size == 32); + CHECK(map->role == (source ? ROCKE_MMA_ROLE_B_SCALE : ROCKE_MMA_ROLE_A_SCALE)); + for(int slot = -1; slot <= count; ++slot) + { + rocke_ir_builder_t b; + CHECK(rocke_ir_builder_init(&b, "scale_layout") == ROCKE_OK); + auto* lane = rocke_b_thread_id_x(&b); + rocke_value_t *x = NULL, *y = NULL; + const bool ok = rocke_layout_map_coord(map, &b, lane, slot, &x, &y); + if(slot < 0 || slot == count) + { + CHECK(!ok && !x && !y && b.status == ROCKE_ERR_VALUE); + CHECK(strstr(b.err, source ? "'b_scale'" : "'a_scale'")); + } + else + CHECK(ok && x && y && b.status == ROCKE_OK); + rocke_ir_builder_free(&b); + } + auto invalid = *op; + (source ? invalid.b_scale_frag_len : invalid.a_scale_frag_len) = count / 2; + CHECK(rejects_query([&] { rocke_scaled_wmma_contract(&invalid); })); + } + } + } + CHECK(scaled == 4); + return 0; +} + int main() { - if(test_scale_contracts()) + if(test_scale_contracts() || test_scale_layouts_and_families()) return 1; int checked = 0; for(const char* gfx : {"gfx950", "gfx1250"}) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_scale_layouts.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_scale_layouts.py new file mode 100644 index 000000000000..42b2dcbbe176 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_scale_layouts.py @@ -0,0 +1,174 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Independent lane-coordinate, tensor-address and packed-scale oracles.""" + +from collections import Counter +from dataclasses import replace +import operator + +import pytest + +from rocke.core.arch import ArchTarget, known_arches +from rocke.core.ir import IRBuilder +from rocke.instances.gfx1250.block_scaled_gemm import ( + BlockScaledGemmSpec, + build_block_scaled_gemm, +) + + +def _atom(dtype, block_k): + return ArchTarget.from_gfx("gfx1250").mma.op_for_shape( + family="wmma_scaled", + a_dtype=dtype, + b_dtype=dtype, + c_dtype="fp32", + m=16, + n=16, + k=128, + scales=("e8m0", "e8m0", block_k), + ) + + +def _evaluate(value, *, lane=0, tile=(0, 0), loads=None): + """Evaluate only the integer SSA dependency graph, never the MMA itself.""" + op = value.op + if op.name == "arith.constant": + return op.attrs["value"] + if op.name == "gpu.thread_id": + return lane + if op.name == "gpu.block_id": + return tile[0 if op.attrs["axis"] == "x" else 1] + + def evaluate(v): + return _evaluate(v, lane=lane, tile=tile, loads=loads) + + if op.name == "memref.global_load_typed": + ptr, index = op.operands + offset = evaluate(index) + role = ptr.name.lstrip("%") + assert role in ("A_scale", "B_scale") + loads.append((role, offset)) + # Distinct address-sensitive bytes expose stride, group and order bugs. + return (offset * 37 + (11 if role == "A_scale" else 73)) % 256 + if op.name == "arith.zext": + return evaluate(op.operands[0]) & 255 + operations = { + "arith.add": operator.add, + "arith.mul": operator.mul, + "arith.div": operator.floordiv, + "arith.mod": operator.mod, + "arith.shl": operator.lshift, + "arith.or": operator.or_, + } + assert op.name in operations, op.name + return operations[op.name](*(evaluate(v) for v in op.operands)) + + +@pytest.mark.parametrize("dtype", ["fp8", "bf8"]) +@pytest.mark.parametrize("block_k", [16, 32]) +@pytest.mark.parametrize("role", ["a_scale", "b_scale"]) +def test_scale_coordinates(dtype, block_k, role): + atom = _atom(dtype, block_k) + layout = getattr(atom, f"{role}_layout")() + count = 128 // block_k + assert getattr(atom, f"{role}_frag_len") == count + assert (layout.role, layout.wave_size, layout.frag_len) == (role, 32, count) + b = IRBuilder("scale_coordinates") + lane = b.thread_id_x() + coords = [layout.coord(b, lane, slot) for slot in range(count)] + seen = Counter() + for l in range(32): + for j, pair in enumerate(coords): + actual = tuple(_evaluate(v, lane=l) for v in pair) + expected = (l % 16, j) if role == "a_scale" else (j, l % 16) + assert actual == expected + seen[actual] += 1 + assert len(seen) == 16 * count + assert set(seen.values()) == {2} + for slot in (-1, count): + with pytest.raises(ValueError, match="fragment slot"): + layout.coord(b, lane, slot) + + +def test_scale_defaults_and_unavailable_maps(): + for gfx in known_arches(): + for atom in ArchTarget.from_gfx(gfx).mma.ops: + if atom.a_scale_dtype is None: + assert atom.a_scale_frag_len == atom.b_scale_frag_len == 0 + for accessor in (atom.a_scale_layout, atom.b_scale_layout): + with pytest.raises(NotImplementedError, match="no verified"): + accessor() + atom = _atom("fp8", 32) + unknown = replace(atom, _a_scale_layout=None, _b_scale_layout=None) + with pytest.raises(NotImplementedError, match="a_scale"): + unknown.a_scale_layout() + with pytest.raises(NotImplementedError, match="b_scale"): + unknown.b_scale_layout() + # Adding scale maps does not invent unverified matrix operand maps. + for accessor in (atom.a_layout, atom.b_layout): + with pytest.raises(NotImplementedError): + accessor() + assert ( + replace( + atom, _a_scale_layout=replace(atom.a_scale_layout(), fn=lambda *_: None) + ) + == atom + ) + + +@pytest.mark.parametrize("role", ["a_scale", "b_scale"]) +def test_scale_metadata_rejects_inconsistent_maps(role): + atom = _atom("fp8", 32) + layout = getattr(atom, f"{role}_layout")() + for change in ({"role": "a"}, {"frag_len": 8}, {"wave_size": 64}): + with pytest.raises(ValueError, match="layout does not match"): + replace(atom, **{f"_{role}_layout": replace(layout, **change)}) + for count in (-1, True, 1.5): + with pytest.raises(ValueError, match="fragment length"): + replace(atom, **{f"{role}_frag_len": count}) + + +@pytest.mark.parametrize("dtype", ["fp8", "bf8"]) +@pytest.mark.parametrize("block_k", [16, 32]) +def test_instance_scale_addresses_and_packed_words(dtype, block_k): + spec = BlockScaledGemmSpec( + "scale_addresses", + M=32, + N=48, + K=256, + dtype_a=dtype, + dtype_b=dtype, + scale_dtype="e8m0", + block_k=block_k, + matrix_path="wmma_scale16" if block_k == 16 else "wmma_scale", + ) + kernel = build_block_scaled_gemm(spec, arch="gfx1250") + mmas = [op for op in kernel.body.ops if op.name == "tile.mma"] + assert len(mmas) == 2 + count = 128 // block_k + for x, y in ((0, 0), (2, 1)): + for lane in range(32): + for step, mma in enumerate(mmas): + for source, value in enumerate(mma.operands[3:]): + offsets = [ + ( + (16 * y + lane % 16) * (256 // block_k) + step * count + j + if source == 0 + else (step * count + j) * 48 + 16 * x + lane % 16 + ) + for j in range(count) + ] + role = "A_scale" if source == 0 else "B_scale" + expected = int.from_bytes( + bytes( + (offset * 37 + (11 if source == 0 else 73)) % 256 + for offset in offsets + ), + "little", + ) + loads = [] + assert ( + _evaluate(value, lane=lane, tile=(x, y), loads=loads) + == expected + ) + assert loads == [(role, offset) for offset in offsets] diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json index 5e2c928fa856..6d02abdb0076 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json @@ -3212,34 +3212,34 @@ }, "gemm/gfx1250/mxfp8/wmma_scale/bf8e5m2": { "arch": "gfx1250", - "bytes": 87339, + "bytes": 89055, "family": "gemm", "kernel_name": "irhash_mxfp8_bf8e5m2_wmma_scale_block_scaled_bf8_bf8_M32N48K256_bk32_t16x16x128_wmma_scale", - "sha256": "11d69f432274ae05c41449645050b93034a0d411811130cc5b057abc5c8834bc", + "sha256": "f9e29be4f431fcbebcdecf60db75024cbd55874a05c3b6d706a998c942335682", "status": "ok" }, "gemm/gfx1250/mxfp8/wmma_scale/fp8e4m3": { "arch": "gfx1250", - "bytes": 87339, + "bytes": 89055, "family": "gemm", "kernel_name": "irhash_mxfp8_fp8e4m3_wmma_scale_block_scaled_fp8_fp8_M32N48K256_bk32_t16x16x128_wmma_scale", - "sha256": "094da13dc1d3249da64c13185df2cfc44eb913fa940a38274d3bb7e1cc569948", + "sha256": "e1fcadf213c82d7941bcbd37561d3f8106841517a018d04239593678782fe284", "status": "ok" }, "gemm/gfx1250/mxfp8/wmma_scale16/bf8e5m2": { "arch": "gfx1250", - "bytes": 92233, + "bytes": 95649, "family": "gemm", "kernel_name": "irhash_mxfp8_bf8e5m2_wmma_scale16_block_scaled_bf8_bf8_M32N48K256_bk16_t16x16x128_wmma_scale16", - "sha256": "2ab30988dfa2bd120babd756d3d2d17bd444a1bbcabfe426ad99722bc8130209", + "sha256": "8ca807c7e2b4bc8b9de57841d00b7175a28de970197b10aefadb75612885b74c", "status": "ok" }, "gemm/gfx1250/mxfp8/wmma_scale16/fp8e4m3": { "arch": "gfx1250", - "bytes": 92233, + "bytes": 95649, "family": "gemm", "kernel_name": "irhash_mxfp8_fp8e4m3_wmma_scale16_block_scaled_fp8_fp8_M32N48K256_bk16_t16x16x128_wmma_scale16", - "sha256": "2e4e68d15ffa755e78bca8dd468eef8e345418cae2b5e5ace7b4761e2faf0f05", + "sha256": "43d60d203f40c6d191e8cbf67792cb2fdfe264eec0cbd891d5e19868709bab6c", "status": "ok" }, "gemm/gfx1250/t32x32x32": { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c index 66a947f45ecf..77993e4bd0dc 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.c @@ -344,6 +344,39 @@ static void build_tensor_transfers(rocke_ir_builder_t* b) rocke_b_ret(b); } +static void build_scale_coordinates(rocke_ir_builder_t* b, rocke_mma_scale_block_k_t block) +{ + const auto* arch = rocke_arch_target_from_gfx("gfx1250"); + const rocke_mma_scale_filter_t scales = {"e8m0", "e8m0", block}; + const auto* atom = rocke_mma_catalog_op_for_shape( + &arch->mma, "wmma_scaled", "fp8", "fp8", "fp32", 16, 16, 128, &scales); + rocke_value_t* out = frag_param(b, "coords", rocke_i32(), false); + rocke_value_t* lane = rocke_b_thread_id_x(b); + const rocke_layout_map_t* maps[] + = {rocke_mma_op_a_scale_layout(atom, b), rocke_mma_op_b_scale_layout(atom, b)}; + for(int source = 0; source < 2; ++source) + { + for(int slot = 0; slot < maps[source]->frag_len; ++slot) + { + rocke_value_t *x = NULL, *y = NULL; + rocke_layout_map_coord(maps[source], b, lane, slot, &x, &y); + rocke_b_global_store(b, out, lane, x, 1); + rocke_b_global_store(b, out, lane, y, 1); + } + } + rocke_b_ret(b); +} + +static void build_scale_coordinates_k32(rocke_ir_builder_t* b) +{ + build_scale_coordinates(b, ROCKE_MMA_SCALE_K32); +} + +static void build_scale_coordinates_k16(rocke_ir_builder_t* b) +{ + build_scale_coordinates(b, ROCKE_MMA_SCALE_K16); +} + typedef void (*build_fn_t)(rocke_ir_builder_t*); typedef struct config @@ -380,6 +413,8 @@ static const config_t CONFIGS[] = { {build_tensor_transfers, "gfx1250"}, {build_wmma_scale_bf8, "gfx1250"}, {build_wmma_scale16_bf8, "gfx1250"}, + {build_scale_coordinates_k32, "gfx1250"}, + {build_scale_coordinates_k16, "gfx1250"}, }; static const int NUM_CONFIGS = (int)(sizeof(CONFIGS) / sizeof(CONFIGS[0])); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py index eb7dffc65f1b..7dd837947469 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gfx1250_lowering_emit.py @@ -273,6 +273,34 @@ def build_tensor_transfers(b: IRBuilder) -> None: ) +def _scale_coordinates(block_k): + atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( + family="wmma_scaled", + a_dtype="fp8", + b_dtype="fp8", + c_dtype="fp32", + m=16, + n=16, + k=128, + scales=("e8m0", "e8m0", block_k), + ) + + def build(b): + out = b.param("coords", PtrType(I32, "global"), noalias=True, align=16) + lane = b.thread_id_x() + for layout in (atom.a_scale_layout(), atom.b_scale_layout()): + for slot in range(layout.frag_len): + x, y = layout.coord(b, lane, slot) + b.global_store(out, lane, x) + b.global_store(out, lane, y) + b.ret() + + return build + + +CONFIGS.extend((_scale_coordinates(block), "gfx1250") for block in (32, 16)) + + def _spec(idx: int): """Config selector: the (builder, arch) pair the shared driver expects.""" if not 0 <= idx < len(CONFIGS): From 0026b45992d0df58184dda254f515e6b93c2bf38 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:35:47 -0400 Subject: [PATCH 10/15] fix(rocke): cache cross-target MMA family lookup --- .../platform/cpp/core/arch/mma_family_index.h | 95 +++++++++++++++ .../rocke/platform/cpp/core/arch/query.cpp | 25 ++-- .../rocke/platform/tests/CMakeLists.txt | 5 +- .../tests/core/test_mma_catalog_lookup.cpp | 108 +++++++++++++++++- 4 files changed, 213 insertions(+), 20 deletions(-) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/mma_family_index.h diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/mma_family_index.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/mma_family_index.h new file mode 100644 index 000000000000..118690ade150 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/mma_family_index.h @@ -0,0 +1,95 @@ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +/* Private immutable counterpart of Python target._op_id_family(). Owns the + * sorted index; borrows IDs and families from the static architecture catalogs. */ +#ifndef ROCKE_MMA_FAMILY_INDEX_H +#define ROCKE_MMA_FAMILY_INDEX_H + +#include +#include + +#include "rocke/arch_target_internal.h" +#include "rocke/error.hpp" + +struct rocke_ati_mma_family_index +{ + rocke_ati_mma_family_index(const rocke_ati_arch_row_t* registry, int count) + { + size_t capacity = 0; + for(int i = 0; i < count; ++i) + if(registry[i].target) + capacity += registry[i].target->mma.num_ops; + if(!capacity) + return; + rows = static_cast(malloc(capacity * sizeof(row))); + if(!rows) + ckc::raise_status(ROCKE_ERR_OOM, "MMA family index allocation failed"); + for(int i = 0; i < count; ++i) + { + const auto* target = registry[i].target; + if(!target) + continue; + for(int j = 0; j < target->mma.num_ops; ++j) + { + const auto& op = target->mma.ops[j]; + rows[size++] = {op.op_id, op.family, false}; + } + } + qsort(rows, size, sizeof(row), [](const void* a, const void* b) { + return strcmp(static_cast(a)->op_id, static_cast(b)->op_id); + }); + size_t unique = 0; + for(size_t i = 0; i < size; ++i) + { + if(unique && strcmp(rows[unique - 1].op_id, rows[i].op_id) == 0) + rows[unique - 1].conflict |= strcmp(rows[unique - 1].family, rows[i].family) != 0; + else + rows[unique++] = rows[i]; + } + size = unique; + } + + ~rocke_ati_mma_family_index() + { + free(rows); + } + rocke_ati_mma_family_index(const rocke_ati_mma_family_index&) = delete; + rocke_ati_mma_family_index& operator=(const rocke_ati_mma_family_index&) = delete; + + const char* lookup(const char* op_id) const + { + if(!op_id) + return NULL; + size_t first = 0, last = size; + while(first < last) + { + const size_t mid = first + (last - first) / 2; + const int cmp = strcmp(op_id, rows[mid].op_id); + if(cmp < 0) + last = mid; + else if(cmp > 0) + first = mid + 1; + else + { + if(rows[mid].conflict) + ckc::raise_status( + ROCKE_ERR_VALUE, + "arch SSOT drift: op_id has inconsistent family across arches"); + return rows[mid].family; + } + } + return NULL; + } + +private: + struct row + { + const char* op_id; + const char* family; + bool conflict; + }; + row* rows = NULL; + size_t size = 0; +}; + +#endif /* ROCKE_MMA_FAMILY_INDEX_H */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp index a33de3abf308..27365b859cfb 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/query.cpp @@ -14,6 +14,8 @@ #include #include +#include "mma_family_index.h" + #include "rocke/arch_target.h" #include "rocke/arch_target_internal.h" #include "rocke/error.hpp" @@ -386,24 +388,11 @@ const char* rocke_arch_mma_op_id_family(const char* op_id) { if(!op_id) return NULL; - const char* family = NULL; - for(int i = 0; i < rocke_ati_arch_registry_len; ++i) - { - const auto* target = rocke_ati_arch_registry[i].target; - if(!target) - continue; - for(int j = 0; j < target->mma.num_ops; ++j) - { - const auto& op = target->mma.ops[j]; - if(strcmp(op.op_id, op_id) != 0) - continue; - if(family && strcmp(family, op.family) != 0) - ckc::raise_status(ROCKE_ERR_VALUE, - "arch SSOT drift: op_id has inconsistent family across arches"); - family = op.family; - } - } - return family; + // Function-local static initialization publishes the immutable index once, + // including when independent builders first query it concurrently. + static const rocke_ati_mma_family_index index(rocke_ati_arch_registry, + rocke_ati_arch_registry_len); + return index.lookup(op_id); } const char* rocke_arch_mma_op_id_c_dtype(const char* op_id) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt b/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt index da2fff71df97..8a27e62f01ef 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt @@ -26,7 +26,10 @@ if(BUILD_TESTING) set_property(GLOBAL APPEND PROPERTY ROCKE_INSTALL_TEST_TARGETS rocke_dtypes) add_executable(rocke_mma_catalog_lookup core/test_mma_catalog_lookup.cpp) - target_link_libraries(rocke_mma_catalog_lookup PRIVATE rocke_core) + find_package(Threads REQUIRED) + target_link_libraries(rocke_mma_catalog_lookup PRIVATE rocke_core Threads::Threads) + target_include_directories(rocke_mma_catalog_lookup PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/../cpp/core/arch") add_test(NAME rocke_mma_catalog_lookup COMMAND rocke_mma_catalog_lookup) set_tests_properties(rocke_mma_catalog_lookup PROPERTIES LABELS "ckc;unit;host") diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp index 8824bce06d0d..279c843a1ab2 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp @@ -2,13 +2,16 @@ // SPDX-License-Identifier: MIT /* Exercise public catalog queries against stored keys, without normalizing * away malformed table entries. Alias queries must find the same atom. */ +#include "mma_family_index.h" #include "rocke/arch_target.h" #include "rocke/error.hpp" #include "rocke/wmma_scale_internal.h" +#include #include #include #include +#include static const char* short_dtype(const char* dtype) { @@ -49,6 +52,107 @@ static bool rejects_query(F query) return false; } +static int test_family_index_first_use() +{ + std::atomic ready{0}; + std::atomic start{false}; + std::thread workers[8]; + bool passed[8] = {}; + for(int i = 0; i < 8; ++i) + workers[i] = std::thread([&, i] { + ++ready; + while(!start.load()) + std::this_thread::yield(); + int count = 0; + const auto* arches = rocke_known_arches(&count); + bool ok = true; + for(int a = 0; a < count; ++a) + { + const auto* arch = rocke_arch_target_from_gfx(arches[(a + i) % count]); + for(int j = 0; j < arch->mma.num_ops; ++j) + { + const auto& op = arch->mma.ops[j]; + char copied_id[256]; + snprintf(copied_id, sizeof(copied_id), "%s", op.op_id); + const char* family = rocke_arch_mma_op_id_family(copied_id); + ok &= family && strcmp(family, op.family) == 0; + ok &= family == rocke_arch_mma_op_id_family(op.op_id); + } + } + passed[i] = ok && !rocke_arch_mma_op_id_family(NULL) && !rocke_arch_mma_op_id_family("") + && !rocke_arch_mma_op_id_family("unknown"); + }); + while(ready.load() != 8) + std::this_thread::yield(); + start = true; + for(auto& worker : workers) + worker.join(); + for(bool ok : passed) + CHECK(ok); + return 0; +} + +static int test_family_index_duplicates() +{ + rocke_mma_op_t ops[6] = {}; + const char* ids[] = {"same", "conflict", "z", "same", "conflict", "a"}; + const char* families[] = {"wmma", "mma", "wmma_scaled", "wmma", "wmma", "mma"}; + for(int i = 0; i < 6; ++i) + { + ops[i].op_id = ids[i]; + ops[i].family = families[i]; + } + rocke_arch_target_t targets[2] = {}; + targets[0].mma = {ops, 3}; + targets[1].mma = {ops + 3, 3}; + const rocke_ati_arch_row_t registry[] + = {{"first", &targets[0]}, {"missing", NULL}, {"second", &targets[1]}}; + const rocke_ati_mma_family_index index(registry, 3); + CHECK(strcmp(index.lookup("same"), "wmma") == 0); + CHECK(strcmp(index.lookup("a"), "mma") == 0); + CHECK(strcmp(index.lookup("z"), "wmma_scaled") == 0); + CHECK(!index.lookup(NULL) && !index.lookup("") && !index.lookup("between")); + bool conflict = false; + try + { + index.lookup("conflict"); + } + catch(const ckc::Error& e) + { + conflict + = e.code() == ROCKE_ERR_VALUE + && strcmp(e.what(), "arch SSOT drift: op_id has inconsistent family across arches") + == 0; + } + CHECK(conflict); + CHECK(strcmp(index.lookup("a"), "mma") == 0); + const rocke_ati_mma_family_index empty(NULL, 0); + CHECK(!empty.lookup("same")); + return 0; +} + +static int test_mma_result_names() +{ + const char* ids[] = {"mfma_f32_16x16x16_f16", + "wmma_f32_16x16x16_f16", + "wmma_gfx1250_f32_16x16x64_fp8_fp8", + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32", + "wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k16", + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k32", + "wmma_gfx1250_f32_16x16x128_bf8_bf8_scale_e8m0_e8m0_k16"}; + for(int i = 0; i < 7; ++i) + { + rocke_ir_builder_t b; + CHECK(rocke_ir_builder_init(&b, "mma_names") == ROCKE_OK); + auto* value = rocke_b_const_i32(&b, 0); + auto* result = rocke_b_mma(&b, ids[i], value, value, value, NULL, 0); + CHECK(result && result->name); + CHECK(strncmp(result->name, i < 3 ? "%acc" : "%mxacc", i < 3 ? 4 : 6) == 0); + rocke_ir_builder_free(&b); + } + return 0; +} + static int test_scale_contracts() { const auto* arch = rocke_arch_target_from_gfx("gfx1250"); @@ -253,7 +357,9 @@ static int test_scale_layouts_and_families() int main() { - if(test_scale_contracts() || test_scale_layouts_and_families()) + // Keep this first: no previous query may prime the shared family index. + if(test_family_index_first_use() || test_family_index_duplicates() || test_mma_result_names() + || test_scale_contracts() || test_scale_layouts_and_families()) return 1; int checked = 0; for(const char* gfx : {"gfx950", "gfx1250"}) From cec35710df745b56a21a17bf9410259086d74612 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:57:01 -0400 Subject: [PATCH 11/15] refactor(rocke): organize MMA metadata docs and imports --- .../dsl_docs/architecture/authoring_model.md | 67 +--------------- .../platform/dsl_docs/reference/api_index.md | 3 + .../dsl_docs/reference/mma_metadata.md | 78 +++++++++++++++++++ .../rocke/platform/python/rocke/core/ir.py | 21 ++--- 4 files changed, 92 insertions(+), 77 deletions(-) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/mma_metadata.md diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md index 3cd67081d8c7..ac9c98f780e7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md @@ -29,70 +29,9 @@ out separately where they differ. Matrix-operation choices start from the exact gfx target. Resolve `ArchTarget.from_gfx(...)`, select an `MmaOp` from that target's `MmaCatalog`, and validate the operation's wave and layout contract. The exact catalog is -listed in [`kernel_taxonomy.md`](./kernel_taxonomy.md). - -MMA metadata uses `a_dtype`, `b_dtype`, and `c_dtype`, with C describing both -the accumulator input and result. Optional `a_scale_dtype`, `b_scale_dtype`, -and `scale_block_k` fields describe independent scale types and a shared K-group -size. `MmaScaleBlockK.K16` and `MmaScaleBlockK.K32` are the only scaled sizes; -all three fields are `None` for unscaled atoms. Integer 16/32 values from JSON -are normalized to the enum. Scale types are independent of matrix dtypes and -packed register types. Matrix fragment and layout accessors retain their A/B/C roles. - -`a_scale_frag_len` and `b_scale_frag_len` count logical scale elements per lane. -`a_scale_layout()` maps `(lane, slot)` to `(row, K-group)`; -`b_scale_layout()` maps it to `(K-group, column)`. These instruction-local maps -are separate from register byte packing and global tensor strides. Unscaled -atoms default to zero scale fragments and absent maps. An unavailable map raises -`NotImplementedError`, including for a scaled atom whose layout is not verified. -The native record appends corresponding counts and map pointers, so native -consumers must rebuild; existing role enum values remain unchanged. - -The four gfx1250 FP8/BF8 scaled atoms provide these scale maps: lane `l` and -slot `j` map to `(l % 16, j)` for A and `(j, l % 16)` for B. Both half-waves -duplicate the scales. K32 uses four E8M0 elements per lane packed into i32; -K16 uses eight packed into i64, with slot `j` at bit offset `8*j`. Matrix A/B -maps for these atoms remain unavailable. The gfx1250 loader applies tile and -instruction-step offsets to the scale coordinates before applying tensor strides. - -For scaled operations, query the complete contract and pass the selected atom -to `IRBuilder.mma`: - -```python -from rocke.core.arch import ArchTarget, MmaScaleBlockK - -atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( - family="wmma_scaled", - a_dtype="fp8", b_dtype="fp8", c_dtype="fp32", - scales=("e8m0", "e8m0", MmaScaleBlockK.K32), - m=16, n=16, k=128, -) -assert atom is not None -# result = builder.mma(atom, a, b, c, scale_a, scale_b) -``` - -Omitting `scales` leaves scales unconstrained. Passing `(a_type, b_type, block_k)` -matches the scale contract exactly; `(None, None, None)` selects unscaled atoms. -Partially specified scale contracts are invalid. Enumeration -and existence queries may match several records, while `op_for_shape` and -`select_largest_k` reject ambiguous exact matches or largest-K ties. The C queries -use the same rules: a NULL `rocke_mma_scale_filter_t` pointer is unconstrained, -and `{NULL, NULL, ROCKE_MMA_SCALE_NONE}` requests an unscaled atom. The C block -enum has `ROCKE_MMA_SCALE_K16` and `ROCKE_MMA_SCALE_K32`, plus the unscaled -sentinel. Scale value formats are `e8m0`, `e4m3`, and `e5m3`, with -`fp8e4m3` accepted as an alias for `e4m3`; actual target support comes from the -catalog. - -Scaled-WMMA IDs have the form -`wmma______scale___k`. -For example, `wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32` -has atom K=128 and one scale per 32 K elements for both inputs. Both scale -types are written even when equal. Lowering reads the catalog fields, never -parses the ID, and selects LLVM intrinsic names and packed carriers separately. -Existing `wmma_scale*_f32_*` and dotted `wmma.scaled.*` IDs are retired, as are -the dedicated scaled builder wrappers; serialized IR using those IDs must be regenerated. -Use `tile.mma` with a resolved catalog atom. Other MMA operation IDs retain -their existing spelling. +listed in [`kernel_taxonomy.md`](./kernel_taxonomy.md). For operand metadata, +scale layouts, query semantics, and migration details, see +[MMA metadata and queries](../reference/mma_metadata.md). Kernel definitions in [`/python/rocke/instances/`](../../python/rocke/instances/) and diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/api_index.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/api_index.md index 284a7c75448b..2856aad6821c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/api_index.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/api_index.md @@ -2,6 +2,9 @@ Top-level re-exports from `rocke` and `rocke.helpers`. Use this as a quick lookup when reading other docs. +For `MmaOp` fields, scale layouts, and `MmaCatalog` selection rules, see +[MMA metadata and queries](mma_metadata.md). + ## `from rocke import ...` ```text diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/mma_metadata.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/mma_metadata.md new file mode 100644 index 000000000000..31b5c59268a6 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/mma_metadata.md @@ -0,0 +1,78 @@ +# MMA Metadata and Queries + +This reference describes the `MmaOp` metadata and `MmaCatalog` queries used to +select matrix operations. For the overall kernel-building workflow, see the +[authoring model](../architecture/authoring_model.md). + +## Operand Metadata + +MMA metadata uses `a_dtype`, `b_dtype`, and `c_dtype`, with C describing both +the accumulator input and result. Optional `a_scale_dtype`, `b_scale_dtype`, +and `scale_block_k` fields describe independent scale types and a shared K-group +size. `MmaScaleBlockK.K16` and `MmaScaleBlockK.K32` are the only scaled sizes; +all three fields are `None` for unscaled atoms. Integer 16/32 values from JSON +are normalized to the enum. Scale types are independent of matrix dtypes and +packed register types. Matrix fragment and layout accessors retain their A/B/C roles. + +## Scale Layouts + +`a_scale_frag_len` and `b_scale_frag_len` count logical scale elements per lane. +`a_scale_layout()` maps `(lane, slot)` to `(row, K-group)`; +`b_scale_layout()` maps it to `(K-group, column)`. These instruction-local maps +are separate from register byte packing and global tensor strides. Unscaled +atoms default to zero scale fragments and absent maps. An unavailable map raises +`NotImplementedError`, including for a scaled atom whose layout is not verified. +The native record appends corresponding counts and map pointers, so native +consumers must rebuild; existing role enum values remain unchanged. + +### gfx1250 Scale Mapping and Packing + +The four gfx1250 FP8/BF8 scaled atoms provide these scale maps: lane `l` and +slot `j` map to `(l % 16, j)` for A and `(j, l % 16)` for B. Both half-waves +duplicate the scales. K32 uses four E8M0 elements per lane packed into i32; +K16 uses eight packed into i64, with slot `j` at bit offset `8*j`. Matrix A/B +maps for these atoms remain unavailable. The gfx1250 loader applies tile and +instruction-step offsets to the scale coordinates before applying tensor strides. + +## Catalog Queries + +For scaled operations, query the complete contract and pass the selected atom +to `IRBuilder.mma`: + +```python +from rocke.core.arch import ArchTarget, MmaScaleBlockK + +atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( + family="wmma_scaled", + a_dtype="fp8", b_dtype="fp8", c_dtype="fp32", + scales=("e8m0", "e8m0", MmaScaleBlockK.K32), + m=16, n=16, k=128, +) +assert atom is not None +# result = builder.mma(atom, a, b, c, scale_a, scale_b) +``` + +Omitting `scales` leaves scales unconstrained. Passing `(a_type, b_type, block_k)` +matches the scale contract exactly; `(None, None, None)` selects unscaled atoms. +Partially specified scale contracts are invalid. Enumeration +and existence queries may match several records, while `op_for_shape` and +`select_largest_k` reject ambiguous exact matches or largest-K ties. The C queries +use the same rules: a NULL `rocke_mma_scale_filter_t` pointer is unconstrained, +and `{NULL, NULL, ROCKE_MMA_SCALE_NONE}` requests an unscaled atom. The C block +enum has `ROCKE_MMA_SCALE_K16` and `ROCKE_MMA_SCALE_K32`, plus the unscaled +sentinel. Scale value formats are `e8m0`, `e4m3`, and `e5m3`, with +`fp8e4m3` accepted as an alias for `e4m3`; actual target support comes from the +catalog. + +## Operation IDs and Migration + +Scaled-WMMA IDs have the form +`wmma______scale___k`. +For example, `wmma_gfx1250_f32_16x16x128_fp8_fp8_scale_e8m0_e8m0_k32` +has atom K=128 and one scale per 32 K elements for both inputs. Both scale +types are written even when equal. Lowering reads the catalog fields, never +parses the ID, and selects LLVM intrinsic names and packed carriers separately. +Existing `wmma_scale*_f32_*` and dotted `wmma.scaled.*` IDs are retired, as are +the dedicated scaled builder wrappers; serialized IR using those IDs must be regenerated. +Use `tile.mma` with a resolved catalog atom. Other MMA operation IDs retain +their existing spelling. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py index 8789d022f978..cab47560124e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py @@ -29,6 +29,8 @@ from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from .arch import target as _arch + # ----------------------------- Types -------------------------------------- @@ -71,9 +73,8 @@ def __repr__(self) -> str: # Both are read from the arch SSOT (``core/arch/target``): fragment lengths from # ``_MMA_FRAGMENT_INFO`` and the accumulator dtype from the JSON catalog. ir.py # keeps *no* private copy of that data — an ``MmaOp`` object supplies both fields -# directly, and a bare op_id string is resolved through the lazy helpers below. -# The arch package is imported lazily (inside the helpers) so ir.py stays -# importable without eagerly loading the arch tree. +# directly, and a bare op_id string is resolved through the helpers below. +# The arch module loads its JSON catalog lazily when a lookup needs it. # op_id -> the ``result_name_hint`` the legacy ISA-named method used. This is # purely ir-side SSA naming (not arch data), kept here so the emitted value @@ -133,12 +134,10 @@ def _check_cachepolicy(op: str, value: int) -> int: def _mma_c_frag_len(op_id: str) -> int: """Accumulator fragment length for ``op_id`` from the arch SSOT. - Resolved through ``core/arch/target._MMA_FRAGMENT_INFO`` (imported lazily); + Resolved through ``core/arch/target._MMA_FRAGMENT_INFO``; ir.py holds no private copy. Unknown op_ids (frag length 0) raise, matching the strictness callers relied on. """ - from rocke.core.arch import target as _arch - frag_len = _arch._frag_info(op_id).c_frag_len if frag_len <= 0: raise ValueError( @@ -152,11 +151,9 @@ def _mma_c_is_int(op_id: str) -> bool: """True when ``op_id`` accumulates in i32 (integer WMMA). Sourced from the arch catalog's accumulator dtype - (``core/arch/data/arch_specs.json`` via ``target._op_id_c_dtype``), imported - lazily. Op_ids absent from the catalog default to the f32 accumulator. + (``core/arch/data/arch_specs.json`` via ``target._op_id_c_dtype``). + Op_ids absent from the catalog default to the f32 accumulator. """ - from rocke.core.arch import target as _arch - return _arch._op_id_c_dtype().get(op_id) == "i32" @@ -1857,11 +1854,9 @@ def mma( c_dtype = getattr(op, "c_dtype", None) is_int_acc = c_dtype == "i32" if c_dtype is not None else _mma_c_is_int(op_id) c_elem = I32 if is_int_acc else F32 - from .arch.target import _op_id_family - hint = ( "mxacc" - if _op_id_family().get(op_id) == "wmma_scaled" + if _arch._op_id_family().get(op_id) == "wmma_scaled" else _MMA_RESULT_HINT.get(op_id, "acc") ) return self._op( From 1f0815f40d95c0ccd84dbf49a4978994590a4b0a Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:12:51 -0400 Subject: [PATCH 12/15] refactor(rocke): use format-neutral scale packing names --- .../rocke/platform/cpp/core/lower_llvm/mma.cpp | 2 +- .../cpp/include/rocke/wmma_scale_internal.h | 13 +++++++------ .../platform/python/rocke/core/arch/wmma_scale.py | 10 ++++------ 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp index 48cb714160be..b998e9263138 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mma.cpp @@ -217,7 +217,7 @@ static void _emit_wmma_scale(rocke_lower_t* L, const rocke_op_t* op) } char packed_type[8]; - snprintf(packed_type, sizeof(packed_type), "i%d", rocke_e8m0_scale_word_bits(&spec->scales)); + snprintf(packed_type, sizeof(packed_type), "i%d", rocke_scale_word_bits(&spec->scales)); scale_ty = packed_type; if(strcmp(op->operands[3]->type->name, scale_ty) != 0 || strcmp(op->operands[4]->type->name, scale_ty) != 0) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h index cfaaa125c78e..c85df088e922 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/wmma_scale_internal.h @@ -1,7 +1,7 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT /* Shared gfx1250 operand contracts. Mirrors core/arch/wmma_scale.py: - * E8M0ScalePacking -> rocke_e8m0_scale_packing_t + * ScalePacking -> rocke_scale_packing_t * ScaledWmmaOp -> rocke_scaled_wmma_op_t * gfx1250_scaled_wmma -> rocke_gfx1250_scaled_wmma */ @@ -15,12 +15,13 @@ #include "rocke/error.hpp" #include "rocke/ir.h" -typedef struct rocke_e8m0_scale_packing +typedef struct rocke_scale_packing { - /* Consecutive K groups, first group in the low byte, independently for A/B. */ + /* Eight-bit encoded scales for consecutive K groups, first group in the + * low byte, independently for A/B. */ int count; int block_k; -} rocke_e8m0_scale_packing_t; +} rocke_scale_packing_t; typedef struct rocke_scaled_wmma_op { @@ -30,7 +31,7 @@ typedef struct rocke_scaled_wmma_op int matrix_words[2]; char declaration_key[160]; char intrinsic[160]; - rocke_e8m0_scale_packing_t scales; + rocke_scale_packing_t scales; } rocke_scaled_wmma_op_t; static inline const rocke_mma_op_t* rocke_gfx1250_scaled_wmma(const char* op_id) @@ -101,7 +102,7 @@ static inline const rocke_mma_op_t* rocke_gfx1250_scaled_wmma_from_op(const rock return rocke_gfx1250_scaled_wmma(op_id ? op_id : op->name); } -static inline int rocke_e8m0_scale_word_bits(const rocke_e8m0_scale_packing_t* packing) +static inline int rocke_scale_word_bits(const rocke_scale_packing_t* packing) { return packing->count * 8; } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py index db247a74c0a8..18d9e07cc13a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py @@ -10,10 +10,10 @@ @dataclass(frozen=True) -class E8M0ScalePacking: +class ScalePacking: """Pack consecutive K groups into a word, first group in its low byte. - Each scale is one E8M0 byte. Both A and B use this ordering independently; + Each encoded scale occupies one byte. A and B use this ordering independently; the integer word is only a carrier for the encoded floating-point scales. """ @@ -40,7 +40,7 @@ class ScaledWmmaOp: atom: MmaOp matrix_formats: tuple[int, int] scale_formats: tuple[int, int] - scales: E8M0ScalePacking + scales: ScalePacking @property def op_id(self) -> str: @@ -99,7 +99,5 @@ def gfx1250_scaled_wmma(op_id: str) -> ScaledWmmaOp | None: atom=atom, matrix_formats=(formats[atom.a_dtype], formats[atom.b_dtype]), scale_formats=(0, 0), # E8M0 for each source. - scales=E8M0ScalePacking( - count=atom.a_scale_frag_len, block_k=atom.scale_block_k - ), + scales=ScalePacking(count=atom.a_scale_frag_len, block_k=atom.scale_block_k), ) From 76b60c7c7bbd80d059e353487a4fdcaad260b901 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:38:23 -0400 Subject: [PATCH 13/15] feat(rocke): add string-backed MMA scale dtype enum --- .../dsl_docs/reference/mma_metadata.md | 14 +++- .../python/rocke/core/arch/__init__.py | 2 + .../platform/python/rocke/core/arch/target.py | 38 +++++++-- .../python/rocke/core/arch/wmma_scale.py | 6 +- .../platform/tests/core/test_arch_mma_ssot.py | 82 +++++++++++++++++-- .../tests/core/test_mma_catalog_lookup.cpp | 24 ++++++ 6 files changed, 146 insertions(+), 20 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/mma_metadata.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/mma_metadata.md index 31b5c59268a6..0d22f35d42f6 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/mma_metadata.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/mma_metadata.md @@ -14,6 +14,16 @@ all three fields are `None` for unscaled atoms. Integer 16/32 values from JSON are normalized to the enum. Scale types are independent of matrix dtypes and packed register types. Matrix fragment and layout accessors retain their A/B/C roles. +Scale dtypes normalize to the string-backed `MmaScaleDType` enum: `E8M0`, +`E4M3`, or `E5M3`. Existing string inputs remain accepted, and string formatting +and JSON retain the canonical `e8m0`, `e4m3`, and `e5m3` spellings. +`fp8e4m3` is an alias for `E4M3`. E5M3 is an unsigned scale format, distinct from +the signed E5M2 matrix format: `e5m2`, `fp8e5m2`, `bf8e5m2`, and `bf8` are +rejected as scale dtypes. LLVM defines separate matrix and scale format names +in [WMMAMods](https://github.com/ROCm/llvm-project/blob/6bd443a039454b7d2d9f34740ce0a6e3c10ac8d4/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h#L146). +Enum membership describes a format; supported operations still come from the +target catalog and backend. + ## Scale Layouts `a_scale_frag_len` and `b_scale_frag_len` count logical scale elements per lane. @@ -40,12 +50,12 @@ For scaled operations, query the complete contract and pass the selected atom to `IRBuilder.mma`: ```python -from rocke.core.arch import ArchTarget, MmaScaleBlockK +from rocke.core.arch import ArchTarget, MmaScaleBlockK, MmaScaleDType atom = ArchTarget.from_gfx("gfx1250").mma.op_for_shape( family="wmma_scaled", a_dtype="fp8", b_dtype="fp8", c_dtype="fp32", - scales=("e8m0", "e8m0", MmaScaleBlockK.K32), + scales=(MmaScaleDType.E8M0, MmaScaleDType.E8M0, MmaScaleBlockK.K32), m=16, n=16, k=128, ) assert atom is not None diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py index 6b1576f5163c..f97e32b8c7b5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/__init__.py @@ -15,6 +15,7 @@ MmaCatalog, MmaOp, MmaScaleBlockK, + MmaScaleDType, ResourceLimits, arch_from_isa, base_arch_from_target_id, @@ -31,6 +32,7 @@ "MmaCatalog", "MmaOp", "MmaScaleBlockK", + "MmaScaleDType", "ResourceLimits", "arch_from_isa", "base_arch_from_target_id", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py index 20543dc19159..fe94ac88f845 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/target.py @@ -20,7 +20,7 @@ import json import re from dataclasses import dataclass, field -from enum import IntEnum +from enum import Enum, IntEnum from functools import lru_cache from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -101,6 +101,27 @@ def coord(self, builder: Any, lane: Any, slot: int) -> Tuple[Any, Any]: return self.fn(builder, lane, slot) +class MmaScaleDType(str, Enum): + """Scale value formats, independent of matrix dtypes and target support. + + E5M3 is an unsigned scale format, distinct from the signed E5M2 matrix + format. Only E4M3 shares the accepted matrix spelling ``fp8e4m3``. + """ + + E8M0 = "e8m0" + E4M3 = "e4m3" + E5M3 = "e5m3" + + def __str__(self) -> str: + return self.value + + @classmethod + def _missing_(cls, value: object) -> MmaScaleDType | None: + if value == "fp8e4m3": + return cls.E4M3 + return None + + class MmaScaleBlockK(IntEnum): """Number of K elements sharing one scale, common to both matrix inputs.""" @@ -110,14 +131,14 @@ class MmaScaleBlockK(IntEnum): def _normalize_mma_scales( a_dtype: str | None, b_dtype: str | None, block_k: int | None -) -> tuple[str | None, str | None, MmaScaleBlockK | None]: +) -> tuple[MmaScaleDType | None, MmaScaleDType | None, MmaScaleBlockK | None]: """Validate the complete scale contract, independently of backend support.""" if a_dtype is None and b_dtype is None and block_k is None: return None, None, None - a = "e4m3" if a_dtype == "fp8e4m3" else a_dtype - b = "e4m3" if b_dtype == "fp8e4m3" else b_dtype - if a not in ("e8m0", "e4m3", "e5m3") or b not in ("e8m0", "e4m3", "e5m3"): - raise ValueError("MMA scale dtype must be e8m0, e4m3, or e5m3") + try: + a, b = MmaScaleDType(a_dtype), MmaScaleDType(b_dtype) + except ValueError: + raise ValueError("MMA scale dtype must be e8m0, e4m3, or e5m3") from None if type(block_k) not in (int, MmaScaleBlockK) or block_k not in (16, 32): raise ValueError("MMA scale_block_k must be an integer equal to 16 or 32") return a, b, MmaScaleBlockK(block_k) @@ -162,8 +183,8 @@ class MmaOp: # All absent means unscaled. The K-group size applies to both inputs; # scale types describe values, not the backend's packed register carrier. - a_scale_dtype: str | None = None - b_scale_dtype: str | None = None + a_scale_dtype: MmaScaleDType | str | None = None + b_scale_dtype: MmaScaleDType | str | None = None scale_block_k: MmaScaleBlockK | None = None a_scale_frag_len: int = 0 @@ -897,6 +918,7 @@ class MmaCatalog: """The arch-selected MMA atoms, with optional exact A/B scale filtering. ``scales=(a_dtype, b_dtype, block_k)`` selects the complete scale contract. + Scale dtypes accept ``MmaScaleDType`` members or their string spellings. ``None`` leaves scales unconstrained; ``(None, None, None)`` selects unscaled atoms. Exact selection and largest-K ties must be unambiguous. """ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py index 18d9e07cc13a..2febb7514b05 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/wmma_scale.py @@ -6,7 +6,7 @@ from dataclasses import dataclass -from .target import ArchTarget, MmaOp +from .target import ArchTarget, MmaOp, MmaScaleDType @dataclass(frozen=True) @@ -85,8 +85,8 @@ def gfx1250_scaled_wmma(op_id: str) -> ScaledWmmaOp | None: if ( atom.a_dtype not in formats or atom.b_dtype not in formats - or atom.a_scale_dtype != "e8m0" - or atom.b_scale_dtype != "e8m0" + or atom.a_scale_dtype != MmaScaleDType.E8M0 + or atom.b_scale_dtype != MmaScaleDType.E8M0 or atom.scale_block_k not in (16, 32) or atom.c_dtype != "fp32" or atom.shape != (16, 16, 128) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py index d0894160bf16..34b7a8576513 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_arch_mma_ssot.py @@ -11,13 +11,14 @@ from __future__ import annotations +import json import unittest -from dataclasses import replace +from dataclasses import asdict, replace from unittest import mock import pytest -from rocke.core.arch import ArchTarget, MmaCatalog, MmaOp, MmaScaleBlockK +from rocke.core.arch import ArchTarget, MmaCatalog, MmaOp, MmaScaleBlockK, MmaScaleDType from rocke.core.arch.wmma_scale import gfx1250_scaled_wmma from rocke.core.arch.target import ( @@ -92,7 +93,10 @@ def _contracts(): return [replace(row, op_id=f"fixture_{i}") for i, row in enumerate(rows)] -def test_full_contract_queries_distinguish_each_scale_type_and_shared_block(): +@pytest.mark.parametrize("use_strings", [False, True]) +def test_full_contract_queries_distinguish_each_scale_type_and_shared_block( + use_strings, +): rows = _contracts() assert len({row.op_id for row in rows}) == len(rows) catalog = MmaCatalog(rows) @@ -102,7 +106,19 @@ def test_full_contract_queries_distinguish_each_scale_type_and_shared_block(): a_dtype=row.a_dtype, b_dtype=row.b_dtype, c_dtype=row.c_dtype, - scales=(row.a_scale_dtype, row.b_scale_dtype, row.scale_block_k), + scales=( + ( + str(row.a_scale_dtype) + if use_strings and row.a_scale_dtype + else row.a_scale_dtype + ), + ( + str(row.b_scale_dtype) + if use_strings and row.b_scale_dtype + else row.b_scale_dtype + ), + row.scale_block_k, + ), m=row.m, n=row.n, ) @@ -151,8 +167,57 @@ def test_scale_alias_selects_the_same_contract(): (("e8m0", "fp8e4m3", 32), "b_scale_dtype"), ): row = catalog.op_for_shape(**query, scales=scales) - assert row is not None and getattr(row, field) == "e4m3" - assert getattr(replace(row, **{field: "fp8e4m3"}), field) == "e4m3" + assert row is not None and getattr(row, field) is MmaScaleDType.E4M3 + assert getattr(replace(row, **{field: "fp8e4m3"}), field) is MmaScaleDType.E4M3 + + +@pytest.mark.parametrize("dtype", list(MmaScaleDType)) +def test_scale_dtype_preserves_string_and_json_behavior(dtype): + assert isinstance(dtype, str) + assert dtype == dtype.value + assert str(dtype) == f"{dtype}" == dtype.value + assert {dtype: 1}[dtype.value] == 1 + assert json.dumps(dtype) == json.dumps(dtype.value) + assert MmaScaleDType(json.loads(json.dumps(dtype))) is dtype + row = replace(_contracts()[0], a_scale_dtype=dtype.value, b_scale_dtype=dtype) + assert row.a_scale_dtype is row.b_scale_dtype is dtype + decoded = json.loads(json.dumps(asdict(row))) + assert decoded["a_scale_dtype"] == decoded["b_scale_dtype"] == dtype.value + restored = MmaOp(**decoded) + assert restored == row + assert restored.a_scale_dtype is restored.b_scale_dtype is dtype + + +def test_scale_dtype_members_and_alias(): + assert [dtype.value for dtype in MmaScaleDType] == ["e8m0", "e4m3", "e5m3"] + assert MmaScaleDType("fp8e4m3") is MmaScaleDType.E4M3 + + +@pytest.mark.parametrize("dtype", ["e5m2", "fp8e5m2", "bf8e5m2", "bf8"]) +@pytest.mark.parametrize("field", ["a_scale_dtype", "b_scale_dtype"]) +def test_e5m2_is_not_a_scale_dtype_alias(dtype, field): + with pytest.raises(ValueError): + MmaScaleDType(dtype) + scales = ["e8m0", "e8m0", 32] + scales[0 if field == "a_scale_dtype" else 1] = dtype + catalog = MmaCatalog(_contracts()) + query = dict( + family="wmma_scaled", + a_dtype="fp8", + b_dtype="bf8", + c_dtype="fp32", + scales=tuple(scales), + m=16, + n=16, + ) + for method, kwargs in ( + (catalog.enumerate, query), + (catalog.has_shape, {**query, "k": 128}), + (catalog.op_for_shape, {**query, "k": 128}), + (catalog.select_largest_k, query), + ): + with pytest.raises(ValueError, match="MMA scale dtype"): + method(**kwargs) def test_scaled_catalog_identity_and_backend_contract(): @@ -167,6 +232,7 @@ def test_scaled_catalog_identity_and_backend_contract(): f"_scale_e8m0_e8m0_k{row.scale_block_k}" ) assert row.a_scale_dtype == row.b_scale_dtype == "e8m0" + assert row.a_scale_dtype is row.b_scale_dtype is MmaScaleDType.E8M0 assert isinstance(row.scale_block_k, MmaScaleBlockK) packing = gfx1250_scaled_wmma(row.op_id) assert packing.atom is row @@ -191,7 +257,9 @@ def test_scale_block_k_is_a_two_value_enum(): @pytest.mark.parametrize("field", ["a_scale_dtype", "b_scale_dtype"]) -@pytest.mark.parametrize("dtype", ["i32", "fp4", "e5m2", "", None]) +@pytest.mark.parametrize( + "dtype", ["i32", "fp4", "e5m2", "fp8e5m2", "bf8e5m2", "bf8", "", None] +) def test_invalid_scale_format(field, dtype): with pytest.raises(ValueError, match="e8m0, e4m3, or e5m3"): replace(_contracts()[0], **{field: dtype}) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp index 279c843a1ab2..472888dc04cd 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp @@ -254,6 +254,30 @@ static int test_scale_contracts() &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, NULL, 0, &scales); })); } + for(const char* dtype : {"e5m2", "fp8e5m2", "bf8e5m2", "bf8"}) + { + for(int input = 0; input < 2; ++input) + { + rocke_mma_scale_filter_t scales = {"e8m0", "e8m0", ROCKE_MMA_SCALE_K32}; + (input == 0 ? scales.a_scale_dtype : scales.b_scale_dtype) = dtype; + CHECK(rejects_query([&] { + rocke_mma_catalog_enumerate( + &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, NULL, 0, &scales); + })); + CHECK(rejects_query([&] { + rocke_mma_catalog_has_shape( + &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128, &scales); + })); + CHECK(rejects_query([&] { + rocke_mma_catalog_op_for_shape( + &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128, &scales); + })); + CHECK(rejects_query([&] { + rocke_mma_catalog_select_largest_k( + &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, -1, &scales); + })); + } + } const rocke_mma_scale_filter_t valid_missing = {"e5m3", "e5m3", ROCKE_MMA_SCALE_K32}; CHECK(!rocke_mma_catalog_op_for_shape( &cat, "wmma_scaled", "fp8", "bf8", "fp32", 16, 16, 128, &valid_missing)); From d6606f7d03883db907b0d5b0011bdb5b89aa2782 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:57:34 -0400 Subject: [PATCH 14/15] fix(rocke): forward scale filters through ArchTarget helper --- .../rocke/platform/cpp/core/arch/helpers.cpp | 6 ++++-- .../cpp/include/rocke/helper_rocke.core.arch.h | 3 ++- .../cpp/instances/common/_matmul_nbits_common.cpp | 3 ++- .../conv_implicit_gemm_conv_spec_descriptors.cpp | 3 ++- .../cpp/instances/common/conv_implicit_gemm_dgrad.cpp | 3 ++- .../rocke/platform/cpp/instances/common/fmha_arch.cpp | 3 ++- .../rocke/platform/cpp/instances/common/gemm_spec.cpp | 3 ++- .../platform/cpp/instances/common/gemm_universal.cpp | 3 ++- .../rocke/platform/cpp/instances/common/mfma_gemm.cpp | 2 +- ...v_pool_build_context_ctx_init_quant_primitives.cpp | 3 ++- .../platform/cpp/instances/gfx1151/wmma_gemm.cpp | 3 ++- .../platform/cpp/instances/gfx1151/wmma_gemm_int8.cpp | 11 +++++++++-- .../platform/cpp/instances/gfx1201/wmma_gemm.cpp | 3 ++- ...tion_tiled_3d_gfx942_attn_tiled_3d_public_glue.cpp | 7 ++++--- .../gfx950/attention_tiled_3d_public_glue.cpp | 7 ++++--- .../platform/tests/core/test_mma_catalog_lookup.cpp | 4 ++++ 16 files changed, 46 insertions(+), 21 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/helpers.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/helpers.cpp index 5f74ed7ddc7d..00abd5e5b4a3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/helpers.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/arch/helpers.cpp @@ -87,14 +87,16 @@ const rocke_mmaop_t* rocke_archtarget_op_for_shape(const rocke_archtarget_t* t, const char* c_dtype, int m, int n, - int k) + int k, + const rocke_mma_scale_filter_t* scales) { /* target.mma.op_for_shape(family=..., a/b/c=..., m, n, k). */ if(t == NULL) { return NULL; } - return rocke_mma_catalog_op_for_shape(&t->mma, family, a_dtype, b_dtype, c_dtype, m, n, k); + return rocke_mma_catalog_op_for_shape( + &t->mma, family, a_dtype, b_dtype, c_dtype, m, n, k, scales); } const rocke_mmaop_t* rocke_archtarget_by_op_id(const rocke_archtarget_t* t, const char* op_id) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h index 1fc0f9462403..5ca54769d25a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h @@ -104,7 +104,8 @@ const rocke_mmaop_t* rocke_archtarget_op_for_shape(const rocke_archtarget_t* t, const char* c_dtype, int m, int n, - int k); + int k, + const rocke_mma_scale_filter_t* scales); /* target.mma.by_op_id(op_id): the catalog atom whose op_id handle matches * `op_id` (the backend's MMA key, e.g. "mfma_f32_16x16x16_f16"), or NULL if the diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/_matmul_nbits_common.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/_matmul_nbits_common.cpp index df41776315e3..e8233bf31b79 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/_matmul_nbits_common.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/_matmul_nbits_common.cpp @@ -306,7 +306,8 @@ bool rocke_matmul_nbits_validate_common_spec(const rocke_matmul_nbits_spec_t* sp ROCKE__WMMA_C_DTYPE, t->warp_tile_m, t->warp_tile_n, - t->warp_tile_k) + t->warp_tile_k, + nullptr) == NULL) { ROCKE_NBITS_REJECT("unsupported fp16 warp_tile (%d, %d, %d) on %s", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp index cf3cded44af4..9b72cb723485 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp @@ -959,7 +959,8 @@ const rocke_mmaop_t* rocke_conv_resolve_op(rocke_ir_builder_t* b, "fp32", spec->warp_tile_m, spec->warp_tile_n, - spec->warp_tile_k); + spec->warp_tile_k, + nullptr); } if(op == NULL) { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp index 52f30b6e97e9..70436929f1b8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp @@ -857,7 +857,8 @@ static const rocke_mmaop_t* "fp32", spec->warp_tile_m, spec->warp_tile_n, - spec->warp_tile_k); + spec->warp_tile_k, + nullptr); if(!op) { rocke_i_set_err(b, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/fmha_arch.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/fmha_arch.cpp index fa15c2ec20be..b3356cbc1eef 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/fmha_arch.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/fmha_arch.cpp @@ -135,7 +135,8 @@ bool rocke_validate_fmha_mfma_atom(const char* dtype, const char* arch, char* ou * has_shape(m,n,k) is true iff a matching op exists; since enumerate() * filters on m,n and has_shape compares the full (m,n,k) shape, this is * exactly op_for_shape(...,k) != None. family defaults to "mma". */ - if(rocke_archtarget_op_for_shape(target, "mma", a_name, a_name, "fp32", blk, blk, blk) == NULL) + if(rocke_archtarget_op_for_shape(target, "mma", a_name, a_name, "fp32", blk, blk, blk, nullptr) + == NULL) { if(out != NULL && out_cap != 0) { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_spec.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_spec.cpp index 2ba203aabc7d..cb28bb0feaa7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_spec.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_spec.cpp @@ -415,7 +415,8 @@ static const rocke_type_t* "fp32", t->warp_tile_m, t->warp_tile_n, - t->warp_tile_k); + t->warp_tile_k, + nullptr); } /* _ab_lds_plan(spec, arch) -> (ab_single, db, two_buf). Pure ints/bools. */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_universal.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_universal.cpp index a07976eef143..8dc0c91ded87 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_universal.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_universal.cpp @@ -81,7 +81,8 @@ static const rocke_mmaop_t* rocke_gemm_resolve_mma_op(const rocke_gemm_universal "fp32", t->warp_tile_m, t->warp_tile_n, - t->warp_tile_k); + t->warp_tile_k, + nullptr); } /* _storage_dtype(spec): validate homogeneous A/B/C + fp32 acc + RCR layout, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/mfma_gemm.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/mfma_gemm.cpp index 2de604ddb873..5fae31871d91 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/mfma_gemm.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/mfma_gemm.cpp @@ -254,7 +254,7 @@ bool rocke_mfma_gemm_is_valid_spec(const rocke_mfma_gemm_spec_t* spec, /* if not target.mma.has_shape(a=cat, b=cat, c="fp32", m, n, k): ... * op_for_shape returns NULL when the shape/dtype combo is absent. */ if(rocke_archtarget_op_for_shape( - target, "mma", cat_dtype, cat_dtype, "fp32", atom->m, atom->n, atom->k) + target, "mma", cat_dtype, cat_dtype, "fp32", atom->m, atom->n, atom->k, nullptr) == NULL) { snprintf(buf, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_build_context_ctx_init_quant_primitives.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_build_context_ctx_init_quant_primitives.cpp index 08a245652c5e..851bb2f993e8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_build_context_ctx_init_quant_primitives.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/deep_fused_conv_pool_build_context_ctx_init_quant_primitives.cpp @@ -576,7 +576,8 @@ rocke_status_t "fp32", ROCKE_GFX1151_DFCP_WMMA, ROCKE_GFX1151_DFCP_WMMA, - ROCKE_GFX1151_DFCP_WMMA); + ROCKE_GFX1151_DFCP_WMMA, + nullptr); /* op0 = target.mma.by_op_id(_OP_ID_IU8) if spec.native_int else op * op1 = target.mma.by_op_id(_OP_ID_IU4) if spec.native_int else op */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/wmma_gemm.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/wmma_gemm.cpp index c7bf55eae136..4e456a9b5b9c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/wmma_gemm.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/wmma_gemm.cpp @@ -145,7 +145,8 @@ bool rocke_wmma_gemm_gfx1151_is_valid_spec(const rocke_wmma_gemm_gfx1151_spec_t* "fp32", ROCKE_WMMA_M, ROCKE_WMMA_N, - ROCKE_WMMA_K) + ROCKE_WMMA_K, + nullptr) == NULL) { snprintf(buf, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/wmma_gemm_int8.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/wmma_gemm_int8.cpp index cad79e6eb429..fb6b0c138189 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/wmma_gemm_int8.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1151/wmma_gemm_int8.cpp @@ -133,8 +133,15 @@ bool rocke_wmma_gemm_int8_is_valid_spec(const rocke_wmma_gemm_int8_spec_t* spec, * Operands are int8 in memory but dequantized to f16 before the MMA, so the * *compute* atom we gate on is the fp16 WMMA 16x16x16 one. * op_for_shape returns NULL when the shape/dtype combo is absent. */ - if(rocke_archtarget_op_for_shape( - target, "wmma", "fp16", "fp16", "fp32", ROCKE_WMMA_M, ROCKE_WMMA_N, ROCKE_WMMA_K) + if(rocke_archtarget_op_for_shape(target, + "wmma", + "fp16", + "fp16", + "fp32", + ROCKE_WMMA_M, + ROCKE_WMMA_N, + ROCKE_WMMA_K, + nullptr) == NULL) { snprintf(buf, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1201/wmma_gemm.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1201/wmma_gemm.cpp index 21748c6fc3bf..d7fa0fdb44bc 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1201/wmma_gemm.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx1201/wmma_gemm.cpp @@ -136,7 +136,8 @@ bool rocke_wmma_gemm_gfx1201_is_valid_spec(const rocke_wmma_gemm_gfx1201_spec_t* "fp32", ROCKE_WMMA_M, ROCKE_WMMA_N, - ROCKE_WMMA_K) + ROCKE_WMMA_K, + nullptr) == NULL) { snprintf(buf, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_public_glue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_public_glue.cpp index fa786496833c..4b9fb4e6cf24 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_public_glue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_public_glue.cpp @@ -101,9 +101,9 @@ static void rocke_attn3d_set_err_buf(char* err, size_t err_cap, const char* msg) static bool rocke_attn3d_narrow_k_available(const rocke_archtarget_t* t) { const rocke_mmaop_t* f16 - = rocke_archtarget_op_for_shape(t, "mma", "f16", "f16", "fp32", 16, 16, 16); + = rocke_archtarget_op_for_shape(t, "mma", "f16", "f16", "fp32", 16, 16, 16, nullptr); const rocke_mmaop_t* bf16 - = rocke_archtarget_op_for_shape(t, "mma", "bf16", "bf16", "fp32", 16, 16, 16); + = rocke_archtarget_op_for_shape(t, "mma", "bf16", "bf16", "fp32", 16, 16, 16, nullptr); return f16 != NULL && bf16 != NULL; } @@ -112,7 +112,8 @@ static bool rocke_attn3d_narrow_k_available(const rocke_archtarget_t* t) * exposed by the C arch surface -- see the ARCH GATE NOTE above). */ static bool rocke_attn3d_wide_k_available(const rocke_archtarget_t* t) { - return rocke_archtarget_op_for_shape(t, "mma", "f16", "f16", "fp32", 16, 16, 32) != NULL; + return rocke_archtarget_op_for_shape(t, "mma", "f16", "f16", "fp32", 16, 16, 32, nullptr) + != NULL; } /* ===================================================================== * diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_public_glue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_public_glue.cpp index 66da4b755d44..26dd78a5f54b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_public_glue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_public_glue.cpp @@ -120,9 +120,9 @@ static void rocke_attn3d950_set_err_buf(char* err, size_t err_cap, const char* m static bool rocke_attn3d950_narrow_k_available(const rocke_archtarget_t* t) { const rocke_mmaop_t* f16 - = rocke_archtarget_op_for_shape(t, "mma", "f16", "f16", "fp32", 16, 16, 16); + = rocke_archtarget_op_for_shape(t, "mma", "f16", "f16", "fp32", 16, 16, 16, nullptr); const rocke_mmaop_t* bf16 - = rocke_archtarget_op_for_shape(t, "mma", "bf16", "bf16", "fp32", 16, 16, 16); + = rocke_archtarget_op_for_shape(t, "mma", "bf16", "bf16", "fp32", 16, 16, 16, nullptr); return f16 != NULL && bf16 != NULL; } @@ -131,7 +131,8 @@ static bool rocke_attn3d950_narrow_k_available(const rocke_archtarget_t* t) * exposed by the C arch surface -- see the ARCH GATE NOTE above). */ static bool rocke_attn3d950_wide_k_available(const rocke_archtarget_t* t) { - return rocke_archtarget_op_for_shape(t, "mma", "f16", "f16", "fp32", 16, 16, 32) != NULL; + return rocke_archtarget_op_for_shape(t, "mma", "f16", "f16", "fp32", 16, 16, 32, nullptr) + != NULL; } /* validate_tiled_attention_arch(arch) gate (shared by supports + config). On diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp index 472888dc04cd..0ca6d1c65b9e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_mma_catalog_lookup.cpp @@ -5,6 +5,7 @@ #include "mma_family_index.h" #include "rocke/arch_target.h" #include "rocke/error.hpp" +#include "rocke/helper_rocke.core.arch.h" #include "rocke/wmma_scale_internal.h" #include @@ -421,6 +422,9 @@ int main() op->k, &scales) != op + || rocke_archtarget_op_for_shape( + arch, op->family, a, b, op->c_dtype, op->m, op->n, op->k, &scales) + != op || rocke_mma_catalog_select_largest_k(&arch->mma, op->family, a, From 81d5512822ce68e53869b244603cf3223a7bf642 Mon Sep 17 00:00:00 2001 From: Max Podkorytov <4273004+tenpercent@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:16:50 -0400 Subject: [PATCH 15/15] docs(rocke): document native ArchTarget atom lookup --- .../include/rocke/helper_rocke.core.arch.h | 3 ++ .../dsl_docs/architecture/authoring_model.md | 3 ++ .../dsl_docs/reference/mma_metadata.md | 42 +++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h index 5ca54769d25a..ec228fb445aa 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.core.arch.h @@ -95,6 +95,9 @@ const rocke_arch_mma_catalog_t* rocke_archtarget_mma(const rocke_archtarget_t* t /* target.mma.op_for_shape(...): the resolved MmaOp for an exact (m, n, k) atom * shape and (normalised) dtype combo, or NULL if absent. `family` NULL => "mma". + * The trailing scales argument is required: NULL leaves scales unconstrained; + * {NULL, NULL, ROCKE_MMA_SCALE_NONE} selects unscaled atoms. Invalid filters or + * ambiguous matches raise ckc::Error; callers handle errors at a C++ boundary. * This is the entry point build_universal_gemm uses to resolve the atom from the * target catalog. Forwards to rocke_mma_catalog_op_for_shape on t->mma. */ const rocke_mmaop_t* rocke_archtarget_op_for_shape(const rocke_archtarget_t* t, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md index ac9c98f780e7..0fd7996f116b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/authoring_model.md @@ -32,6 +32,9 @@ and validate the operation's wave and layout contract. The exact catalog is listed in [`kernel_taxonomy.md`](./kernel_taxonomy.md). For operand metadata, scale layouts, query semantics, and migration details, see [MMA metadata and queries](../reference/mma_metadata.md). +Native builders can use `rocke_archtarget_op_for_shape`; see +[native C/C++ atom lookup](../reference/mma_metadata.md#native-cc-atom-lookup) +for the helper's contract and a complete example. Kernel definitions in [`/python/rocke/instances/`](../../python/rocke/instances/) and diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/mma_metadata.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/mma_metadata.md index 0d22f35d42f6..41642b1170ce 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/mma_metadata.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/mma_metadata.md @@ -74,6 +74,48 @@ sentinel. Scale value formats are `e8m0`, `e4m3`, and `e5m3`, with `fp8e4m3` accepted as an alias for `e4m3`; actual target support comes from the catalog. +### Native C/C++ Atom Lookup + +When holding an `ArchTarget`, use `rocke_archtarget_op_for_shape` from +`rocke/helper_rocke.core.arch.h`. It forwards to +`rocke_mma_catalog_op_for_shape(&target->mma, ...)`, which is also available +directly from `rocke/arch_target.h`. Both select the same catalog record. +Kernel builders use that record's operation ID, fragment sizes, and layouts; +support checks use the lookup to establish whether the requested atom exists. +The `(m, n, k)` arguments describe one instruction atom, not the full GEMM. + +This C++ example selects the gfx1250 FP8 atom with E8M0 scales shared over K32: + +```cpp +#include "rocke/error.hpp" +#include "rocke/helper_rocke.core.arch.h" + +int main() +{ + try + { + const auto* target = rocke_archtarget_from_gfx("gfx1250"); + const rocke_mma_scale_filter_t scales = {"e8m0", "e8m0", ROCKE_MMA_SCALE_K32}; + const auto* atom = rocke_archtarget_op_for_shape( + target, "wmma_scaled", "fp8", "fp8", "fp32", 16, 16, 128, &scales); + return atom ? 0 : 1; + } + catch(const ckc::Error&) + { + return 2; + } +} +``` + +The helper requires the trailing filter argument in both C and C++. +Pass `NULL` (`nullptr` in C++) for unconstrained scales, or a pointer to +`{NULL, NULL, ROCKE_MMA_SCALE_NONE}` to select only unscaled atoms. +The scale-contract rules above apply. No match, or a null target, returns +`NULL`; invalid filters and ambiguous matches raise `ckc::Error`. Handle those +errors at a C++ boundary before returning to C. In particular, the scaled +FP8/BF8 `16x16x128` shapes have both K16 and K32 records, so an unconstrained +lookup is ambiguous. + ## Operation IDs and Migration Scaled-WMMA IDs have the form