From e42e86d281cc4a5fdd7567fb5a7b6ba5b92d8d52 Mon Sep 17 00:00:00 2001 From: Chris Millette Date: Thu, 15 Oct 2026 20:33:00 -0400 Subject: [PATCH 1/4] feat(rocke): add an unsigned compare, a no-unsigned-wrap add and a range marker to the IR builder Adds three integer IR pieces to both engines: - cmp_ult(a, b): arith.cmp with pred "ult", lowered to icmp ult. The constant folder compares the operands as unsigned at their width, and the HIP lowering casts both operands to uint32_t / uint64_t. - add_nuw(a, b): arith.add with wrap "nuw", lowered to add nuw. Plain add keeps nsw, so no existing IR changes. The HIP lowering adds as unsigned and casts back, since a signed C++ sum past the signed maximum is undefined behaviour that clang emits as nsw. - IRBuilder._assume_range: a private marker op, arith.assume_range, that returns its operand and records a checked range and congruence in its attrs for the tiling layer's range walk. It emits no instruction: both LLVM lowerings print the marked value at every use, the HIP lowerings emit a copy, and the folder folds it on a constant. Both constant folders wrap every folded integer result to its width as two's complement, read every integer constant at its width the same way (so the literal 0x80000000 and a folded i32 sum with those bits are one value), and fold zext as the operand read unsigned at its own width, so a folded add_nuw past the signed maximum reaches later signed folds and widenings as the value the register holds. Both builders refuse operands other than two i32 or two i64 values. Both verifiers check the new attrs with identical texts: the add wrap and the compare predicate (each a known string; nuw and ult, the forms of add_nuw and cmp_ult, only on i32 or i64 operands), and the marker's attrs (each an integer), multiple_of, residue, the choice of an int or a runtime upper end, and lo below hi. Both lowerings of every engine refuse a wrap that is not a known string rather than reading it as nsw, and a nuw add on other operand types. The new opcode is appended to rocke_opcode_t. The C API gains rocke_b_add_nuw and rocke_b_cmp_ult; the unsigned HIP type spelling moves to the shared HIP lowering core so the cast and arith handlers use one table. Tests: tests/core/test_unsigned_int_ops.py (IR text, engine agreement, C API against the Python builder, verifier and builder refusals, folding, HIP text and a hipcc compile without nsw, GPU values against numpy) and the rocke_unsigned_int_ops ctest. The C API comparison gets the built program through ROCKE_UNSIGNED_INT_OPS_CLI, which tests/run_all.py resolves from CTest and the installed rocke_pytest ctest sets. The op vocabulary and IR model docs list the new ops. Co-Authored-By: Claude --- ...ROCKE_ENGINE_test_categories_external.yaml | 2 + .../rocke/platform/CMakeLists.txt | 13 +- .../rocke/platform/cpp/core/ir/core_types.cpp | 92 ++- .../rocke/platform/cpp/core/ir/ir_arith.cpp | 51 ++ .../platform/cpp/core/lower_hip/core.cpp | 20 + .../cpp/core/lower_hip/lower_hip_arith.cpp | 65 +- .../cpp/core/lower_hip/lower_hip_cast.cpp | 20 +- .../platform/cpp/core/lower_llvm/arith.cpp | 24 + .../platform/cpp/core/lower_llvm/core.cpp | 7 + .../rocke/platform/cpp/core/passes.cpp | 90 ++- .../rocke/platform/cpp/core/verify.cpp | 120 ++++ .../rocke/platform/cpp/include/rocke/ir.h | 10 + .../platform/cpp/include/rocke/ir_internal.h | 16 + .../cpp/include/rocke/lower_hip_internal.h | 5 + .../platform/dsl_docs/ir_lowering/ir_model.md | 2 + .../dsl_docs/reference/op_vocabulary.md | 5 +- .../rocke/platform/python/rocke/core/ir.py | 180 ++++++ .../platform/python/rocke/core/lower_hip.py | 32 +- .../platform/python/rocke/core/lower_llvm.py | 20 +- .../platform/python/rocke/core/passes.py | 43 +- .../platform/python/rocke/core/verify.py | 35 ++ .../rocke/platform/tests/CMakeLists.txt | 17 + .../rocke/platform/tests/README.md | 21 +- .../tests/core/test_unsigned_int_ops.cpp | 498 +++++++++++++++ .../tests/core/test_unsigned_int_ops.py | 592 ++++++++++++++++++ .../rocke/platform/tests/run_all.py | 5 + .../platform/tests/tools/test_run_all.py | 14 +- 27 files changed, 1939 insertions(+), 60 deletions(-) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.cpp create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.py diff --git a/dnn-providers/hip-kernel-provider/ROCKE_ENGINE_test_categories_external.yaml b/dnn-providers/hip-kernel-provider/ROCKE_ENGINE_test_categories_external.yaml index 12658857fc67..d619f5c0f050 100644 --- a/dnn-providers/hip-kernel-provider/ROCKE_ENGINE_test_categories_external.yaml +++ b/dnn-providers/hip-kernel-provider/ROCKE_ENGINE_test_categories_external.yaml @@ -35,6 +35,7 @@ test_categories: - "rocke_future_intrinsic_lowering" - "rocke_ir_types" - "rocke_opcode_values" + - "rocke_unsigned_int_ops" - "rocke_tiled_attention_2d_reentrancy" - "rocke_unified_attention_scalar_lower_to_llvm" - "rocke_python_import_smoke" @@ -71,6 +72,7 @@ test_categories: - "rocke_future_intrinsic_lowering" - "rocke_ir_types" - "rocke_opcode_values" + - "rocke_unsigned_int_ops" - "rocke_tiled_attention_2d_reentrancy" - "rocke_unified_attention_scalar_lower_to_llvm" - "rocke_python_import_smoke" diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/CMakeLists.txt b/dnn-providers/hip-kernel-provider/rocke/platform/CMakeLists.txt index d24f60c5675f..e8eaccf77bb5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/CMakeLists.txt +++ b/dnn-providers/hip-kernel-provider/rocke/platform/CMakeLists.txt @@ -415,6 +415,17 @@ if(ROCKE_INSTALL_TESTS) "set_tests_properties(rocke_golden_static PROPERTIES LABELS \"ckc;python;golden;host;quick\")\n") if(ROCKE_INSTALL_PLATFORM_PYTEST) + # Native helpers some Python tests drive, as paths relative to the + # CTestTestfile: the portable-IR replay CLI, and the C API program whose + # --print-llvm output test_unsigned_int_ops.py compares with the Python + # builder's text. + set(_rocke_pytest_env + "PYTHONPATH=.;ROCKE_REPLAY_CLI=${ROCKE_TEST_CTEST_REL}tests/portable_ir/rocke_portable_ir_replay_cli${CMAKE_EXECUTABLE_SUFFIX}") + if("rocke_unsigned_int_ops" IN_LIST _ROCKE_INSTALL_TEST_TARGETS) + rocke_cxx_test_installed_name(_rocke_installed_name rocke_unsigned_int_ops) + string(APPEND _rocke_pytest_env + ";ROCKE_UNSIGNED_INT_OPS_CLI=${ROCKE_CXX_TEST_CTEST_REL}${_rocke_installed_name}${CMAKE_EXECUTABLE_SUFFIX}") + endif() # Full pytest suite. PYTHONPATH points at the CTestTestfile's working # directory (which contains the installed rocke package) so `import rocke` # resolves without the source-tree sys.path shims. The library tests have @@ -424,7 +435,7 @@ if(ROCKE_INSTALL_TESTS) string(APPEND _ROCKE_INSTALLED_CTEST_ENTRIES "set_tests_properties(rocke_pytest PROPERTIES\n" " LABELS \"ckc\\;python\\;quick\"\n" - " ENVIRONMENT \"PYTHONPATH=.;ROCKE_REPLAY_CLI=${ROCKE_TEST_CTEST_REL}tests/portable_ir/rocke_portable_ir_replay_cli${CMAKE_EXECUTABLE_SUFFIX}\"\n" + " ENVIRONMENT \"${_rocke_pytest_env}\"\n" " TIMEOUT 300)\n") endif() diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp index 05e44950d748..2ebf95c223c1 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp @@ -519,6 +519,88 @@ const char* rocke_attr_get_str(const rocke_attr_map_t* m, const char* key) return v->u.s; } +/* ir._INT_ADD_WRAPS / ir._INT_CMP_PREDICATES. */ +static const char* const k_int_add_wraps[] = {"nsw", "nuw", NULL}; +static const char* const k_int_cmp_predicates[] = {"lt", "le", "gt", "ge", "eq", "ne", "ult", NULL}; + +/* The shared body of rocke_i_add_wrap_error / rocke_i_cmp_pred_error: the + * error for the string attr `key` of `op_name`, or NULL when it is absent or + * one of `allowed`. The unsigned value `unsigned_value` takes i32 or i64 + * operands only. */ +static const char* string_attr_error(const rocke_attr_map_t* attrs, + const char* op_name, + const char* key, + const char* const* allowed, + const char* not_a_string, + const char* unsigned_value, + const rocke_type_t* operand_type, + char* buf, + size_t cap) +{ + const rocke_attr_value_t* v = rocke_attr_get(attrs, key); + if(!v) + return NULL; + if(v->kind != ROCKE_ATTR_STR) + return not_a_string; + bool known = false; + for(const char* const* s = allowed; *s; ++s) + { + if(strcmp(v->u.s, *s) == 0) + known = true; + } + if(!known) + { + snprintf(buf, cap, "%s: unknown %s '%s'", op_name, key, v->u.s); + return buf; + } + const char* type_name = operand_type ? operand_type->name : "?"; + if(strcmp(v->u.s, unsigned_value) == 0 && strcmp(type_name, "i32") != 0 + && strcmp(type_name, "i64") != 0) + { + snprintf(buf, + cap, + "%s: %s '%s' needs i32 or i64 operands, got %s", + op_name, + key, + unsigned_value, + type_name); + return buf; + } + return NULL; +} + +const char* rocke_i_add_wrap_error(const rocke_attr_map_t* attrs, + const rocke_type_t* operand_type, + char* buf, + size_t cap) +{ + return string_attr_error(attrs, + "arith.add", + "wrap", + k_int_add_wraps, + "arith.add: wrap must be the string 'nsw' or 'nuw'", + "nuw", + operand_type, + buf, + cap); +} + +const char* rocke_i_cmp_pred_error(const rocke_attr_map_t* attrs, + const rocke_type_t* operand_type, + char* buf, + size_t cap) +{ + return string_attr_error(attrs, + "arith.cmp", + "pred", + k_int_cmp_predicates, + "arith.cmp: pred must be a string", + "ult", + operand_type, + buf, + cap); +} + bool rocke_attr_get_bool(const rocke_attr_map_t* m, const char* key, bool dflt) { const rocke_attr_value_t* v = rocke_attr_get(m, key); @@ -804,7 +886,10 @@ static const char* const rocke_opcode_names[ROCKE_OP__COUNT] = { "arith.umax", "arith.umin", "arith.uitofp_f32", - "arith.cvt_f32_to_u8_sat"}; + "arith.cvt_f32_to_u8_sat", + + /* arith.* range marker */ + "arith.assume_range"}; const char* rocke_opcode_name(rocke_opcode_t op) { @@ -1083,7 +1168,10 @@ static const bool rocke_opcode_pure[ROCKE_OP__COUNT] = { /* arith.umax */ true, /* arith.umin */ true, /* arith.uitofp_f32 */ true, - /* arith.cvt_f32_to_u8_sat */ true}; + /* arith.cvt_f32_to_u8_sat */ true, + + /* arith.* range marker */ + /* arith.assume_range */ true}; bool rocke_opcode_is_pure(rocke_opcode_t op) { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_arith.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_arith.cpp index e1d1e5a25268..9bb85116a67d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_arith.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_arith.cpp @@ -146,6 +146,48 @@ rocke_value_t* rocke_b_add(rocke_ir_builder_t* b, rocke_value_t* a, rocke_value_ return rocke_i_binop(b, ROCKE_OP_ARITH_ADD, a, c, "add"); } +/* Python _check_unsigned_int_operands: both lowerings read the operands as + * uint32_t / uint64_t by width, so they are two i32 or two i64 values. Returns + * false (with the builder error set) when they are not. */ +static bool check_unsigned_int_operands(rocke_ir_builder_t* b, + const char* op_name, + const rocke_value_t* a, + const rocke_value_t* c) +{ + if(!a || !c) + { + rocke_i_set_err(b, ROCKE_ERR_VALUE, "%s NULL operand", op_name); + return false; + } + if(!rocke_type_eq(a->type, c->type) + || (!rocke_i_type_is(a->type, "i32") && !rocke_i_type_is(a->type, "i64"))) + { + rocke_i_set_err(b, + ROCKE_ERR_VALUE, + "%s expects two i32 or two i64 operands, got %s / %s", + op_name, + a->type->name, + c->type->name); + return false; + } + return true; +} + +rocke_value_t* rocke_b_add_nuw(rocke_ir_builder_t* b, rocke_value_t* a, rocke_value_t* c) +{ + rocke_value_t* operands[2]; + rocke_attr_map_t at; + if(!rocke_i_live(b)) + return NULL; + if(!check_unsigned_int_operands(b, "add_nuw", a, c)) + return NULL; + operands[0] = a; + operands[1] = c; + at = rocke_i_attrs(b); + rocke_attr_set_str(b, &at, "wrap", "nuw"); + return rocke_i_op1(b, ROCKE_OP_ARITH_ADD, operands, 2, a->type, &at, "add"); +} + rocke_value_t* rocke_b_sub(rocke_ir_builder_t* b, rocke_value_t* a, rocke_value_t* c) { return rocke_i_binop(b, ROCKE_OP_ARITH_SUB, a, c, "sub"); @@ -436,6 +478,15 @@ rocke_value_t* rocke_b_cmp_ne(rocke_ir_builder_t* b, rocke_value_t* a, rocke_val return cmp_impl(b, a, c, "ne", "ne"); } +rocke_value_t* rocke_b_cmp_ult(rocke_ir_builder_t* b, rocke_value_t* a, rocke_value_t* c) +{ + if(!rocke_i_live(b)) + return NULL; + if(!check_unsigned_int_operands(b, "cmp_ult", a, c)) + return NULL; + return cmp_impl(b, a, c, "ult", "ult"); +} + rocke_value_t* rocke_b_fcmp(rocke_ir_builder_t* b, const char* pred, rocke_value_t* a, rocke_value_t* c) { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/core.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/core.cpp index 8d33c2e15930..48631a78ff37 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/core.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/core.cpp @@ -278,6 +278,26 @@ const char* rocke_h_name(rocke_h_lowerer_t* lw, const rocke_value_t* v) return n; } +/* The unsigned HIP spelling of a scalar integer type (Python + * _HIP_UNSIGNED_INT_TYPE); NULL for any other type. */ +const char* rocke_h_unsigned_int_type(const rocke_type_t* t) +{ + const char* name = t ? t->name : NULL; + if(!name) + return NULL; + if(strcmp(name, "i1") == 0) + return "bool"; + if(strcmp(name, "i8") == 0 || strcmp(name, "u8") == 0) + return "uint8_t"; + if(strcmp(name, "i16") == 0) + return "uint16_t"; + if(strcmp(name, "i32") == 0) + return "uint32_t"; + if(strcmp(name, "i64") == 0) + return "uint64_t"; + return NULL; +} + /* The raw _HIP_TYPE dict: scalar IR name -> HIP scalar spelling. NULL if * unknown (caller decides whether that is an error). * No packed-only type has an entry: rocke_i_refuse_packed_only_value keeps diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_arith.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_arith.cpp index 47f912b449f3..ee7de1d2d508 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_arith.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_arith.cpp @@ -211,10 +211,54 @@ static rocke_status_t rocke_h_op_arith_constant_vec(rocke_h_lowerer_t* lw, const /* ----------------------------- arith: int binary --------------------------- */ -/* def _op_arith_add(self, op): self._binary(op, "+") */ +/* def _op_arith_add(self, op): "nsw" (the default) is self._binary(op, "+"); + * "nuw" adds as unsigned and casts back, because a signed a + b that passes + * the signed maximum is undefined behaviour in C++ (clang emits it nsw). */ static rocke_status_t rocke_h_op_arith_add(rocke_h_lowerer_t* lw, const rocke_op_t* op) { - h_binary(lw, op, "+"); + char wrap_buf[128]; + const char* wrap_error + = rocke_i_add_wrap_error(&op->attrs, op->operands[0]->type, wrap_buf, sizeof wrap_buf); + if(wrap_error) + { + return rocke_h_fail(lw, ROCKE_ERR_VALUE, "%s", wrap_error); + } + const char* wrap = rocke_attr_get_str(&op->attrs, "wrap"); + if(wrap == NULL || strcmp(wrap, "nsw") == 0) + { + h_binary(lw, op, "+"); + return lw->status; + } + const rocke_value_t* a = op->operands[0]; + const rocke_value_t* b = op->operands[1]; + const rocke_value_t* r = h_res(op); + const char* unsigned_t = rocke_h_unsigned_int_type(r->type); + if(!unsigned_t) + { + return rocke_h_fail(lw, ROCKE_ERR_KEY, "%s", r->type->name); + } + rocke_h_emitf(lw, + "%s %s = (int%s_t)((%s)%s + (%s)%s);", + rocke_h_type_to_hip(lw, r->type), + rocke_h_name(lw, r), + r->type->name + 1, + unsigned_t, + rocke_h_name(lw, a), + unsigned_t, + rocke_h_name(lw, b)); + return lw->status; +} + +/* def _op_arith_assume_range(self, op): a range marker is a copy of its + * first operand. */ +static rocke_status_t rocke_h_op_arith_assume_range(rocke_h_lowerer_t* lw, const rocke_op_t* op) +{ + const rocke_value_t* r = h_res(op); + rocke_h_emitf(lw, + "%s %s = %s;", + rocke_h_type_to_hip(lw, r->type), + rocke_h_name(lw, r), + rocke_h_name(lw, op->operands[0])); return lw->status; } @@ -264,6 +308,22 @@ static rocke_status_t rocke_h_op_arith_cmp(rocke_h_lowerer_t* lw, const rocke_op { pred = "lt"; } + if(strcmp(pred, "ult") == 0) + { + const char* unsigned_t = rocke_h_unsigned_int_type(a->type); + if(!unsigned_t) + { + return rocke_h_fail(lw, ROCKE_ERR_KEY, "%s", a->type->name); + } + rocke_h_emitf(lw, + "bool %s = (%s)%s < (%s)%s;", + rocke_h_name(lw, r), + unsigned_t, + rocke_h_name(lw, a), + unsigned_t, + rocke_h_name(lw, b)); + return lw->status; + } if(strcmp(pred, "lt") == 0) { c_op = "<"; @@ -868,6 +928,7 @@ const rocke_h_handler_entry_t* rocke_h_handlers_arith(void) {ROCKE_OP_ARITH_CONSTANT, rocke_h_op_arith_constant}, {ROCKE_OP_ARITH_CONSTANT_VEC, rocke_h_op_arith_constant_vec}, {ROCKE_OP_ARITH_ADD, rocke_h_op_arith_add}, + {ROCKE_OP_ARITH_ASSUME_RANGE, rocke_h_op_arith_assume_range}, {ROCKE_OP_ARITH_SUB, rocke_h_op_arith_sub}, {ROCKE_OP_ARITH_MUL, rocke_h_op_arith_mul}, {ROCKE_OP_ARITH_DIV, rocke_h_op_arith_div}, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_cast.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_cast.cpp index 4d2e75746cce..d996cdc1dcb4 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_cast.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_cast.cpp @@ -33,24 +33,6 @@ static const rocke_value_t* h_res(const rocke_op_t* op) /* ----------------------------- arith: int width casts ---------------------- */ -static const char* h_unsigned_int_type(const rocke_type_t* t) -{ - const char* name = t ? t->name : NULL; - if(!name) - return NULL; - if(__builtin_strcmp(name, "i1") == 0) - return "bool"; - if(__builtin_strcmp(name, "i8") == 0 || __builtin_strcmp(name, "u8") == 0) - return "uint8_t"; - if(__builtin_strcmp(name, "i16") == 0) - return "uint16_t"; - if(__builtin_strcmp(name, "i32") == 0) - return "uint32_t"; - if(__builtin_strcmp(name, "i64") == 0) - return "uint64_t"; - return NULL; -} - /* Zero extension must cast through the unsigned spelling of the source type. * A direct int8_t -> int/int64_t cast sign-extends bytes 0x80..0xff. */ static rocke_status_t rocke_h_op_arith_zext(rocke_h_lowerer_t* lw, const rocke_op_t* op) @@ -58,7 +40,7 @@ static rocke_status_t rocke_h_op_arith_zext(rocke_h_lowerer_t* lw, const rocke_o const rocke_value_t* v = op->operands[0]; const rocke_value_t* r = h_res(op); const char* t = rocke_h_type_to_hip(lw, r->type); - const char* source_unsigned = h_unsigned_int_type(v->type); + const char* source_unsigned = rocke_h_unsigned_int_type(v->type); if(!source_unsigned) { return rocke_h_fail(lw, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/arith.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/arith.cpp index 65e2ab4ad972..60fbe8a29b77 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/arith.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/arith.cpp @@ -181,8 +181,31 @@ static void ll_wrapping_binop(rocke_lower_t* L, } static void _op_arith_add(rocke_lower_t* L, const rocke_op_t* op) { + /* Python: an explicit no-unsigned-wrap add is "add nuw"; otherwise the + * type's default (nsw for signed values, no flag for unsigned ones). */ + char wrap_buf[128]; + const char* wrap_error + = rocke_i_add_wrap_error(&op->attrs, op->operands[0]->type, wrap_buf, sizeof wrap_buf); + if(wrap_error) + { + rocke_ll_fail(L, ROCKE_ERR_VALUE, "%s", wrap_error); + return; + } + const char* wrap = rocke_attr_get_str(&op->attrs, "wrap"); + if(wrap && strcmp(wrap, "nuw") == 0) + { + rocke_ll_binop(L, op, "add nuw"); + return; + } ll_wrapping_binop(L, op, "add nsw", "add"); } +/* Python _op_arith_assume_range: emits nothing; rocke_ll_operand prints the + * marked value (operand 0) at every use of the marker's result. */ +static void _op_arith_assume_range(rocke_lower_t* L, const rocke_op_t* op) +{ + (void)L; + (void)op; +} static void _op_arith_sub(rocke_lower_t* L, const rocke_op_t* op) { ll_wrapping_binop(L, op, "sub nsw", "sub"); @@ -890,6 +913,7 @@ void rocke_ll_register_arith(void) rocke_ll_set_handler(ROCKE_OP_ARITH_CONSTANT, _op_arith_constant); rocke_ll_set_handler(ROCKE_OP_ARITH_CONSTANT_VEC, _op_arith_constant_vec); rocke_ll_set_handler(ROCKE_OP_ARITH_ADD, _op_arith_add); + rocke_ll_set_handler(ROCKE_OP_ARITH_ASSUME_RANGE, _op_arith_assume_range); rocke_ll_set_handler(ROCKE_OP_ARITH_SUB, _op_arith_sub); rocke_ll_set_handler(ROCKE_OP_ARITH_MUL, _op_arith_mul); rocke_ll_set_handler(ROCKE_OP_ARITH_DIV, _op_arith_div); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp index 11a61c9819dc..602e3694d834 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp @@ -905,6 +905,11 @@ const char* rocke_ll_operand(rocke_lower_t* L, const rocke_value_t* v) { return v->name; } + if(op->opcode == ROCKE_OP_ARITH_ASSUME_RANGE && op->num_operands >= 1) + { + /* A range marker is its first operand: it emits no instruction. */ + return rocke_ll_operand(L, op->operands[0]); + } if(op->opcode == ROCKE_OP_ARITH_CONSTANT) { const char* ity = rocke_attr_get_str(&op->attrs, "ity"); @@ -1020,6 +1025,8 @@ const char* rocke_ll_icmp_predicate(rocke_lower_t* L, const char* signed_form; const char* unsigned_form; } k_icmp[] = { + /* "ult" is the explicit unsigned compare: unsigned whatever the type. */ + {"ult", "ult", "ult"}, {"lt", "slt", "ult"}, {"le", "sle", "ule"}, {"gt", "sgt", "ugt"}, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/passes.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/passes.cpp index 168230163f3b..aaa457f0327d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/passes.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/passes.cpp @@ -444,8 +444,42 @@ static const char* constant_ity(const rocke_type_t* t) return "i32"; } +/* Python _int_width(t): bit width of an integer constant's type (the folder's + * three widths). */ +static int int_width(const rocke_type_t* t) +{ + if(t && t->name) + { + if(strcmp(t->name, "i1") == 0) + return 1; + if(strcmp(t->name, "i64") == 0) + return 64; + } + return 32; +} + +/* Python _wrap_to_width(value, t): `value` as the hardware holds it in an + * integer of type `t`. The fold result stands for a fixed-width register, so it + * wraps as two's complement: an i32 sum of 2^31 - 1 and 1 (defined for + * add_nuw) reads back as -2^31, which is what later signed folds must compare. + * An i1 keeps 0 / 1, the form the compare folds and the printers use. */ +static int64_t wrap_to_width(int64_t value, const rocke_type_t* t) +{ + const int width = int_width(t); + if(width == 1) + return value & 1; + if(width == 64) + return value; + const uint64_t mask = ((uint64_t)1 << width) - 1; + const uint64_t bits = (uint64_t)value & mask; + const uint64_t sign = (uint64_t)1 << (width - 1); + /* Sign-extend without a signed overflow: (bits ^ sign) - sign. */ + return (int64_t)((bits ^ sign) - sign); +} + /* Python _const_int(v): returns the int value of a constant operand iff its op - * is "arith.constant" with ity in {i1,i32,i64}. *out is set; returns true. */ + * is "arith.constant" with ity in {i1,i32,i64}, read at the constant's width. + * *out is set; returns true. */ static bool const_int(const rocke_value_t* v, int64_t* out) { rocke_op_t* op; @@ -467,21 +501,17 @@ static bool const_int(const rocke_value_t* v, int64_t* out) /* Python int(op.attrs["value"]). Constant ints are stored as INT; tolerate * a FLOAT just in case (truncate toward zero). */ if(val->kind == ROCKE_ATTR_INT) - { *out = val->u.i; - return true; - } - if(val->kind == ROCKE_ATTR_FLOAT) - { + else if(val->kind == ROCKE_ATTR_FLOAT) *out = (int64_t)val->u.f; - return true; - } - if(val->kind == ROCKE_ATTR_BOOL) - { + else if(val->kind == ROCKE_ATTR_BOOL) *out = val->u.b ? 1 : 0; - return true; - } - return false; + else + return false; + /* Read at the constant's width, as folded results are written: an i32 + * literal 0x80000000 and a folded i32 sum of the same bits are one value. */ + *out = wrap_to_width(*out, v->type); + return true; } /* Python _try_fold(op): returns true and writes *out for a foldable integer op @@ -492,6 +522,10 @@ static bool try_fold(const rocke_op_t* op, int64_t* out) int i; if(op->num_results != 1) return false; + /* Python: the marker is its first operand; a runtime end (operand 1) does + * not stop a constant marked value from folding. */ + if(op->opcode == ROCKE_OP_ARITH_ASSUME_RANGE) + return op->num_operands >= 1 && const_int(op->operands[0], out); /* Every foldable op takes 1..3 operands; a different arity cannot be one. */ if(op->num_operands < 1 || op->num_operands > 3) return false; @@ -507,17 +541,19 @@ static bool try_fold(const rocke_op_t* op, int64_t* out) case ROCKE_OP_ARITH_ADD: if(op->num_operands != 2) return false; - *out = ints[0] + ints[1]; + /* Unsigned arithmetic: wraps at 64 bits where a signed sum would be + * undefined, matching Python's sum wrapped to i64. */ + *out = (int64_t)((uint64_t)ints[0] + (uint64_t)ints[1]); return true; case ROCKE_OP_ARITH_SUB: if(op->num_operands != 2) return false; - *out = ints[0] - ints[1]; + *out = (int64_t)((uint64_t)ints[0] - (uint64_t)ints[1]); return true; case ROCKE_OP_ARITH_MUL: if(op->num_operands != 2) return false; - *out = ints[0] * ints[1]; + *out = (int64_t)((uint64_t)ints[0] * (uint64_t)ints[1]); return true; case ROCKE_OP_ARITH_DIV: if(op->num_operands != 2 || ints[1] == 0) @@ -548,6 +584,15 @@ static bool try_fold(const rocke_op_t* op, int64_t* out) *out = ints[0] | ints[1]; return true; case ROCKE_OP_ARITH_ZEXT: + if(op->num_operands != 1) + return false; + /* Zero-extend: the operand read as unsigned at its own width. */ + { + const int width = int_width(op->operands[0]->type); + const uint64_t mask = width == 64 ? ~(uint64_t)0 : (((uint64_t)1 << width) - 1); + *out = (int64_t)((uint64_t)ints[0] & mask); + } + return true; case ROCKE_OP_ARITH_SEXT: if(op->num_operands != 1) return false; @@ -564,6 +609,14 @@ static bool try_fold(const rocke_op_t* op, int64_t* out) pred = "lt"; /* op.attrs.get("pred", "lt") */ a = ints[0]; bc = ints[1]; + if(strcmp(pred, "ult") == 0) + { + /* Unsigned at the operand width: -1 reads as the largest value. */ + int width = int_width(op->operands[0]->type); + uint64_t mask = width == 64 ? ~(uint64_t)0 : (((uint64_t)1 << width) - 1); + *out = (((uint64_t)a & mask) < ((uint64_t)bc & mask)); + return true; + } if(strcmp(pred, "lt") == 0) *out = (a < bc); else if(strcmp(pred, "le") == 0) @@ -593,7 +646,8 @@ static bool try_fold(const rocke_op_t* op, int64_t* out) /* Rewrite `op` in place into an arith.constant of integer value `value`. * Python: * op.name = "arith.constant"; op.operands = [] - * op.attrs = {"value": value, "ity": _constant_ity(op.result.type)} + * op.attrs = {"value": _wrap_to_width(value, op.result.type), + * "ity": _constant_ity(op.result.type)} * op.regions = [] */ static void fold_to_constant(rocke_ir_builder_t* b, rocke_op_t* op, int64_t value) @@ -606,7 +660,7 @@ static void fold_to_constant(rocke_ir_builder_t* b, rocke_op_t* op, int64_t valu op->regions = NULL; op->num_regions = 0; rocke_attr_map_init(&op->attrs); - rocke_attr_set_int(b, &op->attrs, "value", value); + rocke_attr_set_int(b, &op->attrs, "value", wrap_to_width(value, rty)); rocke_attr_set_str(b, &op->attrs, "ity", constant_ity(rty)); } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/verify.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/verify.cpp index 13494047fdf4..ca5c86f1669a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/verify.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/verify.cpp @@ -32,6 +32,7 @@ #include #include "rocke/ir.h" +#include "rocke/ir_internal.h" /* ----------------------------------------------------- diagnostic buffer */ @@ -294,8 +295,12 @@ static const char* const* required_attrs(rocke_opcode_t o, int* n) static const char* asm_keys[] = {"template", "constraints"}; static const char* scope_order_keys[] = {"scope", "ordering"}; static const char* scope_keys[] = {"scope"}; + static const char* assume_range_keys[] = {"multiple_of", "residue"}; switch(o) { + case ROCKE_OP_ARITH_ASSUME_RANGE: + *n = 2; + return assume_range_keys; case ROCKE_OP_ARITH_CONSTANT: *n = 2; return constant_keys; @@ -355,6 +360,70 @@ static void check_sync_attrs(verifier_t* v, const rocke_op_t* op, const char* co v_errf(v, op, "%s: unknown ordering '%s'", op->name, ordering); } +/* Python ir._assume_range_errors: every rule an arith.assume_range marker's + * attrs break, in order, with the same texts. */ +static void check_assume_range_attrs(verifier_t* v, const rocke_op_t* op) +{ + /* ir._ASSUME_RANGE_INT_ATTRS: every present attr must be an integer before + * the value rules below compare it. */ + static const char* const int_attrs[] = {"multiple_of", "residue", "lo", "hi", "hi_plus"}; + bool typed = true; + for(const char* key : int_attrs) + { + const rocke_attr_value_t* value = rocke_attr_get(&op->attrs, key); + if(value && value->kind != ROCKE_ATTR_INT) + { + v_errf(v, op, "arith.assume_range: %s must be an integer", key); + typed = false; + } + } + if(!typed) + { + return; + } + int64_t multiple_of = 1; + int64_t residue = 0; + int64_t lo = 0; + int64_t hi = 0; + int64_t unused = 0; + rocke_attr_get_int(&op->attrs, "multiple_of", &multiple_of); + rocke_attr_get_int(&op->attrs, "residue", &residue); + if(multiple_of < 1) + { + v_errf(v, + op, + "arith.assume_range: multiple_of must be >= 1, got %lld", + (long long)multiple_of); + } + else if(!(residue >= 0 && residue < multiple_of)) + { + v_errf(v, + op, + "arith.assume_range: residue must be in [0, %lld), got %lld", + (long long)multiple_of, + (long long)residue); + } + const bool has_hi_value = op->num_operands == 2; + const bool has_lo = rocke_attr_get_int(&op->attrs, "lo", &lo); + const bool has_hi = rocke_attr_get_int(&op->attrs, "hi", &hi); + if(has_hi && has_hi_value) + { + v_errf(v, op, "arith.assume_range: hi and hi_value are both given; state at most one end"); + } + if(rocke_attr_get_int(&op->attrs, "hi_plus", &unused) && !has_hi_value) + { + v_errf(v, op, "arith.assume_range: hi_plus is given without hi_value"); + } + if(has_lo && has_hi && lo >= hi) + { + v_errf(v, + op, + "arith.assume_range: lo (%lld) must be below hi (%lld)", + (long long)lo, + (long long)hi); + } +} + static void check_contract(verifier_t* v, const rocke_op_t* op) { if(op->opcode == ROCKE_OP_TILE_MMA) @@ -394,6 +463,15 @@ static void check_contract(verifier_t* v, const rocke_op_t* op) { v_errf(v, op, "%s: result type %s != operand type %s", name, tn(r), tn(a)); } + char wrap_buf[128]; + const char* wrap_error + = o == ROCKE_OP_ARITH_ADD + ? rocke_i_add_wrap_error(&op->attrs, a, wrap_buf, sizeof wrap_buf) + : NULL; + if(wrap_error) + { + v_errf(v, op, "%s", wrap_error); + } } else if(is_unary_same_type(o)) { @@ -438,6 +516,48 @@ static void check_contract(verifier_t* v, const rocke_op_t* op) { v_errf(v, op, "%s: result must be i1, got %s", name, tn(op->results[0]->type)); } + char pred_buf[128]; + const char* pred_error + = o == ROCKE_OP_ARITH_CMP + ? rocke_i_cmp_pred_error(&op->attrs, a, pred_buf, sizeof pred_buf) + : NULL; + if(pred_error) + { + v_errf(v, op, "%s", pred_error); + } + } + else if(o == ROCKE_OP_ARITH_ASSUME_RANGE) + { + if((op->num_operands != 1 && op->num_operands != 2) || op->num_results != 1) + { + v_errf(v, + op, + "%s: expected 1 or 2 operands / 1 result, got %d / %d", + name, + op->num_operands, + op->num_results); + return; + } + const rocke_type_t* t = op->operands[0]->type; + if(!rocke_type_eq(op->results[0]->type, t)) + { + v_errf(v, + op, + "%s: result type %s != operand type %s", + name, + tn(op->results[0]->type), + tn(t)); + } + if(op->num_operands == 2 && !rocke_type_eq(op->operands[1]->type, t)) + { + v_errf(v, + op, + "%s: hi_value type %s != operand type %s", + name, + tn(op->operands[1]->type), + tn(t)); + } + check_assume_range_attrs(v, op); } else if(o == ROCKE_OP_ARITH_SELECT) { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h index 9e21c5182d1e..ac2150c8f6fe 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h @@ -465,6 +465,10 @@ typedef enum rocke_opcode ROCKE_OP_ARITH_UITOFP_F32, ROCKE_OP_ARITH_CVT_F32_TO_U8_SAT, + /* Range marker (Python IRBuilder._assume_range): its result is its first + * operand; it emits no instruction. Appended so no existing index shifts. */ + ROCKE_OP_ARITH_ASSUME_RANGE, + ROCKE_OP__COUNT } rocke_opcode_t; @@ -798,6 +802,9 @@ rocke_value_t* rocke_b_zero_vec(rocke_ir_builder_t* b, const rocke_type_t* elem, /* ----- arith integer / logic ----- */ rocke_value_t* rocke_b_add(rocke_ir_builder_t* b, rocke_value_t* a, rocke_value_t* c); +/* Python add_nuw: integer a + c that does not wrap as unsigned ("add nuw"). + * Operands are two i32 or two i64 values. */ +rocke_value_t* rocke_b_add_nuw(rocke_ir_builder_t* b, rocke_value_t* a, rocke_value_t* c); rocke_value_t* rocke_b_sub(rocke_ir_builder_t* b, rocke_value_t* a, rocke_value_t* c); rocke_value_t* rocke_b_mul(rocke_ir_builder_t* b, rocke_value_t* a, rocke_value_t* c); rocke_value_t* rocke_b_div(rocke_ir_builder_t* b, rocke_value_t* a, rocke_value_t* c); @@ -843,6 +850,9 @@ rocke_value_t* rocke_b_cmp_gt(rocke_ir_builder_t* b, rocke_value_t* a, rocke_val rocke_value_t* rocke_b_cmp_ge(rocke_ir_builder_t* b, rocke_value_t* a, rocke_value_t* c); rocke_value_t* rocke_b_cmp_eq(rocke_ir_builder_t* b, rocke_value_t* a, rocke_value_t* c); rocke_value_t* rocke_b_cmp_ne(rocke_ir_builder_t* b, rocke_value_t* a, rocke_value_t* c); +/* Python cmp_ult: unsigned a < c ("icmp ult"). Operands are two i32 or two i64 + * values. */ +rocke_value_t* rocke_b_cmp_ult(rocke_ir_builder_t* b, rocke_value_t* a, rocke_value_t* c); /* pred in {olt,ole,ogt,oge,oeq,one,ord,uno} */ rocke_value_t* rocke_b_fcmp(rocke_ir_builder_t* b, const char* pred, rocke_value_t* a, rocke_value_t* c); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir_internal.h index f56f4876e9b8..b8204eedc453 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir_internal.h @@ -206,6 +206,22 @@ rocke_attr_map_t rocke_i_attrs(rocke_ir_builder_t* b); * attrs pointer; Op.attrs is dict(attrs or {}) in Python). */ void rocke_i_attrs_copy(rocke_ir_builder_t* b, rocke_attr_map_t* dst, const rocke_attr_map_t* src); +/* Mirrors ir._int_add_wrap_error: the error for arith.add's "wrap" attr on + * operands of type `operand_type`, or NULL when it is absent (meaning nsw) or + * valid; "nuw" takes i32 or i64 operands. A formatted message is written into + * `buf`, which the result then points to. */ +const char* rocke_i_add_wrap_error(const rocke_attr_map_t* attrs, + const rocke_type_t* operand_type, + char* buf, + size_t cap); + +/* Mirrors ir._int_cmp_pred_error: the same for arith.cmp's "pred" attr + * (absent means lt; "ult" takes i32 or i64 operands). */ +const char* rocke_i_cmp_pred_error(const rocke_attr_map_t* attrs, + const rocke_type_t* operand_type, + char* buf, + size_t cap); + /* ------------------------------------------------------------- DPP helpers */ /* DPP control word that lowers tile.dpp_xor with `xor_mask` (Python diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/lower_hip_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/lower_hip_internal.h index 68bcd80b78f3..f870454e3546 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/lower_hip_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/lower_hip_internal.h @@ -164,6 +164,11 @@ const char* rocke_h_hip_scalar(const char* ir_scalar_name); * element) is refused with one message wherever it is met. */ const char* rocke_h_require_hip_scalar(rocke_h_lowerer_t* lw, const char* ir_type_name); +/* The unsigned HIP spelling of a scalar integer type (Python + * _HIP_UNSIGNED_INT_TYPE): "bool" / "uint8_t" / "uint16_t" / "uint32_t" / + * "uint64_t" for i1 / i8 and u8 / i16 / i32 / i64. Returns NULL for any other type. */ +const char* rocke_h_unsigned_int_type(const rocke_type_t* t); + /* The "f16x"/"bf16x"/"f32x"/"i32x"/"i16x"/"i8x" vector-name prefix for an IR * scalar element name, used by the smem/global vN load/store handlers. Returns * the Python fallback "f16x" for names not in the small map (matching the diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/ir_lowering/ir_model.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/ir_lowering/ir_model.md index 619237bfc178..62e922907d9d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/ir_lowering/ir_model.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/ir_lowering/ir_model.md @@ -103,10 +103,12 @@ zero_vec(elem, n) # f32 / f16 / bf16 / f64 / fp8e4m3 / bf8e5m2 / i8 / i ```text add, sub, mul, div, mod # integer arithmetic +add_nuw # integer add, no unsigned wrap (i32/i64) fadd, fsub, fmul, fdiv, fneg # f64/f32/f16/bf16 arithmetic fmax, fmin # llvm.maxnum / llvm.minnum (f64/f32/f16/bf16) fcmp(pred, a, b) # pred in {olt,ole,ogt,oge,oeq,one,ord,uno} cmp_lt, cmp_le, cmp_gt, cmp_ge, cmp_eq, cmp_ne # integer compare (-> i1) +cmp_ult # unsigned compare (i32/i64 -> i1) land, lor, lnot # bitwise + i1 logic zext(v, target_int_type) sext(v, target_int_type) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/op_vocabulary.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/op_vocabulary.md index a4980daf17c5..6b3aa5f1bdc6 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/op_vocabulary.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/op_vocabulary.md @@ -8,7 +8,7 @@ Complete reference of operations recognized by `core/ir.py` and lowered to AMDGP |------------------------|-----------------------------------|----------------------------------------| | `arith.constant` | `const_i32`, `const_i64`, `const_f32`, `const_f64`, `fp16_zero` | constant value | | `arith.constant_vec` | `zero_vec_f32`, `zero_vec` | `zeroinitializer` vector | -| `arith.add` | `add` | `add` (integer) | +| `arith.add` | `add`, `add_nuw` | `add nsw` (`add`), `add nuw` (`add_nuw`, attr `wrap = "nuw"`) | | `arith.sub` | `sub` | `sub` | | `arith.mul` | `mul` | `mul` | | `arith.div` | `div` | `sdiv` | @@ -21,7 +21,8 @@ Complete reference of operations recognized by `core/ir.py` and lowered to AMDGP | `arith.fneg` | `fneg` | `fneg` | | `arith.fmax` | `fmax` | `llvm.maxnum.{f64,f32,f16,bf16}` | | `arith.fmin` | `fmin` | `llvm.minnum.{f64,f32,f16,bf16}` | -| `arith.cmp` | `cmp_lt`, `cmp_le`, `cmp_gt`, `cmp_ge`, `cmp_eq`, `cmp_ne` | `icmp {slt,sle,sgt,sge,eq,ne}` | +| `arith.cmp` | `cmp_lt`, `cmp_le`, `cmp_gt`, `cmp_ge`, `cmp_eq`, `cmp_ne`, `cmp_ult` | `icmp {slt,sle,sgt,sge,eq,ne,ult}` | +| `arith.assume_range` | (private `_assume_range`, for checked callers only) | nothing: uses of the result print the marked value; the attrs record a range and congruence for the tiling layer's range walk | | `arith.fcmp` | `fcmp(pred, ...)` | `fcmp {olt,ole,ogt,oge,oeq,one,ord,uno}` | | `arith.and` | `land` | `and` | | `arith.or` | `lor` | `or` | diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py index b99c6173cb6a..1c6624a8096e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py @@ -307,6 +307,109 @@ def _is_flag_ptr(t: "Type") -> bool: return isinstance(t, PtrType) and t.space == "global" and t.pointee.name == "i32" +# ``arith.cmp`` predicates: the signed orderings, equality, and one unsigned +# ordering (``ult``, which reads both operands as unsigned at their width). +_INT_CMP_PREDICATES = ("lt", "le", "gt", "ge", "eq", "ne", "ult") +# ``arith.add`` overflow flag: ``nsw`` (no signed wrap, the default when the +# attr is absent) or ``nuw`` (no unsigned wrap). +_INT_ADD_WRAPS = ("nsw", "nuw") + + +def _int_add_wrap_error(wrap: Any, operand_type: str) -> Optional[str]: + """The error for an ``arith.add`` ``wrap`` attr on operands of type + ``operand_type`` (a type name), or None when it is valid. + + Absent (None) means ``nsw``. ``nuw`` is ``add_nuw``'s form, so it takes the + builder's operands: i32 or i64. The verifier and both lowerings use it, and + the C++ engine reports the same texts. + """ + if wrap is None: + return None + if not isinstance(wrap, str): + return "arith.add: wrap must be the string 'nsw' or 'nuw'" + if wrap not in _INT_ADD_WRAPS: + return f"arith.add: unknown wrap '{wrap}'" + if wrap == "nuw" and operand_type not in ("i32", "i64"): + return f"arith.add: wrap 'nuw' needs i32 or i64 operands, got {operand_type}" + return None + + +def _int_cmp_pred_error(pred: Any, operand_type: str) -> Optional[str]: + """The error for an ``arith.cmp`` ``pred`` attr on operands of type + ``operand_type`` (a type name), or None when it is valid. + + Absent (None) means ``lt``. ``ult`` is ``cmp_ult``'s form, so it takes the + builder's operands: i32 or i64. The C++ verifier reports the same texts. + """ + if pred is None: + return None + if not isinstance(pred, str): + return "arith.cmp: pred must be a string" + if pred not in _INT_CMP_PREDICATES: + return f"arith.cmp: unknown pred '{pred}'" + if pred == "ult" and operand_type not in ("i32", "i64"): + return f"arith.cmp: pred 'ult' needs i32 or i64 operands, got {operand_type}" + return None + + +# ``arith.assume_range`` integer attrs, in the order their type is checked. +_ASSUME_RANGE_INT_ATTRS = ("multiple_of", "residue", "lo", "hi", "hi_plus") + + +def _assume_range_errors(attrs: Dict[str, Any], num_operands: int) -> List[str]: + """Every rule an ``arith.assume_range`` marker's attrs break, in order. + + The one home of the marker's attr rules: the builder raises the first + message and the verifier reports all of them. The C++ verifier emits the + same texts. ``num_operands`` is 2 when the end is a runtime value + (``hi_value``) and 1 otherwise. + """ + errors = [ + f"arith.assume_range: {key} must be an integer" + for key in _ASSUME_RANGE_INT_ATTRS + if key in attrs + and (not isinstance(attrs[key], int) or isinstance(attrs[key], bool)) + ] + if errors: + # The value rules below compare these attrs as integers. + return errors + multiple_of = attrs.get("multiple_of", 1) + residue = attrs.get("residue", 0) + if multiple_of < 1: + errors.append( + f"arith.assume_range: multiple_of must be >= 1, got {multiple_of}" + ) + elif not 0 <= residue < multiple_of: + errors.append( + f"arith.assume_range: residue must be in [0, {multiple_of}), got {residue}" + ) + has_hi_value = num_operands == 2 + if "hi" in attrs and has_hi_value: + errors.append( + "arith.assume_range: hi and hi_value are both given; state at most one end" + ) + if "hi_plus" in attrs and not has_hi_value: + errors.append("arith.assume_range: hi_plus is given without hi_value") + if "lo" in attrs and "hi" in attrs and attrs["lo"] >= attrs["hi"]: + errors.append( + f"arith.assume_range: lo ({attrs['lo']}) must be below hi ({attrs['hi']})" + ) + return errors + + +def _check_unsigned_int_operands(op_name: str, a: "Value", b: "Value") -> None: + """Refuse operands an unsigned integer op cannot take. + + Both lowerings read the operands as ``uint32_t`` / ``uint64_t`` by width, + so the operands are two i32 or two i64 values. + """ + if a.type != b.type or a.type.name not in ("i32", "i64"): + raise ValueError( + f"{op_name} expects two i32 or two i64 operands, got " + f"{a.type.name} / {b.type.name}" + ) + + def _check_flag_ptr(op_name: str, ptr: "Value") -> None: if not _is_flag_ptr(ptr.type): raise ValueError(f"{op_name} expects a global i32 pointer, got {ptr.type.name}") @@ -1103,6 +1206,19 @@ def const_f64(self, value: float) -> Value: def add(self, a: Value, b: Value) -> Value: return self._op("arith.add", [a, b], [a.type], result_name_hint="add").result + def add_nuw(self, a: Value, b: Value) -> Value: + """Integer ``a + b`` that the caller knows does not wrap as unsigned. + + Lowers to ``add nuw`` (plain :meth:`add` is ``add nsw``). Use it for a + sum of non-negative values that may pass the signed maximum but never + the unsigned one, where ``nsw`` would make LLVM treat the result as + poison. Operands are two i32 or two i64 values. + """ + _check_unsigned_int_operands("add_nuw", a, b) + return self._op( + "arith.add", [a, b], [a.type], attrs={"wrap": "nuw"}, result_name_hint="add" + ).result + def sub(self, a: Value, b: Value) -> Value: return self._op("arith.sub", [a, b], [a.type], result_name_hint="sub").result @@ -1223,6 +1339,69 @@ def cmp_ne(self, a: Value, b: Value) -> Value: "arith.cmp", [a, b], [I1], attrs={"pred": "ne"}, result_name_hint="ne" ).result + def cmp_ult(self, a: Value, b: Value) -> Value: + """Unsigned ``a < b`` (``icmp ult``): both operands read as unsigned. + + A negative value reads as a large unsigned one, so ``cmp_ult(x, n)`` + with ``n >= 0`` is ``0 <= x < n`` in one compare. Operands are two + i32 or two i64 values. + """ + _check_unsigned_int_operands("cmp_ult", a, b) + return self._op( + "arith.cmp", [a, b], [I1], attrs={"pred": "ult"}, result_name_hint="ult" + ).result + + def _assume_range( + self, + v: Value, + *, + lo: Optional[int] = None, + hi: Optional[int] = None, + hi_value: Optional[Value] = None, + hi_plus: int = 0, + multiple_of: int = 1, + residue: int = 0, + ) -> Value: + """Return ``v`` unchanged, marked with facts its caller has checked. + + The marker records ``lo <= v``, ``v < hi`` (an int ``hi``, or + ``hi_value + hi_plus`` for an end only known at run time) and + ``v == residue (mod multiple_of)``. It emits no instruction: both + lowerings print ``v`` wherever the result is used. Its one reader is + the tiling layer's IR range walk, which trusts every marker. + + Private on purpose: a fact stated here is never checked, so only + callers that checked or proved it first may state it. Those callers + write ``b._assume_range(...) # noqa: SLF001``. + """ + if v.type not in (I32, I64): + raise ValueError(f"_assume_range: v must be i32 or i64, got {v.type.name}") + if hi_value is not None and hi_value.type != v.type: + raise ValueError( + f"_assume_range: hi_value type {hi_value.type.name} != v type {v.type.name}" + ) + attrs: Dict[str, Any] = {"multiple_of": multiple_of, "residue": residue} + if lo is not None: + attrs["lo"] = lo + if hi is not None: + attrs["hi"] = hi + operands = [v] + if hi_value is not None: + operands.append(hi_value) + attrs["hi_plus"] = hi_plus + elif hi_plus != 0: + attrs["hi_plus"] = hi_plus + errors = _assume_range_errors(attrs, len(operands)) + if errors: + raise ValueError(errors[0]) + return self._op( + "arith.assume_range", + operands, + [v.type], + attrs=attrs, + result_name_hint="range", + ).result + def fcmp(self, pred: str, a: Value, b: Value) -> Value: if pred not in ("olt", "ole", "ogt", "oge", "oeq", "one", "ord", "uno"): raise ValueError(f"unsupported fcmp predicate {pred!r}") @@ -5644,6 +5823,7 @@ def scf_if_else(self, cond: Value): "arith.cmp", "arith.fcmp", "arith.select", + "arith.assume_range", "arith.fmax", "arith.fmin", "arith.select", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py index 00afeb721cf6..4c4965fe2d76 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py @@ -25,6 +25,7 @@ _FLAG_STORE_ORDERINGS, F32, I32, + _int_add_wrap_error, KernelDef, Op, PtrType, @@ -459,7 +460,28 @@ def _binary(self, op: Op, c_op: str) -> None: ) def _op_arith_add(self, op: Op) -> None: - self._binary(op, "+") + wrap_error = _int_add_wrap_error(op.attrs.get("wrap"), op.operands[0].type.name) + if wrap_error: + raise ValueError(wrap_error) + wrap = op.attrs.get("wrap", "nsw") + if wrap == "nsw": + self._binary(op, "+") + return + # Unsigned sum, cast back: a signed ``a + b`` that passes the signed + # maximum is undefined behaviour in C++, which clang emits as ``nsw``. + a, b = op.operands + width = op.result.type.name + unsigned = _HIP_UNSIGNED_INT_TYPE[width] + self._emit( + f"{_type_to_hip(op.result.type)} {_name(op.result)} = " + f"(int{width[1:]}_t)(({unsigned}){_name(a)} + ({unsigned}){_name(b)});" + ) + + def _op_arith_assume_range(self, op: Op) -> None: + # A range marker is a copy of its first operand. + self._emit( + f"{_type_to_hip(op.result.type)} {_name(op.result)} = {_name(op.operands[0])};" + ) def _op_arith_sub(self, op: Op) -> None: self._binary(op, "-") @@ -475,10 +497,16 @@ def _op_arith_mod(self, op: Op) -> None: def _op_arith_cmp(self, op: Op) -> None: pred = op.attrs.get("pred", "lt") + a, b = op.operands + if pred == "ult": + unsigned = _HIP_UNSIGNED_INT_TYPE[a.type.name] + self._emit( + f"bool {_name(op.result)} = ({unsigned}){_name(a)} < ({unsigned}){_name(b)};" + ) + return c_op = {"lt": "<", "le": "<=", "gt": ">", "ge": ">=", "eq": "==", "ne": "!="}[ pred ] - a, b = op.operands self._emit(f"bool {_name(op.result)} = {_name(a)} {c_op} {_name(b)};") def _op_arith_select(self, op: Op) -> None: diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py index ac858d40727d..91f2c993c616 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py @@ -44,6 +44,7 @@ _FENCE_ORDERINGS, _FENCE_SCOPES, _FLAG_STORE_ORDERINGS, + _int_add_wrap_error, BF16, F32, F16, @@ -1797,6 +1798,9 @@ def _operand(self, v: Value) -> str: op = v.op if op is None: return v.name + if op.name == "arith.assume_range": + # A range marker is its first operand: it emits no instruction. + return self._operand(op.operands[0]) if op.name == "arith.constant": ity = op.attrs.get("ity", "i32") val = op.attrs["value"] @@ -2403,7 +2407,19 @@ def _wrapping_binop(self, op: Op, llvm_op: str) -> None: self._binop(op, llvm_op + flag) def _op_arith_add(self, op: Op) -> None: - self._wrapping_binop(op, "add") + wrap_error = _int_add_wrap_error(op.attrs.get("wrap"), op.operands[0].type.name) + if wrap_error: + raise ValueError(wrap_error) + # An explicit no-unsigned-wrap add; otherwise the type's default + # (nsw for signed values, no flag for unsigned ones). + if op.attrs.get("wrap") == "nuw": + self._binop(op, "add nuw") + else: + self._wrapping_binop(op, "add") + + def _op_arith_assume_range(self, op: Op) -> None: + # Emits nothing: _operand prints the marked value at every use. + return def _op_arith_sub(self, op: Op) -> None: self._wrapping_binop(op, "sub") @@ -7251,6 +7267,8 @@ def _encode_waitcnt_gfx11(vmcnt: int, expcnt: int, lgkmcnt: int) -> int: # arith.cmp / vector.cmp predicate -> (signed icmp predicate, unsigned one). _ICMP_PREDICATES = { + # "ult" is the explicit unsigned compare: unsigned whatever the type. + "ult": ("ult", "ult"), "lt": ("slt", "ult"), "le": ("sle", "ule"), "gt": ("sgt", "ugt"), diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/passes.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/passes.py index 662dac9c4f7f..8b8e7bb860c5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/passes.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/passes.py @@ -63,7 +63,10 @@ def canonicalize_region(region: Region) -> PassStats: if folded_value is not None and len(op.results) == 1: op.name = "arith.constant" op.operands = [] - op.attrs = {"value": folded_value, "ity": _constant_ity(op.result.type)} + op.attrs = { + "value": _wrap_to_width(folded_value, op.result.type), + "ity": _constant_ity(op.result.type), + } op.regions = [] folded += 1 @@ -163,12 +166,39 @@ def _const_int(v: Value) -> Optional[int]: ity = op.attrs.get("ity", "i32") if ity not in ("i1", "i32", "i64"): return None - return int(op.attrs["value"]) + # Read at the constant's width, as folded results are written: an i32 + # literal 0x80000000 and a folded i32 sum of the same bits are one value. + return _wrap_to_width(int(op.attrs["value"]), v.type) + + +def _int_width(t: Type) -> int: + """Bit width of an integer constant's type (the folder's three widths).""" + return {"i1": 1, "i64": 64}.get(t.name, 32) + + +def _wrap_to_width(value: int, t: Type) -> int: + """``value`` as the hardware holds it in an integer of type ``t``. + + The fold computes with unbounded ints, but the folded constant stands for + a fixed-width register, so it wraps as two's complement: an i32 sum of + 2**31 - 1 and 1 (defined for ``add_nuw``) reads back as -2**31, which is + what later signed folds must compare. An i1 keeps 0 / 1, the form the + compare folds and the printers use. + """ + width = _int_width(t) + if width == 1: + return value & 1 + value &= (1 << width) - 1 + return value - (1 << width) if value >> (width - 1) else value def _try_fold(op: Op) -> Optional[int]: if len(op.results) != 1: return None + if op.name == "arith.assume_range": + # The marker is its first operand; a runtime end (operand 1) does not + # stop a constant marked value from folding. + return _const_int(op.operands[0]) ints = [_const_int(v) for v in op.operands] if any(v is None for v in ints): return None @@ -187,11 +217,18 @@ def _try_fold(op: Op) -> Optional[int]: return ints[0] & ints[1] if op.name == "arith.or": return ints[0] | ints[1] - if op.name == "arith.zext" or op.name == "arith.sext": + if op.name == "arith.zext": + # Zero-extend: the operand read as unsigned at its own width. + return ints[0] & ((1 << _int_width(op.operands[0].type)) - 1) + if op.name == "arith.sext": return ints[0] if op.name == "arith.cmp": pred = op.attrs.get("pred", "lt") a, b = ints + if pred == "ult": + # Unsigned at the operand width: -1 reads as the largest value. + mask = (1 << _int_width(op.operands[0].type)) - 1 + return int((a & mask) < (b & mask)) return int( { "lt": a < b, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/verify.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/verify.py index 2cb447d78f97..e403b53af7dc 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/verify.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/verify.py @@ -38,6 +38,9 @@ _FENCE_ORDERINGS, _FENCE_SCOPES, _FLAG_STORE_ORDERINGS, + _assume_range_errors, + _int_add_wrap_error, + _int_cmp_pred_error, _is_flag_ptr, KernelDef, Op, @@ -121,6 +124,7 @@ def __str__(self) -> str: "arith.constant": ["value", "ity"], "arith.cmp": ["pred"], "arith.fcmp": ["pred"], + "arith.assume_range": ["multiple_of", "residue"], "tile.mma": ["op_id"], "scf.yield": ["num"], "tile.inline_asm": ["template", "constraints"], @@ -316,6 +320,10 @@ def _check_contract(self, op: Op, scope: Dict[str, Value]) -> None: f"{name}: result type {r.type.name} != operand type {a.type.name}", op, ) + if name == "arith.add": + wrap_error = _int_add_wrap_error(op.attrs.get("wrap"), a.type.name) + if wrap_error: + self.err(wrap_error, op) elif name in _UNARY_SAME_TYPE: if len(op.operands) != 1 or len(op.results) != 1: self.err( @@ -351,6 +359,33 @@ def _check_contract(self, op: Op, scope: Dict[str, Value]) -> None: self.err( f"{name}: result must be i1, got {op.results[0].type.name}", op ) + if name == "arith.cmp": + pred_error = _int_cmp_pred_error(op.attrs.get("pred"), a.type.name) + if pred_error: + self.err(pred_error, op) + elif name == "arith.assume_range": + if len(op.operands) not in (1, 2) or len(op.results) != 1: + self.err( + f"{name}: expected 1 or 2 operands / 1 result, got " + f"{len(op.operands)} / {len(op.results)}", + op, + ) + return + v = op.operands[0] + if op.results[0].type != v.type: + self.err( + f"{name}: result type {op.results[0].type.name} != " + f"operand type {v.type.name}", + op, + ) + if len(op.operands) == 2 and op.operands[1].type != v.type: + self.err( + f"{name}: hi_value type {op.operands[1].type.name} != " + f"operand type {v.type.name}", + op, + ) + for message in _assume_range_errors(op.attrs, len(op.operands)): + self.err(message, op) elif name == "arith.select": if len(op.operands) != 3 or len(op.results) != 1: self.err(f"{name}: expected 3 operands / 1 result", op) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt b/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt index 03fe88388f9a..b7edbccafafd 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/CMakeLists.txt @@ -197,6 +197,23 @@ if(BUILD_TESTING) rocke_xf32_bf16x3_lowering) endif() + # Unsigned compare, no-unsigned-wrap add and range marker: C API builders, + # both C++ lowerers, the verifier and the constant folder. + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/core/test_unsigned_int_ops.cpp") + add_executable(rocke_unsigned_int_ops core/test_unsigned_int_ops.cpp) + target_link_libraries(rocke_unsigned_int_ops PRIVATE rocke_core) + add_test(NAME rocke_unsigned_int_ops COMMAND rocke_unsigned_int_ops) + set_tests_properties(rocke_unsigned_int_ops + PROPERTIES LABELS "ckc;unit;host") + rocke_cxx_test_installed_name(_rocke_installed_name rocke_unsigned_int_ops) + set_target_properties(rocke_unsigned_int_ops + PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") + install(TARGETS rocke_unsigned_int_ops + RUNTIME DESTINATION ${ROCKE_CXX_TEST_INSTALL_DIR}) + set_property(GLOBAL APPEND PROPERTY ROCKE_INSTALL_TEST_TARGETS + rocke_unsigned_int_ops) + endif() + # f64 op rejection and f64 vector arithmetic in both C++ lowerers. if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/core/test_f64_lowering.cpp") add_executable(rocke_f64_lowering core/test_f64_lowering.cpp) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md b/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md index 85c38b651d25..fb1bd10cd33c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/README.md @@ -131,17 +131,20 @@ divergence - the Python builder correctly rejects wave32 WMMA on gfx942.) `run_all.py --build-root ` builds all configured targets before pytest, then obtains each native parity executable from CTest, as listed in `NATIVE_PARITY_TESTS`: `rocke_storage` (`ROCKE_STORAGE_TEST`), -`rocke_ir_types` (`ROCKE_IR_TYPES_TEST`) and `rocke_nontemporal_hip` -(`ROCKE_NONTEMPORAL_HIP_TEST`). Both pytest passes receive those paths, so the -storage IR/HIP parity, serialization, C++ builder type-rule parity and -nontemporal HIP-source parity tests run automatically. `--config` selects the native test -configuration (default `Release`). A build or fixture-discovery failure stops the -runner instead of silently skipping coverage. +`rocke_ir_types` (`ROCKE_IR_TYPES_TEST`), `rocke_nontemporal_hip` +(`ROCKE_NONTEMPORAL_HIP_TEST`) and `rocke_unsigned_int_ops` +(`ROCKE_UNSIGNED_INT_OPS_CLI`). Both pytest passes receive those paths, so the +storage IR/HIP parity, serialization, C++ builder type-rule parity, nontemporal +HIP-source parity and unsigned integer op C API parity tests run automatically. +The installed `rocke_pytest` ctest sets `ROCKE_UNSIGNED_INT_OPS_CLI` itself. +`--config` selects the native test configuration (default `Release`). A build +or fixture-discovery failure stops the runner instead of silently skipping +coverage. An explicit `ROCKE_STORAGE_TEST` / `ROCKE_IR_TYPES_TEST` / -`ROCKE_NONTEMPORAL_HIP_TEST` overrides -discovery for that binary and must name an existing executable; it does not skip -the configured build. With no configured build or explicit override, the runner +`ROCKE_NONTEMPORAL_HIP_TEST` / `ROCKE_UNSIGNED_INT_OPS_CLI` overrides discovery +for that binary and must name an existing executable; it does not skip the +configured build. With no configured build or explicit override, the runner reports that parity as skipped; direct pytest invocations can use the same overrides. diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.cpp new file mode 100644 index 000000000000..0e114fac02d5 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.cpp @@ -0,0 +1,498 @@ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +/* + * test_unsigned_int_ops.cpp -- the C++ engine's unsigned compare (cmp_ult), its + * no-unsigned-wrap add (add_nuw) and the arith.assume_range range marker. + * + * Run with no argument it checks, through the C API: the LLVM and HIP text of + * both ops, the builders' operand-type refusals, the verifier's acceptance of + * the new attrs and its refusals of bad marker attrs (texts identical to the + * Python verifier), and the constant folder (unsigned compare at the operand + * width; constants read at their width; a marker on a constant folds to the + * constant), and the verifier's i32 / i64 rule for the nuw and ult forms. + * + * Run with --print-llvm it prints the LLVM text of the two-op function built + * through rocke_b_add_nuw / rocke_b_cmp_ult, so the Python test can compare it + * with the same function built by the Python builder. + */ +#include +#include +#include +#include +#include + +#include "rocke/error.hpp" +#include "rocke/ir.h" +#include "rocke/ir_internal.h" +#include "rocke/lower_hip.h" +#include "rocke/lower_llvm.h" +#include "rocke/passes.h" +#include "rocke/strbuf.h" +#include "rocke/verify.h" + +namespace +{ + +int g_failures = 0; + +void fail(const std::string& what, int line) +{ + fprintf(stderr, "FAIL (line %d): %s\n", line, what.c_str()); + ++g_failures; +} + +#define EXPECT(cond, what) \ + do \ + { \ + if(!(cond)) \ + fail((what), __LINE__); \ + } while(0) + +/* The two-op function the Python test builds too: params a, b of `type`, then + * add_nuw(a, b) and cmp_ult(a, b). */ +void build_two_op(rocke_ir_builder_t* b, const rocke_type_t* type) +{ + rocke_value_t* a = rocke_b_param(b, "a", type, nullptr); + rocke_value_t* c = rocke_b_param(b, "b", type, nullptr); + rocke_b_add_nuw(b, a, c); + rocke_b_cmp_ult(b, a, c); + rocke_b_ret(b); +} + +std::string lower_llvm(rocke_ir_builder_t* b) +{ + char* text = nullptr; + std::string out; + if(rocke_lower_kernel_to_llvm( + rocke_ir_builder_kernel(b), ROCKE_LLVM_FLAVOR_LLVM20, "gfx942", &text) + == ROCKE_OK + && text) + out = text; + free(text); + return out; +} + +std::string lower_hip(rocke_ir_builder_t* b) +{ + rocke_strbuf_t out; + rocke_strbuf_init(&out, 0); + rocke_lower_hip_opts_t opts{}; + opts.arch = "gfx942"; + std::string hip; + if(rocke_lower_kernel_to_hip(b, rocke_ir_builder_kernel(b), &opts, &out) == ROCKE_OK) + hip = rocke_strbuf_cstr(&out); + rocke_strbuf_free(&out); + return hip; +} + +/* A range marker op built directly (the C API has no builder entry for it): + * operands [v] or [v, hi_value]; attrs set by `set_attrs`. */ +template +rocke_value_t* + marker(rocke_ir_builder_t* b, rocke_value_t* v, rocke_value_t* hi_value, SetAttrs set_attrs) +{ + rocke_value_t* operands[2] = {v, hi_value}; + rocke_attr_map_t attrs = rocke_i_attrs(b); + set_attrs(&attrs); + return rocke_i_op1( + b, ROCKE_OP_ARITH_ASSUME_RANGE, operands, hi_value ? 2 : 1, v->type, &attrs, "range"); +} + +std::string verify_errors(rocke_ir_builder_t* b) +{ + rocke_diag_t* diags = nullptr; + size_t count = 0; + std::string joined; + if(rocke_verify(rocke_ir_builder_kernel(b), &diags, &count) != ROCKE_OK) + return "verifier failed to run"; + for(size_t i = 0; i < count; ++i) + if(diags[i].severity == ROCKE_DIAG_ERROR) + joined += std::string(diags[i].message) + "\n"; + rocke_diags_free(diags, count); + return joined; +} + +void case_llvm_text() +{ + for(const rocke_type_t* type : {rocke_i32(), rocke_i64()}) + { + rocke_ir_builder_t b; + rocke_ir_builder_init(&b, "unsigned_int_ops"); + build_two_op(&b, type); + const std::string ll = lower_llvm(&b); + const std::string ty = type->name; + EXPECT(ll.find("= add nuw " + ty + " %a, %b") != std::string::npos, "add nuw line"); + EXPECT(ll.find("= icmp ult " + ty + " %a, %b") != std::string::npos, "icmp ult line"); + rocke_ir_builder_free(&b); + } +} + +void case_hip_text() +{ + rocke_ir_builder_t b; + rocke_ir_builder_init(&b, "unsigned_int_ops_hip"); + rocke_value_t* a = rocke_b_param(&b, "a", rocke_i32(), nullptr); + rocke_value_t* c = rocke_b_param(&b, "b", rocke_i32(), nullptr); + rocke_value_t* sum = rocke_b_add_nuw(&b, a, c); + rocke_value_t* below = rocke_b_cmp_ult(&b, a, c); + rocke_value_t* range = marker(&b, a, nullptr, [&](rocke_attr_map_t* m) { + rocke_attr_set_int(&b, m, "multiple_of", 1); + rocke_attr_set_int(&b, m, "residue", 0); + }); + const std::string hip = lower_hip(&b); + const std::string s = std::string(sum->name + 1); + const std::string u = std::string(below->name + 1); + const std::string r = std::string(range->name + 1); + EXPECT(hip.find("int " + s + " = (int32_t)((uint32_t)a + (uint32_t)b);") != std::string::npos, + "HIP add_nuw form"); + EXPECT(hip.find("bool " + u + " = (uint32_t)a < (uint32_t)b;") != std::string::npos, + "HIP cmp_ult form"); + EXPECT(hip.find("int " + r + " = a;") != std::string::npos, "HIP marker copy"); + rocke_ir_builder_free(&b); +} + +void expect_builder_refusal(bool add, const rocke_type_t* ta, const rocke_type_t* tb) +{ + rocke_ir_builder_t b; + rocke_ir_builder_init(&b, "unsigned_int_ops_refusal"); + rocke_value_t* a = rocke_b_param(&b, "a", ta, nullptr); + rocke_value_t* c = rocke_b_param(&b, "b", tb, nullptr); + const char* op_name = add ? "add_nuw" : "cmp_ult"; + const std::string want = std::string(op_name) + " expects two i32 or two i64 operands, got " + + ta->name + " / " + tb->name; + bool refused = false; + try + { + rocke_value_t* r = add ? rocke_b_add_nuw(&b, a, c) : rocke_b_cmp_ult(&b, a, c); + refused = r == nullptr && rocke_ir_builder_status(&b) == ROCKE_ERR_VALUE + && want == rocke_ir_builder_error(&b); + } + catch(const ckc::Error& error) + { + refused = error.code() == ROCKE_ERR_VALUE && want == error.what(); + } + EXPECT(refused, want); + rocke_ir_builder_free(&b); +} + +void case_builder_refusals() +{ + expect_builder_refusal(true, rocke_f32(), rocke_f32()); + expect_builder_refusal(false, rocke_i32(), rocke_i64()); + expect_builder_refusal(true, rocke_i1(), rocke_i1()); +} + +void case_verifier_accepts() +{ + rocke_ir_builder_t b; + rocke_ir_builder_init(&b, "unsigned_int_ops_valid"); + rocke_value_t* a = rocke_b_param(&b, "a", rocke_i32(), nullptr); + rocke_value_t* n = rocke_b_param(&b, "n", rocke_i32(), nullptr); + rocke_b_add_nuw(&b, a, n); + rocke_b_cmp_ult(&b, a, n); + marker(&b, a, n, [&](rocke_attr_map_t* m) { + rocke_attr_set_int(&b, m, "multiple_of", 4); + rocke_attr_set_int(&b, m, "residue", 3); + rocke_attr_set_int(&b, m, "lo", 0); + rocke_attr_set_int(&b, m, "hi_plus", 1); + }); + marker(&b, a, nullptr, [&](rocke_attr_map_t* m) { + rocke_attr_set_int(&b, m, "multiple_of", 1); + rocke_attr_set_int(&b, m, "residue", 0); + rocke_attr_set_int(&b, m, "lo", 0); + rocke_attr_set_int(&b, m, "hi", 64); + }); + rocke_b_ret(&b); + const std::string errors = verify_errors(&b); + EXPECT(errors.empty(), "verifier rejected valid attrs: " + errors); + rocke_ir_builder_free(&b); +} + +/* One bad marker per case; the verifier must report exactly `want`. */ +template +void expect_marker_rejected(bool runtime_end, SetAttrs set_attrs, const char* want) +{ + rocke_ir_builder_t b; + rocke_ir_builder_init(&b, "unsigned_int_ops_bad_marker"); + rocke_value_t* a = rocke_b_param(&b, "a", rocke_i32(), nullptr); + rocke_value_t* n = rocke_b_param(&b, "n", rocke_i32(), nullptr); + marker(&b, a, runtime_end ? n : nullptr, [&](rocke_attr_map_t* m) { set_attrs(&b, m); }); + rocke_b_ret(&b); + const std::string errors = verify_errors(&b); + EXPECT(errors == std::string(want) + "\n", + std::string("want '") + want + "', got '" + errors + "'"); + rocke_ir_builder_free(&b); +} + +void case_verifier_rejects_marker_attrs() +{ + expect_marker_rejected( + false, + [](rocke_ir_builder_t* b, rocke_attr_map_t* m) { + rocke_attr_set_int(b, m, "multiple_of", 0); + rocke_attr_set_int(b, m, "residue", 0); + }, + "arith.assume_range: multiple_of must be >= 1, got 0"); + expect_marker_rejected( + false, + [](rocke_ir_builder_t* b, rocke_attr_map_t* m) { + rocke_attr_set_int(b, m, "multiple_of", 4); + rocke_attr_set_int(b, m, "residue", 4); + }, + "arith.assume_range: residue must be in [0, 4), got 4"); + expect_marker_rejected( + true, + [](rocke_ir_builder_t* b, rocke_attr_map_t* m) { + rocke_attr_set_int(b, m, "multiple_of", 1); + rocke_attr_set_int(b, m, "residue", 0); + rocke_attr_set_int(b, m, "hi", 8); + rocke_attr_set_int(b, m, "hi_plus", 0); + }, + "arith.assume_range: hi and hi_value are both given; state at most one end"); + expect_marker_rejected( + false, + [](rocke_ir_builder_t* b, rocke_attr_map_t* m) { + rocke_attr_set_int(b, m, "multiple_of", 1); + rocke_attr_set_int(b, m, "residue", 0); + rocke_attr_set_int(b, m, "lo", 8); + rocke_attr_set_int(b, m, "hi", 8); + }, + "arith.assume_range: lo (8) must be below hi (8)"); + /* A value that is not an integer is refused before any value rule reads it. */ + expect_marker_rejected( + false, + [](rocke_ir_builder_t* b, rocke_attr_map_t* m) { + rocke_attr_set_str(b, m, "multiple_of", "4"); + rocke_attr_set_int(b, m, "residue", 0); + }, + "arith.assume_range: multiple_of must be an integer"); + expect_marker_rejected( + false, + [](rocke_ir_builder_t* b, rocke_attr_map_t* m) { + rocke_attr_set_int(b, m, "multiple_of", 1); + rocke_attr_set_int(b, m, "residue", 0); + rocke_attr_set_bool(b, m, "lo", true); + }, + "arith.assume_range: lo must be an integer"); + expect_marker_rejected( + false, + [](rocke_ir_builder_t* b, rocke_attr_map_t* m) { + rocke_attr_set_int(b, m, "multiple_of", 1); + rocke_attr_set_int(b, m, "residue", 0); + rocke_attr_set_float(b, m, "hi", 8.5); + }, + "arith.assume_range: hi must be an integer"); + expect_marker_rejected( + true, + [](rocke_ir_builder_t* b, rocke_attr_map_t* m) { + rocke_attr_set_int(b, m, "multiple_of", 1); + rocke_attr_set_int(b, m, "residue", 0); + rocke_attr_set_bool(b, m, "hi_plus", false); + }, + "arith.assume_range: hi_plus must be an integer"); +} + +/* nuw and ult take i32 or i64 operands (add_nuw's and cmp_ult's forms); the + * verifier refuses raw IR of another type with the Python texts. */ +void case_unsigned_forms_need_i32_or_i64() +{ + for(const rocke_type_t* type : {rocke_i1(), rocke_i8()}) + { + rocke_ir_builder_t b; + rocke_ir_builder_init(&b, "unsigned_int_ops_narrow"); + rocke_value_t* a = rocke_b_param(&b, "a", type, nullptr); + rocke_value_t* operands[2] = {a, a}; + rocke_attr_map_t add_attrs = rocke_i_attrs(&b); + rocke_attr_set_str(&b, &add_attrs, "wrap", "nuw"); + rocke_i_op1(&b, ROCKE_OP_ARITH_ADD, operands, 2, type, &add_attrs, "sum"); + rocke_attr_map_t cmp_attrs = rocke_i_attrs(&b); + rocke_attr_set_str(&b, &cmp_attrs, "pred", "ult"); + rocke_i_op1(&b, ROCKE_OP_ARITH_CMP, operands, 2, rocke_i1(), &cmp_attrs, "below"); + rocke_b_ret(&b); + const std::string name = type->name; + const std::string want = "arith.add: wrap 'nuw' needs i32 or i64 operands, got " + name + + "\narith.cmp: pred 'ult' needs i32 or i64 operands, got " + name + + "\n"; + const std::string errors = verify_errors(&b); + EXPECT(errors == want, "want '" + want + "', got '" + errors + "'"); + rocke_ir_builder_free(&b); + } +} + +/* A wrap or pred of another attr kind is refused, not read as the default + * (nsw, lt): by the verifier, and for the wrap by both lowerers. */ +void case_non_string_wrap_and_pred() +{ + const char* wrap_error = "arith.add: wrap must be the string 'nsw' or 'nuw'"; + rocke_ir_builder_t b; + rocke_ir_builder_init(&b, "unsigned_int_ops_non_string"); + rocke_value_t* a = rocke_b_param(&b, "a", rocke_i32(), nullptr); + rocke_value_t* out + = rocke_b_param(&b, "out", rocke_ptr_type(&b, rocke_i32(), "global"), nullptr); + rocke_value_t* operands[2] = {a, a}; + rocke_attr_map_t add_attrs = rocke_i_attrs(&b); + rocke_attr_set_int(&b, &add_attrs, "wrap", 1); + rocke_value_t* sum + = rocke_i_op1(&b, ROCKE_OP_ARITH_ADD, operands, 2, rocke_i32(), &add_attrs, "sum"); + rocke_attr_map_t cmp_attrs = rocke_i_attrs(&b); + rocke_attr_set_int(&b, &cmp_attrs, "pred", 1); + rocke_i_op1(&b, ROCKE_OP_ARITH_CMP, operands, 2, rocke_i1(), &cmp_attrs, "cmp"); + rocke_b_global_store(&b, out, rocke_b_const_i32(&b, 0), sum, 1); + rocke_b_ret(&b); + const std::string errors = verify_errors(&b); + const std::string want = std::string(wrap_error) + "\narith.cmp: pred must be a string\n"; + EXPECT(errors == want, "want '" + want + "', got '" + errors + "'"); + EXPECT(lower_llvm(&b).empty(), "LLVM lowering accepted a non-string wrap"); + EXPECT(lower_hip(&b).empty(), "HIP lowering accepted a non-string wrap"); + rocke_ir_builder_free(&b); +} + +/* The folder's value for the op stored by the kernel's only store. */ +bool folded_store_value(rocke_ir_builder_t* b, int64_t* out) +{ + rocke_kernel_def_t* k = rocke_ir_builder_kernel(b); + rocke_optimize_kernel(b, k, 0); + for(int i = 0; i < k->body->num_ops; ++i) + { + const rocke_op_t* op = k->body->ops[i]; + if(op->opcode != ROCKE_OP_MEMREF_GLOBAL_STORE_TYPED) + continue; + const rocke_op_t* producer = op->operands[2]->op; + return producer && producer->opcode == ROCKE_OP_ARITH_CONSTANT + && rocke_attr_get_int(&producer->attrs, "value", out); + } + return false; +} + +void case_constant_folding() +{ + const int64_t n = 7; + for(bool wide : {false, true}) + for(int64_t x : {int64_t{-1}, int64_t{0}, n - 1, n}) + { + rocke_ir_builder_t b; + rocke_ir_builder_init(&b, "unsigned_int_ops_fold"); + rocke_value_t* out + = rocke_b_param(&b, "out", rocke_ptr_type(&b, rocke_i1(), "global"), nullptr); + rocke_value_t* xv = wide ? rocke_b_const_i64(&b, x) : rocke_b_const_i32(&b, x); + rocke_value_t* nv = wide ? rocke_b_const_i64(&b, n) : rocke_b_const_i32(&b, n); + rocke_b_global_store(&b, out, rocke_b_const_i32(&b, 0), rocke_b_cmp_ult(&b, xv, nv), 1); + int64_t got = -2; + const int64_t want = (x >= 0 && x < n) ? 1 : 0; + EXPECT(folded_store_value(&b, &got) && got == want, + "cmp_ult fold of " + std::to_string(x) + (wide ? " (i64)" : " (i32)")); + rocke_ir_builder_free(&b); + } + + /* A folded add_nuw holds the value the hardware computes: the i32 register + * holds 0x80000000 (-2^31 read as signed), a signed compare sees it below + * 0, and widening it as unsigned keeps 2^31. Same values as the Python + * test. */ + enum class NuwUse + { + sum, + signed_compare, + zext + }; + for(NuwUse use : {NuwUse::sum, NuwUse::signed_compare, NuwUse::zext}) + { + rocke_ir_builder_t nb; + rocke_ir_builder_init(&nb, "unsigned_int_ops_fold_nuw"); + const rocke_type_t* out_type = use == NuwUse::zext ? rocke_i64() : rocke_i32(); + rocke_value_t* out + = rocke_b_param(&nb, "out", rocke_ptr_type(&nb, out_type, "global"), nullptr); + rocke_value_t* sum + = rocke_b_add_nuw(&nb, rocke_b_const_i32(&nb, INT32_MAX), rocke_b_const_i32(&nb, 1)); + rocke_value_t* stored = sum; + int64_t want = INT32_MIN; + const char* label = "sum"; + if(use == NuwUse::signed_compare) + { + stored = rocke_b_select(&nb, + rocke_b_cmp_lt(&nb, sum, rocke_b_const_i32(&nb, 0)), + rocke_b_const_i32(&nb, 1), + rocke_b_const_i32(&nb, 2)); + want = 1; + label = "signed compare"; + } + else if(use == NuwUse::zext) + { + stored = rocke_b_zext(&nb, sum, rocke_i64()); + want = int64_t{1} << 31; + label = "zext"; + } + rocke_b_global_store(&nb, out, rocke_b_const_i32(&nb, 0), stored, 1); + int64_t got = -2; + EXPECT(folded_store_value(&nb, &got) && got == want, + std::string("folded add_nuw ") + label + " gave " + std::to_string(got) + ", want " + + std::to_string(want)); + rocke_ir_builder_free(&nb); + } + + /* A literal written in unsigned form is the same i32 bits as a folded sum: + * 0x80000000 + 0 equals the literal 0x80000000. */ + { + rocke_ir_builder_t lb; + rocke_ir_builder_init(&lb, "unsigned_int_ops_fold_literal"); + rocke_value_t* out + = rocke_b_param(&lb, "out", rocke_ptr_type(&lb, rocke_i32(), "global"), nullptr); + rocke_value_t* sum + = rocke_b_add(&lb, rocke_b_const_i32(&lb, 0x80000000LL), rocke_b_const_i32(&lb, 0)); + rocke_value_t* same = rocke_b_cmp_eq(&lb, sum, rocke_b_const_i32(&lb, 0x80000000LL)); + rocke_value_t* stored + = rocke_b_select(&lb, same, rocke_b_const_i32(&lb, 1), rocke_b_const_i32(&lb, 2)); + rocke_b_global_store(&lb, out, rocke_b_const_i32(&lb, 0), stored, 1); + int64_t got = -2; + EXPECT(folded_store_value(&lb, &got) && got == 1, + "folded 0x80000000 + 0 == 0x80000000 gave " + std::to_string(got) + ", want 1"); + rocke_ir_builder_free(&lb); + } + + /* A marker on a constant folds to the constant, even with a runtime end. */ + rocke_ir_builder_t b; + rocke_ir_builder_init(&b, "unsigned_int_ops_fold_marker"); + rocke_value_t* out + = rocke_b_param(&b, "out", rocke_ptr_type(&b, rocke_i32(), "global"), nullptr); + rocke_value_t* n_param = rocke_b_param(&b, "n", rocke_i32(), nullptr); + rocke_value_t* range = marker(&b, rocke_b_const_i32(&b, 5), n_param, [&](rocke_attr_map_t* m) { + rocke_attr_set_int(&b, m, "multiple_of", 1); + rocke_attr_set_int(&b, m, "residue", 0); + rocke_attr_set_int(&b, m, "hi_plus", 0); + }); + rocke_b_global_store(&b, out, rocke_b_const_i32(&b, 0), range, 1); + int64_t got = -2; + EXPECT(folded_store_value(&b, &got) && got == 5, "marker on a constant folds"); + rocke_ir_builder_free(&b); +} + +} // namespace + +int main(int argc, char** argv) +{ + if(argc > 1 && strcmp(argv[1], "--print-llvm") == 0) + { + rocke_ir_builder_t b; + rocke_ir_builder_init(&b, "unsigned_int_ops"); + build_two_op(&b, rocke_i32()); + const std::string ll = lower_llvm(&b); + rocke_ir_builder_free(&b); + if(ll.empty()) + return 1; + fputs(ll.c_str(), stdout); + return 0; + } + case_llvm_text(); + case_hip_text(); + case_builder_refusals(); + case_verifier_accepts(); + case_verifier_rejects_marker_attrs(); + case_non_string_wrap_and_pred(); + case_unsigned_forms_need_i32_or_i64(); + case_constant_folding(); + if(g_failures) + { + fprintf(stderr, "%d unsigned integer op check(s) failed\n", g_failures); + return 1; + } + return 0; +} diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.py new file mode 100644 index 000000000000..eeae95c9f889 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.py @@ -0,0 +1,592 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""The unsigned compare (``cmp_ult``), the no-unsigned-wrap add (``add_nuw``) +and the private range marker (``IRBuilder._assume_range``). + +Covers the IR text both engines emit, the C API builders against the Python +builder, both verifiers' attr rules, the constant folder, the HIP lowering and, +when a GPU is present, the values the hardware computes against numpy. The C++ +verifier, folder and HIP checks run in the ``rocke_unsigned_int_ops`` ctest with +the same texts as here. +""" + +from __future__ import annotations + +import ctypes +import os +import shutil +import struct +import subprocess +from pathlib import Path + +import numpy as np +import pytest + +from rocke.core.ir import F32, I1, I8, I32, I64, IRBuilder, PtrType +from rocke.core.ir_print import print_ir +from rocke.core.ir_serialize import parse, serialize +from rocke.core.lower_hip import lower_kernel_to_hip +from rocke.core.lower_llvm import _lower_kernel_to_llvm_python +from rocke.core.passes import optimize_kernel +from rocke.core.verify import ERROR, verify + +FLAVORS = ("llvm20", "llvm22", "llvm23") + + +def _two_op_kernel(int_type): + """The function the C API test builds too: ``add_nuw`` and ``cmp_ult``.""" + b = IRBuilder("unsigned_int_ops") + a = b.param("a", int_type) + c = b.param("b", int_type) + b.add_nuw(a, c) + b.cmp_ult(a, c) + b.ret() + return b.kernel + + +def _errors(kernel): + return [d.message for d in verify(kernel) if d.severity == ERROR] + + +def _assert_engines_agree(kernel, arch="gfx942"): + """Both engines emit the same text for ``kernel`` at every flavor.""" + native = pytest.importorskip("rocke_engine") + ir = serialize(kernel) + for flavor in FLAVORS: + python_text = _lower_kernel_to_llvm_python( + kernel, arch=arch, llvm_flavor=flavor + ) + assert native.lower_serialized_ir(ir, arch=arch, flavor=flavor) == python_text + + +# -------------------------------------------------------------------------- +# IR text +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("int_type", [I32, I64]) +def test_llvm_text_has_add_nuw_and_icmp_ult(int_type): + text = _lower_kernel_to_llvm_python(_two_op_kernel(int_type), arch="gfx942") + ty = int_type.name + assert f"%add1 = add nuw {ty} %a, %b" in text + assert f"%ult2 = icmp ult {ty} %a, %b" in text + + +@pytest.mark.parametrize("int_type", [I32, I64]) +def test_both_engines_emit_the_same_text(int_type): + _assert_engines_agree(_two_op_kernel(int_type)) + + +def test_plain_add_still_emits_nsw(): + b = IRBuilder("plain_add") + a = b.param("a", I32) + b.add(a, a) + b.ret() + assert "add nsw i32 %a, %a" in _lower_kernel_to_llvm_python(b.kernel, arch="gfx942") + assert "wrap" not in b.kernel.body.ops[0].attrs + + +def test_c_api_builders_match_the_python_builder(): + """The C API's rocke_b_add_nuw / rocke_b_cmp_ult build the same function. + + ``ROCKE_UNSIGNED_INT_OPS_CLI`` names the built ``rocke_unsigned_int_ops``: + ``tests/run_all.py`` resolves it from a configured build and the installed + ``rocke_pytest`` ctest sets it. A value that names no file is a broken + setup, so it fails instead of skipping. + """ + executable = os.environ.get("ROCKE_UNSIGNED_INT_OPS_CLI") + if not executable: + pytest.skip( + "Set ROCKE_UNSIGNED_INT_OPS_CLI to the built rocke_unsigned_int_ops" + ) + path = Path(executable).resolve() + assert path.is_file(), f"ROCKE_UNSIGNED_INT_OPS_CLI names {path}, not a file" + printed = subprocess.run( + [str(path), "--print-llvm"], + capture_output=True, + text=True, + check=True, + timeout=30, + ) + assert printed.stdout == _lower_kernel_to_llvm_python( + _two_op_kernel(I32), arch="gfx942", llvm_flavor="llvm20" + ) + + +@pytest.mark.parametrize( + "method,lhs,rhs", + [("add_nuw", F32, F32), ("cmp_ult", I32, I64), ("add_nuw", I1, I1)], +) +def test_builders_refuse_operands_other_than_i32_or_i64(method, lhs, rhs): + b = IRBuilder("refused") + a = b.param("a", lhs) + c = b.param("b", rhs) + with pytest.raises( + ValueError, + match=rf"^{method} expects two i32 or two i64 operands, got {lhs.name} / {rhs.name}$", + ): + getattr(b, method)(a, c) + + +# -------------------------------------------------------------------------- +# Verifier +# -------------------------------------------------------------------------- + + +def test_verifier_accepts_the_new_attrs(): + b = IRBuilder("valid") + a = b.param("a", I32) + n = b.param("n", I32) + b.add_nuw(a, n) + b.cmp_ult(a, n) + b._assume_range(a, lo=0, hi_value=n, hi_plus=1, multiple_of=4, residue=3) + b._assume_range(a, lo=0, hi=64) + b.ret() + assert _errors(b.kernel) == [] + + +def test_verifier_rejects_an_unknown_wrap_and_pred(): + b = IRBuilder("bad_enums") + a = b.param("a", I32) + b._op("arith.add", [a, a], [I32], attrs={"wrap": "nw"}) + b._op("arith.cmp", [a, a], [I1], attrs={"pred": "ugt"}) + b.ret() + assert _errors(b.kernel) == [ + "arith.add: unknown wrap 'nw'", + "arith.cmp: unknown pred 'ugt'", + ] + + +def test_verifier_rejects_a_wrap_or_pred_that_is_not_a_string(): + # A raw or parsed op can carry any attr kind; a non-string must be refused, + # not read as the default (nsw, lt). The C++ verifier reports the same texts. + b = IRBuilder("non_string_enums") + a = b.param("a", I32) + b._op("arith.add", [a, a], [I32], attrs={"wrap": 1}) + b._op("arith.cmp", [a, a], [I1], attrs={"pred": 1}) + b.ret() + assert _errors(b.kernel) == [ + "arith.add: wrap must be the string 'nsw' or 'nuw'", + "arith.cmp: pred must be a string", + ] + + +def test_both_lowerings_refuse_a_wrap_that_is_not_a_string(): + b = IRBuilder("non_string_wrap") + a = b.param("a", I32) + out = b.param("out", PtrType(I32, "global")) + b.store(b._op("arith.add", [a, a], [I32], attrs={"wrap": 1}).result, out) + b.ret() + message = "arith.add: wrap must be the string 'nsw' or 'nuw'" + with pytest.raises(ValueError, match=message): + _lower_kernel_to_llvm_python(b.kernel, arch="gfx942") + with pytest.raises(ValueError, match=message): + lower_kernel_to_hip(b.kernel, arch="gfx942") + native = pytest.importorskip("rocke_engine") + with pytest.raises(Exception, match=message): + native.lower_serialized_ir(serialize(b.kernel), arch="gfx942", flavor="llvm20") + + +@pytest.mark.parametrize("int_type", [I1, I8]) +def test_unsigned_forms_need_i32_or_i64_operands(int_type): + # nuw and ult are add_nuw's and cmp_ult's forms, which take i32 or i64. Raw + # IR of another type is refused by the verifier (an i1 nuw sum would print + # a HIP type that does not exist). The C++ verifier reports the same texts. + b = IRBuilder("unsigned_forms_narrow") + a = b.param("a", int_type) + b._op("arith.add", [a, a], [int_type], attrs={"wrap": "nuw"}) + b._op("arith.cmp", [a, a], [I1], attrs={"pred": "ult"}) + b.ret() + name = int_type.name + assert _errors(b.kernel) == [ + f"arith.add: wrap 'nuw' needs i32 or i64 operands, got {name}", + f"arith.cmp: pred 'ult' needs i32 or i64 operands, got {name}", + ] + + +def test_both_lowerings_refuse_a_narrow_nuw_add(): + b = IRBuilder("narrow_nuw") + a = b.param("a", I1) + out = b.param("out", PtrType(I1, "global")) + b.store(b._op("arith.add", [a, a], [I1], attrs={"wrap": "nuw"}).result, out) + b.ret() + message = "arith.add: wrap 'nuw' needs i32 or i64 operands, got i1" + with pytest.raises(ValueError, match=message): + _lower_kernel_to_llvm_python(b.kernel, arch="gfx942") + with pytest.raises(ValueError, match=message): + lower_kernel_to_hip(b.kernel, arch="gfx942") + native = pytest.importorskip("rocke_engine") + with pytest.raises(Exception, match=message): + native.lower_serialized_ir(serialize(b.kernel), arch="gfx942", flavor="llvm20") + + +# The texts the C++ verifier reports for the same markers (the ctest). +_BAD_MARKERS = [ + ( + False, + {"multiple_of": 0, "residue": 0}, + "arith.assume_range: multiple_of must be >= 1, got 0", + ), + ( + False, + {"multiple_of": 4, "residue": 4}, + "arith.assume_range: residue must be in [0, 4), got 4", + ), + ( + True, + {"multiple_of": 1, "residue": 0, "hi": 8, "hi_plus": 0}, + "arith.assume_range: hi and hi_value are both given; state at most one end", + ), + ( + False, + {"multiple_of": 1, "residue": 0, "lo": 8, "hi": 8}, + "arith.assume_range: lo (8) must be below hi (8)", + ), + # A value that is not an integer is refused before any value rule reads it. + ( + False, + {"multiple_of": "4", "residue": 0}, + "arith.assume_range: multiple_of must be an integer", + ), + ( + False, + {"multiple_of": 1, "residue": 0, "lo": True}, + "arith.assume_range: lo must be an integer", + ), + ( + False, + {"multiple_of": 1, "residue": 0, "hi": 8.5}, + "arith.assume_range: hi must be an integer", + ), + ( + True, + {"multiple_of": 1, "residue": 0, "hi_plus": False}, + "arith.assume_range: hi_plus must be an integer", + ), +] + + +@pytest.mark.parametrize("runtime_end,attrs,message", _BAD_MARKERS) +def test_verifier_rejects_bad_marker_attrs(runtime_end, attrs, message): + b = IRBuilder("bad_marker") + a = b.param("a", I32) + n = b.param("n", I32) + operands = [a, n] if runtime_end else [a] + b._op("arith.assume_range", operands, [I32], attrs=dict(attrs)) + b.ret() + assert _errors(b.kernel) == [message] + + +@pytest.mark.parametrize("runtime_end,attrs,message", _BAD_MARKERS) +def test_marker_builder_refuses_the_same_attrs(runtime_end, attrs, message): + b = IRBuilder("bad_marker_builder") + a = b.param("a", I32) + n = b.param("n", I32) + kwargs = {k: v for k, v in attrs.items() if k != "hi_plus"} + if runtime_end: + kwargs["hi_value"] = n + kwargs["hi_plus"] = attrs["hi_plus"] + with pytest.raises(ValueError) as raised: + b._assume_range(a, **kwargs) + assert str(raised.value) == message + + +# -------------------------------------------------------------------------- +# Constant folding +# -------------------------------------------------------------------------- + + +def _folded_store_value(kernel): + optimize_kernel(kernel) + (store,) = [op for op in kernel.body.ops if op.name == "memref.global_store_typed"] + producer = store.operands[2].op + assert producer.name == "arith.constant" + return producer.attrs["value"] + + +def _fold_cmp_ult(x, n, int_type): + b = IRBuilder("fold_ult") + out = b.param("out", PtrType(I1, "global")) + const = b.const_i64 if int_type == I64 else b.const_i32 + b.store(b.cmp_ult(const(x), const(n)), out) + b.ret() + return _folded_store_value(b.kernel) + + +@pytest.mark.parametrize("int_type", [I32, I64]) +def test_folded_cmp_ult_reads_operands_as_unsigned(int_type): + n = 7 + unsigned = np.uint64 if int_type == I64 else np.uint32 + for x in (-1, 0, n - 1, n): + expected = int(np.array(x).astype(unsigned) < unsigned(n)) + assert _fold_cmp_ult(x, n, int_type) == expected, x + + +def _store_one(name, out_type, compute): + """A kernel that stores ``compute(b)`` to ``out[0]`` (an ``out_type`` array).""" + b = IRBuilder(name) + out = b.param("out", PtrType(out_type, "global")) + b.store(compute(b), out) + b.ret() + return b.kernel + + +def _nuw_sum_past_signed_max(b): + """``add_nuw(2**31 - 1, 1)``: defined for nuw, 0x80000000 in the register.""" + return b.add_nuw(b.const_i32(2**31 - 1), b.const_i32(1)) + + +_NUW_FOLD_CASES = [ + # The i32 register holds 0x80000000, which reads as -2**31 when signed. + ("nuw_sum", I32, _nuw_sum_past_signed_max, -(2**31)), + # A signed compare sees the wrapped value: 0x80000000 is below 0. + ( + "nuw_sum_signed_compare", + I32, + lambda b: b.select( + b.cmp_lt(_nuw_sum_past_signed_max(b), b.const_i32(0)), + b.const_i32(1), + b.const_i32(2), + ), + 1, + ), + # Widening as unsigned keeps 2**31. + ( + "nuw_sum_zext", + I64, + lambda b: b.zext(_nuw_sum_past_signed_max(b), I64), + 2**31, + ), +] + + +@pytest.mark.parametrize( + "name,out_type,compute,expected", + _NUW_FOLD_CASES, + ids=[case[0] for case in _NUW_FOLD_CASES], +) +def test_folded_add_nuw_wraps_like_the_hardware(name, out_type, compute, expected): + """A folded ``add_nuw`` holds the value the hardware computes. + + The folder wraps every integer result to its width, so a sum past the + signed maximum reaches later signed folds as the negative register value, + not as an unbounded positive int. The unfolded kernel keeps ``add nuw``; + both engines print the folded constant the same way. + """ + unfolded = _lower_kernel_to_llvm_python( + _store_one(name, out_type, compute), arch="gfx942" + ) + assert "add nuw i32 2147483647, 1" in unfolded + + kernel = _store_one(name, out_type, compute) + assert _folded_store_value(kernel) == expected + assert f"store {out_type.name} {expected}," in _lower_kernel_to_llvm_python( + kernel, arch="gfx942" + ) + _assert_engines_agree(kernel) + + +def test_constants_are_read_at_their_width(): + # A literal written in unsigned form is the same i32 bits as a folded sum, + # so 0x80000000 + 0 equals the literal 0x80000000 (the ctest checks the C++ + # folder with the same kernel). + b = IRBuilder("fold_literal") + out = b.param("out", PtrType(I32, "global")) + total = b.add(b.const_i32(0x80000000), b.const_i32(0)) + same = b.cmp_eq(total, b.const_i32(0x80000000)) + b.store(b.select(same, b.const_i32(1), b.const_i32(2)), out) + b.ret() + assert _folded_store_value(b.kernel) == 1 + + +def test_marker_on_a_constant_folds_to_the_constant(): + b = IRBuilder("fold_marker") + out = b.param("out", PtrType(I32, "global")) + n = b.param("n", I32) + b.store(b._assume_range(b.const_i32(5), hi_value=n), out) + b.ret() + assert _folded_store_value(b.kernel) == 5 + + +# -------------------------------------------------------------------------- +# The range marker +# -------------------------------------------------------------------------- + + +def test_marker_is_private(): + assert not hasattr(IRBuilder, "assume_range") + + +def test_marker_lowers_to_the_text_without_it(): + b = IRBuilder("marked") + x = b.param("x", I32) + n = b.param("n", I32) + out = b.param("out", PtrType(I32, "global")) + marked = b._assume_range(x, lo=0, hi_value=n, hi_plus=1, multiple_of=2, residue=0) + b.store(b.add(marked, b.const_i32(1)), out) + b.ret() + kernel = b.kernel + with_marker = _lower_kernel_to_llvm_python(kernel, arch="gfx942") + _assert_engines_agree(kernel) + + # The same kernel with the marker removed and its use reading x directly. + (marker_op,) = [op for op in kernel.body.ops if op.name == "arith.assume_range"] + kernel.body.ops.remove(marker_op) + for op in kernel.body.ops: + op.operands = [x if v is marked else v for v in op.operands] + assert _lower_kernel_to_llvm_python(kernel, arch="gfx942") == with_marker + _assert_engines_agree(kernel) + + +def test_marker_attrs_round_trip(): + b = IRBuilder("marker_attrs") + x = b.param("x", I32) + n = b.param("n", I32) + b._assume_range(x, lo=0, hi_value=n, hi_plus=1, multiple_of=4, residue=2) + b._assume_range(x, hi=16) + b.ret() + printed = print_ir(b.kernel) + assert "{hi_plus = 1, lo = 0, multiple_of = 4, residue = 2}" in printed + assert "{hi = 16, multiple_of = 1, residue = 0}" in printed + ir = serialize(b.kernel) + assert serialize(parse(ir)) == ir + assert print_ir(parse(ir)) == printed + + +# -------------------------------------------------------------------------- +# HIP lowering +# -------------------------------------------------------------------------- + + +def _hip_kernel(): + b = IRBuilder("unsigned_int_ops_hip") + a = b.param("a", I32) + c = b.param("b", I32) + out = b.param("out", PtrType(I32, "global")) + below = b.param("below", PtrType(I1, "global")) + b.store(b.add_nuw(a, c), out) + b.store(b.cmp_ult(a, c), below) + b.store(b._assume_range(a, lo=0), out, b.const_i32(1)) + b.ret() + return b.kernel + + +def test_hip_lowering_spells_unsigned_forms(): + hip = lower_kernel_to_hip(_hip_kernel(), arch="gfx942") + assert "int add1 = (int32_t)((uint32_t)a + (uint32_t)b);" in hip + assert "bool ult3 = (uint32_t)a < (uint32_t)b;" in hip + assert "int range5 = a;" in hip + + +def test_hip_add_nuw_compiles_without_nsw(tmp_path): + hipcc = shutil.which("hipcc") + if hipcc is None: + pytest.skip("hipcc not found") + source = tmp_path / "unsigned_int_ops.hip" + source.write_text(lower_kernel_to_hip(_hip_kernel(), arch="gfx942")) + device_ir = tmp_path / "unsigned_int_ops.ll" + subprocess.run( + [ + hipcc, + "--cuda-device-only", + "--offload-arch=gfx942", + "-O1", + "-S", + "-emit-llvm", + str(source), + "-o", + str(device_ir), + ], + check=True, + capture_output=True, + timeout=300, + ) + text = device_ir.read_text() + adds = [line for line in text.splitlines() if " = add " in line] + assert adds, text + assert not any("nsw" in line for line in adds), adds + + +# -------------------------------------------------------------------------- +# GPU values +# -------------------------------------------------------------------------- + + +def _gpu_arch(): + try: + from rocke.runtime.hip_module import get_device_arch + except ImportError: + return None + try: + return get_device_arch() + except Exception: # noqa: BLE001 - no HIP runtime means no GPU test + return None + + +def _gpu_kernel(): + """Lane i writes ult(cx[i], cy[i]) and add_nuw(sx[i], sy[i]).""" + b = IRBuilder("unsigned_int_ops_gpu") + ptrs = [b.param(name, PtrType(I32, "global")) for name in ("cx", "cy", "sx", "sy")] + below_out = b.param("below", PtrType(I32, "global")) + sum_out = b.param("sum", PtrType(I32, "global")) + lane = b.thread_id_x() + cx, cy, sx, sy = (b.global_load(p, lane, I32) for p in ptrs) + b.store(b.zext(b.cmp_ult(cx, cy), I32), below_out, lane) + b.store(b.add_nuw(sx, sy), sum_out, lane) + b.ret() + return b.kernel + + +def test_gpu_values_match_numpy(): + arch = _gpu_arch() + if not arch: + pytest.skip("no GPU") + from rocke.runtime.comgr import build_hsaco_from_llvm_ir + from rocke.runtime.hip_module import Runtime + + n = 1000 + cx = np.array([-1, 0, n - 1, n, 2**31 - 1, 0], dtype=np.int32) + cy = np.full_like(cx, n) + # Each sum passes the signed i32 maximum but stays below 2^32. + sx = np.array([2**30, 2**31 - 1, 1_610_612_736, 3, 2**31 - 1, 0], np.uint32) + sy = np.array([2**30, 1, 1_073_741_824, 4, 2**31 - 1, 0], np.uint32) + sx, sy = sx.view(np.int32), sy.view(np.int32) + expected_below = (cx.view(np.uint32) < cy.view(np.uint32)).astype(np.int32) + expected_sum = (sx.view(np.uint32) + sy.view(np.uint32)).view(np.int32) + + kernel = _gpu_kernel() + blob, _ = build_hsaco_from_llvm_ir( + _lower_kernel_to_llvm_python(kernel, arch=arch), + isa=f"amdgcn-amd-amdhsa--{arch}", + ) + runtime = Runtime() + module = runtime.load_module(blob) + arrays = [cx, cy, sx, sy, np.full_like(cx, -7), np.full_like(cx, -7)] + arrays = [np.ascontiguousarray(a).copy() for a in arrays] + ptrs = [] + try: + for array in arrays: + ptr = runtime.alloc(array.nbytes) + ptrs.append(ptr) + buf = (ctypes.c_ubyte * array.nbytes).from_buffer(array) + runtime.memcpy_h2d(ptr, buf, array.nbytes) + runtime.launch_blocking( + module.get_function(kernel.name), + (1, 1, 1), + (len(cx), 1, 1), + struct.pack(f"{len(ptrs)}Q", *ptrs), + ) + for array, ptr in zip(arrays[4:], ptrs[4:]): + buf = (ctypes.c_ubyte * array.nbytes).from_buffer(array) + runtime.memcpy_d2h(buf, ptr, array.nbytes) + finally: + for ptr in ptrs: + runtime.free(ptr) + module.unload() + below, total = arrays[4], arrays[5] + np.testing.assert_array_equal(below, expected_below) + np.testing.assert_array_equal(total, expected_sum) + # The folder agrees with the hardware on the compare. + for x, y, got in zip(cx[:4], cy[:4], below[:4]): + assert _fold_cmp_ult(int(x), int(y), I32) == got diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py index bd582c4fbea9..2c67a60f3a83 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/run_all.py @@ -139,6 +139,11 @@ def ctest_ready(build_root: Path, config: str) -> bool: "rocke_nontemporal_hip", "native nontemporal HIP parity", ), + ( + "ROCKE_UNSIGNED_INT_OPS_CLI", + "rocke_unsigned_int_ops", + "native unsigned integer op C API parity", + ), ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py index 607cce7bd2b4..6e5385aa3d5a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/tools/test_run_all.py @@ -47,6 +47,8 @@ def test_runner_passes_registered_fixture_to_both_backends( ir_types.touch() nontemporal = tmp_path / "Debug" / "rocke_nontemporal_hip" nontemporal.touch() + unsigned_int_ops = tmp_path / "Debug" / "rocke_unsigned_int_ops" + unsigned_int_ops.touch() selected = tmp_path / "override" if override else executable if override: selected.touch() @@ -62,6 +64,10 @@ def run(command, **kwargs): {"name": "rocke_storage", "command": [str(executable)]}, {"name": "rocke_ir_types", "command": [str(ir_types)]}, {"name": "rocke_nontemporal_hip", "command": [str(nontemporal)]}, + { + "name": "rocke_unsigned_int_ops", + "command": [str(unsigned_int_ops)], + }, ] } return subprocess.CompletedProcess(command, 0, stdout=json.dumps(listing)) @@ -93,6 +99,10 @@ def run(command, **kwargs): env["ROCKE_NONTEMPORAL_HIP_TEST"] == str(nontemporal) for _, env in children ) assert all(env["ROCKE_TEST_ENGINE_ARCHIVE"] == str(archive) for _, env in children) + assert all( + env["ROCKE_UNSIGNED_INT_OPS_CLI"] == str(unsigned_int_ops) + for _, env in children + ) assert "ROCKE_BACKEND" not in children[0][1] assert children[-1][1]["ROCKE_BACKEND"] == "both" assert calls[0][0] == [ @@ -300,7 +310,8 @@ def test_fresh_build_prepares_entire_ctest_suite( "enable_testing()\n" 'add_custom_target(rocke_core COMMAND "${CMAKE_COMMAND}" -E touch core.ready)\n' 'get_filename_component(tool_suffix "${CMAKE_COMMAND}" LAST_EXT)\n' - "foreach(name rocke_storage rocke_ir_types rocke_nontemporal_hip rocke_dtypes)\n" + "foreach(name rocke_storage rocke_ir_types rocke_nontemporal_hip\n" + " rocke_unsigned_int_ops rocke_dtypes)\n" ' set(output "${CMAKE_CURRENT_BINARY_DIR}/fixtures/${name}${tool_suffix}")\n' ' add_custom_command(OUTPUT "${output}"\n' ' COMMAND "${CMAKE_COMMAND}" -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/fixtures"\n' @@ -327,6 +338,7 @@ def test_fresh_build_prepares_entire_ctest_suite( assert Path(env["ROCKE_STORAGE_TEST"]).is_file() assert Path(env["ROCKE_IR_TYPES_TEST"]).is_file() assert Path(env["ROCKE_NONTEMPORAL_HIP_TEST"]).is_file() + assert Path(env["ROCKE_UNSIGNED_INT_OPS_CLI"]).is_file() if override: assert env["ROCKE_STORAGE_TEST"] == str(executable) assert runner.ctest_ready(build, "Release") From c5bc9f110b0a2eca9814d092692d7944d5511275 Mon Sep 17 00:00:00 2001 From: Chris Millette Date: Thu, 15 Oct 2026 20:55:12 -0400 Subject: [PATCH 2/4] fix(rocke): emit magic division's sum without signed-overflow poison Magic division computes (umul_hi(n, multiplier) + n) >> shift. For a dividend below 2^31 the sum can pass the signed i32 maximum while staying below 2^32 (from 1,610,612,736 for divisor 3), and it was emitted as add nsw, so LLVM was free to treat the quotient as poison inside the documented range. The sum is now a no-unsigned-wrap add (add nuw) everywhere it is built: do_magic_division and do_magic_division_dynamic in helpers/transforms.py, their C++ mirrors in cpp/helpers/transforms.cpp, and the two hand-written C++ copies in the conv forward load phase and the conv dgrad kernel. The C++ helper header comment is updated to match. The LLVM IR of the conv, conv dgrad, conv wgrad, direct conv, fused conv-pool and MoE sorting families changes accordingly; the representative IR golden is re-blessed for all three LLVM flavors. Tests: tests/core/test_magic_division_sum.py checks that the lowered sum is add nuw with no add nsw, that both engines emit the same text, and, on a GPU, the static and runtime-multiplier quotients of dividends 2^30 - 1, 2^30, 1,610,612,736 and 2^31 - 1 by divisors 3, 9, 1025 and 64 against numpy. Co-Authored-By: Claude --- .../rocke/platform/cpp/helpers/transforms.cpp | 4 +- .../rocke/helper_rocke.helpers.transforms.h | 2 +- .../conv_implicit_gemm_conv_load_phase.cpp | 4 +- .../common/conv_implicit_gemm_dgrad.cpp | 2 +- .../python/rocke/helpers/transforms.py | 12 +- .../tests/core/test_magic_division_sum.py | 180 ++++++++ .../rocke_representative_ir_sha256.json | 402 +++++++++--------- 7 files changed, 396 insertions(+), 210 deletions(-) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_magic_division_sum.py diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/transforms.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/transforms.cpp index bd824943b91f..fcc1409c6ee3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/transforms.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/transforms.cpp @@ -126,7 +126,7 @@ rocke_value_t* rocke_do_magic_division(rocke_ir_builder_t* b, } tmp = rocke_b_umul_hi_i32(b, dividend, rocke_b_const_i32(b, mult_i32)); - summed = rocke_b_add(b, tmp, dividend); + summed = rocke_b_add_nuw(b, tmp, dividend); if(shift == 0) { return summed; @@ -804,7 +804,7 @@ static rocke_value_t* rocke_i_do_magic_division_dynamic(rocke_ir_builder_t* b, return NULL; } tmp = rocke_b_umul_hi_i32(b, dividend, mult_v); - summed = rocke_b_add(b, tmp, dividend); + summed = rocke_b_add_nuw(b, tmp, dividend); /* lshr unconditionally (shift==0 is a no-op) */ return rocke_b_lshr(b, summed, shift_v); } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.transforms.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.transforms.h index c33209ed88e1..e717c8e3cae7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.transforms.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.transforms.h @@ -75,7 +75,7 @@ bool rocke_calculate_magic_numbers(rocke_ir_builder_t* b, * Emits `dividend // divisor` via the mul-hi magic sequence: * mult_i32 = (multiplier >= 2**31) ? multiplier - 2**32 : multiplier * tmp = b.umul_hi_i32(dividend, b.const_i32(mult_i32)) - * summed = b.add(tmp, dividend) + * summed = b.add_nuw(tmp, dividend) * return shift == 0 ? summed : b.lshr(summed, b.const_i32(shift)) * * `dividend` is an i32 SSA value (treated unsigned); the result is the i32 diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_load_phase.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_load_phase.cpp index a5de617f71e3..49d1c52f97fc 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_load_phase.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_load_phase.cpp @@ -264,9 +264,9 @@ void rocke_conv_magic_divmod(rocke_ir_builder_t* b, rocke_value_t** quot, rocke_value_t** rem) { - /* do_magic_division_dynamic: umul_hi, add, lshr */ + /* do_magic_division_dynamic: umul_hi, no-unsigned-wrap add, lshr */ rocke_value_t* tmp = rocke_b_umul_hi_i32(b, val, mult); - rocke_value_t* summed = rocke_b_add(b, tmp, val); + rocke_value_t* summed = rocke_b_add_nuw(b, tmp, val); rocke_value_t* q = rocke_b_lshr(b, summed, shift); rocke_value_t* prod = u24 ? rocke_conv_mul_u24(b, q, dim) : rocke_b_mul(b, q, dim); *quot = q; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp index e34d00f36af3..d1e68f0a61cd 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_dgrad.cpp @@ -1356,7 +1356,7 @@ static void _magic_divmod(rocke_ir_builder_t* b, rocke_value_t** r_out) { rocke_value_t* tmp = rocke_b_umul_hi_i32(b, n, d->mult); - rocke_value_t* summed = rocke_b_add(b, tmp, n); + rocke_value_t* summed = rocke_b_add_nuw(b, tmp, n); rocke_value_t* q = rocke_b_lshr(b, summed, d->shift); rocke_value_t* qd = rocke_b_mul(b, q, d->value); *q_out = q; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/transforms.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/transforms.py index ad5908f6aa2a..355e52962587 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/transforms.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/transforms.py @@ -158,13 +158,18 @@ def do_magic_division( ``dividend`` is an i32 SSA value interpreted as unsigned; the result is the i32 quotient. ``multiplier`` / ``shift`` come from :func:`calculate_magic_numbers`. + + The sum ``tmp + dividend`` is emitted as a no-unsigned-wrap add: for a + dividend near 2**31 it passes the signed maximum (from 1,610,612,736 for + divisor 3) while staying below 2**32, so a no-signed-wrap add would make + the quotient poison inside the documented range. """ # ``multiplier`` is a uint32 that may exceed the i32 signed range; bake # it into the i32 constant as its two's-complement bit pattern so the # unsigned ``umul_hi`` sees the right bits (matches the C++ uint32 path). mult_i32 = multiplier - (1 << 32) if multiplier >= (1 << 31) else multiplier tmp = b.umul_hi_i32(dividend, b.const_i32(mult_i32)) - summed = b.add(tmp, dividend) + summed = b.add_nuw(tmp, dividend) if shift == 0: return summed return b.lshr(summed, b.const_i32(shift)) @@ -176,10 +181,11 @@ def do_magic_division_dynamic( """Like :func:`do_magic_division` but ``multiplier`` and ``shift`` are runtime SSA ``Value`` objects (e.g. kernel arguments) instead of Python ``int``. Emits the same ``umul_hi + add + lshr`` sequence; ``lshr`` is - always emitted (shift == 0 produces a no-op right-shift). + always emitted (shift == 0 produces a no-op right-shift). The sum is a + no-unsigned-wrap add, as in :func:`do_magic_division`. """ tmp = b.umul_hi_i32(dividend, multiplier) - summed = b.add(tmp, dividend) + summed = b.add_nuw(tmp, dividend) return b.lshr(summed, shift) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_magic_division_sum.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_magic_division_sum.py new file mode 100644 index 000000000000..58cf96fa2d2b --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_magic_division_sum.py @@ -0,0 +1,180 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""Magic division stays exact for dividends near 2**31. + +``do_magic_division`` and ``do_magic_division_dynamic`` compute +``(umul_hi(n, multiplier) + n) >> shift``. For a dividend below 2**31 the sum +can pass the signed i32 maximum (from 1,610,612,736 for divisor 3) while +staying below 2**32, so it must be a no-unsigned-wrap add: a no-signed-wrap add +would make the quotient poison inside the range the helpers document. + +These tests check the emitted LLVM text (the sum is ``add nuw``, never +``add nsw``) and, when a GPU is present, the quotients the hardware computes +for dividends on both sides of that point against numpy. +""" + +from __future__ import annotations + +import ctypes +import re +import struct + +import numpy as np +import pytest + +from rocke.core.ir import I32, IRBuilder, PtrType +from rocke.core.ir_serialize import serialize +from rocke.core.lower_llvm import _lower_kernel_to_llvm_python +from rocke.helpers.transforms import ( + calculate_magic_numbers, + do_magic_division, + do_magic_division_dynamic, +) + +FLAVORS = ("llvm20", "llvm22", "llvm23") +DIVIDENDS = (2**30 - 1, 2**30, 1_610_612_736, 2**31 - 1) +DIVISORS = (3, 9, 1025, 64) + + +def _static_kernel(divisor): + """Lane i writes ``n[i] // divisor`` with the divisor's constants baked in.""" + b = IRBuilder(f"magic_div_static_{divisor}") + dividends = b.param("n", PtrType(I32, "global")) + out = b.param("q", PtrType(I32, "global")) + lane = b.thread_id_x() + multiplier, shift = calculate_magic_numbers(divisor) + quotient = do_magic_division( + b, b.global_load(dividends, lane, I32), multiplier, shift + ) + b.store(quotient, out, lane) + b.ret() + return b.kernel + + +def _dynamic_kernel(): + """Lane i writes ``n[i] // d[i]`` from the runtime magic pair of ``d[i]``.""" + b = IRBuilder("magic_div_dynamic") + ptrs = [b.param(name, PtrType(I32, "global")) for name in ("n", "mult", "shift")] + out = b.param("q", PtrType(I32, "global")) + lane = b.thread_id_x() + dividend, multiplier, shift = (b.global_load(p, lane, I32) for p in ptrs) + b.store(do_magic_division_dynamic(b, dividend, multiplier, shift), out, lane) + b.ret() + return b.kernel + + +def _sum_lines(text): + """The ``add`` lines of the lowered magic-division sequence.""" + return [line.strip() for line in text.splitlines() if re.search(r"= add ", line)] + + +# -------------------------------------------------------------------------- +# IR text +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "kernel", [_static_kernel(3), _dynamic_kernel()], ids=["static", "dynamic"] +) +def test_the_sum_is_a_no_unsigned_wrap_add(kernel): + for flavor in FLAVORS: + text = _lower_kernel_to_llvm_python(kernel, arch="gfx942", llvm_flavor=flavor) + lines = _sum_lines(text) + assert lines, text + assert all( + line.split("=", 1)[1].startswith(" add nuw ") for line in lines + ), lines + assert "add nsw" not in text + + +@pytest.mark.parametrize( + "kernel", [_static_kernel(3), _dynamic_kernel()], ids=["static", "dynamic"] +) +def test_both_engines_emit_the_same_text(kernel): + native = pytest.importorskip("rocke_engine") + ir = serialize(kernel) + for flavor in FLAVORS: + python_text = _lower_kernel_to_llvm_python( + kernel, arch="gfx942", llvm_flavor=flavor + ) + assert ( + native.lower_serialized_ir(ir, arch="gfx942", flavor=flavor) == python_text + ) + + +# -------------------------------------------------------------------------- +# GPU values +# -------------------------------------------------------------------------- + + +def _gpu_arch(): + try: + from rocke.runtime.hip_module import get_device_arch + except ImportError: + return None + try: + return get_device_arch() + except Exception: # noqa: BLE001 - no HIP runtime means no GPU test + return None + + +def _run(arch, kernel, inputs): + """Launch ``kernel`` on one wave over ``inputs``; return its i32 output.""" + from rocke.runtime.comgr import build_hsaco_from_llvm_ir + from rocke.runtime.hip_module import Runtime + + blob, _ = build_hsaco_from_llvm_ir( + _lower_kernel_to_llvm_python(kernel, arch=arch), + isa=f"amdgcn-amd-amdhsa--{arch}", + ) + runtime = Runtime() + module = runtime.load_module(blob) + out = np.full_like(inputs[0], -7) + arrays = [np.ascontiguousarray(a).copy() for a in inputs] + [out] + ptrs = [] + try: + for array in arrays: + ptr = runtime.alloc(array.nbytes) + ptrs.append(ptr) + buf = (ctypes.c_ubyte * array.nbytes).from_buffer(array) + runtime.memcpy_h2d(ptr, buf, array.nbytes) + runtime.launch_blocking( + module.get_function(kernel.name), + (1, 1, 1), + (len(out), 1, 1), + struct.pack(f"{len(ptrs)}Q", *ptrs), + ) + buf = (ctypes.c_ubyte * out.nbytes).from_buffer(out) + runtime.memcpy_d2h(buf, ptrs[-1], out.nbytes) + finally: + for ptr in ptrs: + runtime.free(ptr) + module.unload() + return out + + +@pytest.fixture(scope="module") +def gpu_arch(): + arch = _gpu_arch() + if not arch: + pytest.skip("no GPU") + return arch + + +@pytest.mark.parametrize("divisor", DIVISORS) +def test_static_quotients_match_numpy(gpu_arch, divisor): + dividends = np.array(DIVIDENDS, dtype=np.int32) + got = _run(gpu_arch, _static_kernel(divisor), [dividends]) + np.testing.assert_array_equal(got, dividends // divisor) + + +def test_dynamic_quotients_match_numpy(gpu_arch): + cases = [(n, d) for d in DIVISORS for n in DIVIDENDS] + dividends = np.array([n for n, _ in cases], dtype=np.int32) + magic = [calculate_magic_numbers(d) for _, d in cases] + multipliers = np.array([m for m, _ in magic], dtype=np.uint32).view(np.int32) + shifts = np.array([s for _, s in magic], dtype=np.int32) + divisors = np.array([d for _, d in cases], dtype=np.int32) + got = _run(gpu_arch, _dynamic_kernel(), [dividends, multipliers, shifts]) + np.testing.assert_array_equal(got, dividends // divisors) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json index 232dc53fcb1e..28ce62a25724 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json @@ -231,7 +231,7 @@ "bytes": 46260, "family": "attention", "kernel_name": "rocke_unified_attention_2d_scalar_q64_h4_kv2_d128_b16_fp16_rbu", - "sha256": "008548f5f6d31c428277b729f14b164786652fa151f3fe80f5c70f1b033d945d", + "sha256": "534453293b4d4318daf0a9fad16ddf298a7ffaf8dbbfeb9cc901f269d46f4765", "status": "ok" }, "attention/gfx950/scalar2d_fp16_d128_sw16_rb8": { @@ -239,7 +239,7 @@ "bytes": 46594, "family": "attention", "kernel_name": "rocke_unified_attention_2d_scalar_q64_h4_kv2_d128_b16_fp16_sw_rb8", - "sha256": "699f9b4dbc08b4c4ad928697c274f2547eaedb58836b85c2c2b5600d982d84c9", + "sha256": "d81575919d468f5c88b915fac219132c9808b5fa2ccdab6031f7d868c0489c03", "status": "ok" }, "attention/gfx950/scalar2d_fp16_d128_top_left": { @@ -247,7 +247,7 @@ "bytes": 46364, "family": "attention", "kernel_name": "rocke_unified_attention_2d_scalar_q64_h4_kv2_d128_b16_fp16_tl", - "sha256": "e2f2c9f8cb62058993f5eff623dcfbed4ac8285a7a0818ed4822be5dc8b24f03", + "sha256": "e8104b7de52d1e84b12f28a0e836dcee13477021397d3b6dbea935c1a296a335", "status": "ok" }, "attention/gfx950/scalar3d_fp16_d128_sw16_rb8": { @@ -255,7 +255,7 @@ "bytes": 49095, "family": "attention", "kernel_name": "rocke_unified_attention_3d_scalar_q64_h4_kv2_d128_b16_seg8_fp16_rb8_sw", - "sha256": "fa87ddc448136a1484251d6a8c81467c080da5e643cf9579ae3d2e30db051131", + "sha256": "5d8bf24697392fca54352be7717cfeac566b5d6dd1ed8df23f7b652cdf8a7357", "status": "ok" }, "attention/gfx950/scalar3d_fp16_d128_top_left": { @@ -263,7 +263,7 @@ "bytes": 48855, "family": "attention", "kernel_name": "rocke_unified_attention_3d_scalar_q64_h4_kv2_d128_b16_seg8_fp16_tl", - "sha256": "cea72ae4dd4246f4429a2246174c313a7891e75ccf6380618c7f313204566b38", + "sha256": "49a72d16ffa227df568072c50f77647261cb8b8023669fcec8321c76fd9a6429", "status": "ok" }, "attention_d128_swa/gfx942/4warp_gqa_fold": { @@ -647,7 +647,7 @@ "bytes": 30310, "family": "conv", "kernel_name": "irhash_conv_1151_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_cshuffle", - "sha256": "0cde673df1aec68b6fe5a2b5e3454c39dd8d338ce9b8ed312cf194555c198ff4", + "sha256": "7dfcb1802671848602f38b6285da34cd6426fc73752948198762eb3aee5b01bc", "status": "ok" }, "conv/gfx1151/n2h16c32k32r1": { @@ -663,7 +663,7 @@ "bytes": 23945, "family": "conv", "kernel_name": "irhash_conv_1201_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_cshuffle", - "sha256": "7dc71d25b9c9650680175288e87d9c27724a2be89ed3cc4b6133d145a88503d0", + "sha256": "960dde16f771e8e06fe7eebf29f1a9e479847318e4af5c0837c7e9aba97d2baa", "status": "ok" }, "conv/gfx90a/n1h8c16k32r3": { @@ -671,7 +671,7 @@ "bytes": 23613, "family": "conv", "kernel_name": "irhash_conv_90a_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_cshuffle", - "sha256": "9d65cedf614e4d35f2d2a3d9d03f197c08c31b38c4a9c0ca3a5a70ff8ea1355b", + "sha256": "50bbe0562119634c43fa5299df32462c816e53465e48def611d33506e60f2124", "status": "ok" }, "conv/gfx942/n1h8c16k32r3": { @@ -679,7 +679,7 @@ "bytes": 23613, "family": "conv", "kernel_name": "irhash_conv_942_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_cshuffle", - "sha256": "a412610f2cf881ed508fd8201087034dc8c51ca5bc32046e28b8ede2c0e18058", + "sha256": "be06a34fbf81ddb68398d99bda3a35a90d78b18d57c2f8f65d3a15a074e83195", "status": "ok" }, "conv/gfx942/n2h14c64k64r3g64/dw_gm8_cshuffle": { @@ -687,7 +687,7 @@ "bytes": 24886, "family": "conv", "kernel_name": "irhash_conv_942_dw_gm8_N2H14W14C64_K64Y3X3G64_t64x32x16_w2x2_a16x16x16_mem_cshuffle_gm8", - "sha256": "dcca513a5fd36b18b1c90ede2e93c6c7fc3d79aedcb1ddc3c55df6d950603922", + "sha256": "5820c7a5de3f5881b6a1fbb34b7cfec4aeb080df31f9869135a8dba85cd64b53", "status": "ok" }, "conv/gfx950/n1h8c16k32r3": { @@ -695,7 +695,7 @@ "bytes": 24951, "family": "conv", "kernel_name": "irhash_conv_950_a_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_compv4_cshuffle", - "sha256": "026229bb820ea2d9a7f27035131a895130336306cf5a4b2bf79c925179202e6a", + "sha256": "6ffd1aa2f1eb471ceeb46f07c6aff5984f142a0b14225cef8b27eeb7da7b7e8c", "status": "ok" }, "conv/gfx950/n1h8c16k32r3/basic_cshuffle": { @@ -703,7 +703,7 @@ "bytes": 24631, "family": "conv", "kernel_name": "irhash_conv_950_basic_b_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_basic_cshuffle", - "sha256": "cc255f6523e5ddd61eae734de463402f9e91c903a587bacb2ddf6bed94c73722", + "sha256": "8dbc7f5d29b1a2b14cbc9c2963bb59ebe31b8a61bd777f3d497e72567216cddf", "status": "ok" }, "conv/gfx950/n1h8c16k32r3/basic_default": { @@ -711,7 +711,7 @@ "bytes": 39845, "family": "conv", "kernel_name": "irhash_conv_950_basic_a_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_basic_default", - "sha256": "741e83efaecab3224edbba605a9e6d06668565f51eb4b0b2137633d24923e573", + "sha256": "19b0b593aaa1589ff9020d58e0241fef9995426a21c047f640cef6ee1a197012", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_gm32_cshuffle": { @@ -719,7 +719,7 @@ "bytes": 55218, "family": "conv", "kernel_name": "irhash_conv_950_dw_gm32_cshuffle_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_cshuffle_gm32", - "sha256": "1206d4996392e7179f1444df16817040e7bd22619577dd40dfd6ce5a23fbdc49", + "sha256": "f46ca87228b64ed5ecbff003c9f5736009fffe099e3117b9cd173fcb5103c577", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_gm4_direct": { @@ -727,7 +727,7 @@ "bytes": 77867, "family": "conv", "kernel_name": "irhash_conv_950_dw_gm4_direct_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_default_gm4", - "sha256": "a584de99183fd52df95a5d0fc396d3787eec3ca927f674b32b2a5189ed0acdcd", + "sha256": "a5caaaacce77bbdfafaef850f331c92892d603c3f0295e93c694bbb094cc4bc8", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_gm64_cshuffle": { @@ -735,7 +735,7 @@ "bytes": 55218, "family": "conv", "kernel_name": "irhash_conv_950_dw_gm64_cshuffle_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_cshuffle_gm64", - "sha256": "69953d0e82d2cf47f1e648d1af884938c758384f5717c485fe0573188a45126c", + "sha256": "70f305056ae27afbd5dc1f07d96a2b5d34ba0029138bd93b45cf467db13b3e80", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_gm8_cshuffle": { @@ -743,7 +743,7 @@ "bytes": 55140, "family": "conv", "kernel_name": "irhash_conv_950_dw_gm8_cshuffle_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_cshuffle_gm8", - "sha256": "96ae166005f1fcfee367d84809aa4581ba141465a6536960d575a94fb2eade31", + "sha256": "b77a257f9babc3568635df1cfb8cda9995370f87b618d069fe19f75edad9025d", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_unmerged": { @@ -751,7 +751,7 @@ "bytes": 129043, "family": "conv", "kernel_name": "irhash_conv_950_dw_unmerged_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_default", - "sha256": "1a4f970d5b8d24dcc7ace17254433677ad8fc498fabd38d88cf812ea91f4d027", + "sha256": "3a270329465d1a15e30da7b32e9dae49a8dd3da60e2081ddd2bca3ceccfb26f3", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_unmerged_cshuffle": { @@ -759,7 +759,7 @@ "bytes": 143412, "family": "conv", "kernel_name": "irhash_conv_950_dw_unmerged_cshuffle_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_cshuffle", - "sha256": "89a6de936ac390828d698130be0cd7d06db7482b1627eadff197410a278ed396", + "sha256": "629e5d9daad6a09e9b1bdaa2854f9f9079ba161213db357d4f250494fee309c5", "status": "ok" }, "conv/gfx950/n2h16c32k32r1": { @@ -775,7 +775,7 @@ "bytes": 55906, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_1151_s1_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "6e4e1e8b9e9acc61cc20525665043bcff7b6de2fd9680e7f031c434e944f5798", + "sha256": "0d3bd82fde4d26a685067109040be97e8c97feb88abfd945236a415db92806cb", "status": "ok" }, "conv_dgrad/gfx1151/n1h8c64k64r3_g4": { @@ -783,7 +783,7 @@ "bytes": 56815, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_1151_g4_s1_N1H8W8C64_K64Y3X3G4_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "00188154e87f4a4b1d0a8e3428a28709ba6ab44a16ee6c62ecb01b7c1cc28595", + "sha256": "1acf816f8ab329b948803c9d4ea7e0d450c76c4ce4e3b9888ab6a9bb5f96bc4b", "status": "ok" }, "conv_dgrad/gfx1250/n1h8c64k64r3_g4": { @@ -791,7 +791,7 @@ "bytes": 65307, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_1250_g4_s1_N1H8W8C64_K64Y3X3G4_t32x32x32_w2x1_a16x16x32_mem_default", - "sha256": "a52511cbbb26eff52e96b2a201e223def26d9fadded99d7565c393e9ca93d38f", + "sha256": "d8d0a4e1c7a935d0d43788ba416a8b574c9a61e5a61326ac61623f0d3e81e5d9", "status": "ok" }, "conv_dgrad/gfx90a/n1h8c16k32r3_s1": { @@ -799,7 +799,7 @@ "bytes": 75435, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_90a_s1_N1H8W8C16_K32Y3X3_t64x64x64_w2x2_a32x32x8_mem_default", - "sha256": "998df8292c07f11bbeb4623d092befdd717ba9a972a7f73c06be01c41c5f2222", + "sha256": "f24fd2326881322fc9fcf2de927fcae795dfab746f5bab905ed70a3fac27346f", "status": "ok" }, "conv_dgrad/gfx942/n1h8c16k32r3_s1": { @@ -807,7 +807,7 @@ "bytes": 75435, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_942_s1_N1H8W8C16_K32Y3X3_t64x64x64_w2x2_a32x32x8_mem_default", - "sha256": "23b00d96f195398da05f27b00d9708e34c76eb9a2ed06eb13cf9eb301329413e", + "sha256": "c9f92fa164455cfbd61ed0a0a81e6bb27fa42ea2f754c1db64c0e8c1c2292d97", "status": "ok" }, "conv_dgrad/gfx942/n1h8c64k64r3_g4": { @@ -815,7 +815,7 @@ "bytes": 76507, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_942_g4_s1_N1H8W8C64_K64Y3X3G4_t64x64x64_w2x2_a32x32x8_mem_default", - "sha256": "bc77b28132f9e3b23395a3040172d3ef42ee34410be3be0afaf8efde878f2cb5", + "sha256": "dd0ea5e043feec79bcd3c0ad920bdc49ee8f9d608e9f734bda808a14f1337b71", "status": "ok" }, "conv_dgrad/gfx942/n1h8c64k64r3_g4_s2": { @@ -823,7 +823,7 @@ "bytes": 65777, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_942_g4_s2_N1H8W8C64_K64Y3X3G4_t64x64x32_w2x2_a16x16x16_mem_default", - "sha256": "89d177758a9f158457c04b8f9c3f308fabd01d73ab9e4f1477dd4226ed5e888f", + "sha256": "468a87a8d9cd99176a3108b2d8669745f6a569b1d3761849f400860d34229f00", "status": "ok" }, "conv_dgrad/gfx942/n2h16c32k32r3_s2": { @@ -831,7 +831,7 @@ "bytes": 64834, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_942_s2_N2H16W16C32_K32Y3X3_t64x64x32_w2x2_a16x16x16_mem_default", - "sha256": "69647a52f79467e4fcfc40f79bc8ba6bb1aee2ea0e5025c85607b36a26b676c2", + "sha256": "68149ecf2f41567ded5defd82a09ebd273b191607364f49ece3b5ca25233e7c3", "status": "ok" }, "conv_dgrad/gfx950/n1h8c16k32r3_s1": { @@ -839,7 +839,7 @@ "bytes": 58873, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_950_s1_N1H8W8C16_K32Y3X3_t64x64x64_w2x2_a32x32x16_mem_default", - "sha256": "39c3caaee65b18d46b23862b677d1982abf094d7de816103051baac24791895c", + "sha256": "a3a4c353bae5898c2292fffdfb6f5f9e3f61a969aadce136ce6508602087de94", "status": "ok" }, "conv_dgrad/gfx950/n1h8c64k64r3_g4": { @@ -847,7 +847,7 @@ "bytes": 59941, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_950_g4_s1_N1H8W8C64_K64Y3X3G4_t64x64x64_w2x2_a32x32x16_mem_default", - "sha256": "59d25ce523613009debe9f3105afbde7b5cc0a7387b5514d21fffa53d6522ed7", + "sha256": "304992f448ed7920862ea5b393d89e4f4b13a5637796cb0d35cb5dd0433681d1", "status": "ok" }, "conv_dgrad/gfx950/n2h16c32k32r3_s2": { @@ -855,7 +855,7 @@ "bytes": 64563, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_950_s2_N2H16W16C32_K32Y3X3_t64x64x64_w2x2_a32x32x16_mem_default", - "sha256": "5ca5f8fcd0e567169c1ef6a52528b752ed90368dbb72aa301e98b33f0bc30706", + "sha256": "90b531be0c062f718571f481969324f38edead86d63f0525ff909fa9bf0a0fbf", "status": "ok" }, "conv_direct/gfx942/16c_n1h8_fold_false": { @@ -863,7 +863,7 @@ "bytes": 53428, "family": "conv_direct", "kernel_name": "irhash_direct16c_942_a_N1H8W8_g8_c16k16_bq16_bg4_db", - "sha256": "186edf0eb9337be8de1f97d5b5fad6a96c83060a6997a4f5519b4a8ead4d1277", + "sha256": "35bd5bbce108eb65ad71fdcda15682c59bab58f161b2e8df06789992fa95c677", "status": "ok" }, "conv_direct/gfx942/32c_n2h8": { @@ -871,7 +871,7 @@ "bytes": 195363, "family": "conv_direct", "kernel_name": "irhash_direct32c_942_a_N2H8W8_g4_c32k32_bq32_bg4_db", - "sha256": "a699e7f10c1cd99b3d4b7390a54b2551a0eaa7ffe77b4664647eea3746f9c1ec", + "sha256": "5f0ab561744bdbd077116dba04d8e35742f4c7cc92e180d6269a4ffaf0528b74", "status": "ok" }, "conv_direct/gfx942/8c_n4h16": { @@ -879,7 +879,7 @@ "bytes": 37021, "family": "conv_direct", "kernel_name": "irhash_direct8c_942_a_N4H16W16_g8_c8k8_bq16_bg8_db", - "sha256": "4ff1af388370624ce0f558ff623711d26be94085b29eee3c597c95b10a53f1ee", + "sha256": "e0a08dbaa7bb81bcae70e3977e551855dff63f6a601d22b50fefe2fb91bee057", "status": "ok" }, "conv_direct/gfx942/dw_col_s1_fp16": { @@ -903,7 +903,7 @@ "bytes": 46783, "family": "conv_direct", "kernel_name": "irhash_direct16c_950_bg4_N32H8W8_g16_c16k16_bq16_bg4_db_k32", - "sha256": "f355efc1a398918a22ae5325ff37ed5b6ff5875a0fc9412941f704a9b5f5daf7", + "sha256": "57ab23d89d5dd22d1969fd539901a4ea65f9dad2ff537033144152c54f48920e", "status": "ok" }, "conv_direct/gfx950/16c_n32h8_bg8_fold": { @@ -911,7 +911,7 @@ "bytes": 46796, "family": "conv_direct", "kernel_name": "irhash_direct16c_950_bg8_N32H8W8_g16_c16k16_bq16_bg8_db_k32", - "sha256": "b110989e76777cf6cc11cf5ef09116fa08edfd5b2fe1b525e796279a25b96e09", + "sha256": "da047beeaf423be69f209d727544a577dcf77f145d8ee66925a5f269609a8fa5", "status": "ok" }, "conv_direct/gfx950/32c_n2h8": { @@ -919,7 +919,7 @@ "bytes": 195363, "family": "conv_direct", "kernel_name": "irhash_direct32c_950_a_N2H8W8_g4_c32k32_bq32_bg4_db", - "sha256": "288249761cf39628560a4ff51a50a65de9b4908251aa0fb9cc7a29aa12dd3246", + "sha256": "cea6be5581a343fac023c2a09a14138ffecefc0dd66e74165d5bdb653c5dfc31", "status": "ok" }, "conv_direct/gfx950/4c_n1h8_small": { @@ -943,7 +943,7 @@ "bytes": 37021, "family": "conv_direct", "kernel_name": "irhash_direct8c_950_a_N4H16W16_g8_c8k8_bq16_bg8_db", - "sha256": "cc044a4cc7099faf5a91f1eafecfaeacc275c153d69ee7a88077850cc4c2b477", + "sha256": "1c6df5cc6fb06023d1b04de7b3356a11c5a27939430d6ccc4524e52705a7d74e", "status": "ok" }, "conv_direct/gfx950/dw_col_bw1_tail": { @@ -1151,7 +1151,7 @@ "bytes": 53578, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_1151_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "3fff47fef1c94c503c2e12795ab0e8946bbf1f03d1a5e8f5813fe8c00a1b285a", + "sha256": "be21ba9d6943831bffd89a8be60c7c6d76fa0c9baae2e66b92fabd36cfc855eb", "status": "ok" }, "conv_wgrad/gfx1201/n1h8c16k32r3": { @@ -1159,7 +1159,7 @@ "bytes": 47153, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_1201_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "f63f66171740fc8114e05a7ccaddf9307d3748268aae34897ec3793119e5473f", + "sha256": "e7f3de3dd59168e6b207215b03d70ad5cb811e9203dc4a692c9ab8a5fdea234f", "status": "ok" }, "conv_wgrad/gfx1250/n1h8c64k64r3_g4": { @@ -1167,7 +1167,7 @@ "bytes": 66522, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_1250_g4_N1H8W8C64_K64Y3X3G4_t32x32x32_w2x1_a16x16x32_mem_default", - "sha256": "31b7548aa1b9b4f85543f16089238aa692f2002362871aca946221f8993afc02", + "sha256": "12b3aeebc16eee6aa445c2bad6fd46ef7a5bafcbfd0b44e5b2b85e445c199b89", "status": "ok" }, "conv_wgrad/gfx90a/n1h8c16k32r3": { @@ -1175,7 +1175,7 @@ "bytes": 21866, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_90a_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_cshuffle", - "sha256": "bf4e0d4aa877c9bde4ddb494cc51ecbf6b0604d9e398a6eec575f6b4415d0877", + "sha256": "23904cb835090b8e977ffbc80561ae67594fdfe0057a2964c205fbc5dd30daa8", "status": "ok" }, "conv_wgrad/gfx942/n1h8c16k32r3": { @@ -1183,7 +1183,7 @@ "bytes": 21866, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_cshuffle", - "sha256": "5c0a33026f958a437bea9ad18b33d3e9dbcc9821923b2373e203f3edb1cb0222", + "sha256": "189776b3f32a1eec804862c5c0eddc23480f40558cf287f6b240d68a886c3422", "status": "ok" }, "conv_wgrad/gfx942/n1h8c16k32r3_sk_atomic_pers": { @@ -1191,7 +1191,7 @@ "bytes": 25156, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_sk_atomic_pers_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_default_sk_skpers_skratomic", - "sha256": "1d70097054f9a71c691f4b89a87c5490e8b2371cc3c243cb7a725f8640744d20", + "sha256": "cfc724cd8c8f8f4a41568701dc7ab7811f6d6b5d6228c1fb9a8fa5e44c01399a", "status": "ok" }, "conv_wgrad/gfx942/n1h8c64k64r3_g4": { @@ -1199,7 +1199,7 @@ "bytes": 22466, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_g4_N1H8W8C64_K64Y3X3G4_t64x32x16_w2x2_a16x16x16_mem_cshuffle", - "sha256": "5bcf94f8ea7b81d9f15f2fd7edc5c7523831d876e472ba422b031cf0e4f9e21e", + "sha256": "a835c3377e76a4a116e4fadfec5b061f75c15877f477333713c41df5bc9d4924", "status": "ok" }, "conv_wgrad/gfx942/n1h8c64k64r3_g4_sk_tree_pers": { @@ -1207,7 +1207,7 @@ "bytes": 44396, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_g4_sk_tree_pers_N1H8W8C64_K64Y3X3G4_t64x32x16_w2x2_a16x16x16_mem_default_sk_skpers_skrtree", - "sha256": "4c3f744ae6f1ae5198f47461f06f90611d4d15fac5322b87147501133956c0c2", + "sha256": "719938a01239f2c665c2802f6209e5e2f1a6d34a1ab165f7a603a278533f8d0f", "status": "ok" }, "conv_wgrad/gfx942/n1h8c64k64r3_g4_sk_workspace": { @@ -1215,7 +1215,7 @@ "bytes": 25798, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_g4_sk_workspace_N1H8W8C64_K64Y3X3G4_t64x32x16_w2x2_a16x16x16_mem_default_sk_skrworkspace", - "sha256": "21bee95d9ae4d04fde79971cb14764bc3207d0c63d590a45eb2e1e6340a01b0e", + "sha256": "cead988dae6fa0b4b3a11d394394dd2788e3e6a76156be964222f6d43884e157", "status": "ok" }, "conv_wgrad/gfx942/n1h8c64k64r3_g4_spk4": { @@ -1223,7 +1223,7 @@ "bytes": 26896, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_g4_spk4_N1H8W8C64_K64Y3X3G4_t64x32x16_w2x2_a16x16x16_mem_cshuffle_spk", - "sha256": "b1ab83e217ce915a0d2e4c7cab9bcdea3f7f4dcd85813c7e965ee5da558745bf", + "sha256": "b679eee7b5aea368d4e7996ba4d5a95e3f652baecc95529dfa36a662b1cd9f87", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3": { @@ -1231,7 +1231,7 @@ "bytes": 29219, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_a_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_mem_cshuffle", - "sha256": "3554714549ab49184e12b1c41aa45d1eb8aea9de8250f926d3674d9b3d8e4b32", + "sha256": "2c0e6c03ca8a40fdd6c8f4c90f2243c9912f9d03d2bb7b87718ff3c0f7b3d5ef", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3_sk_atomic": { @@ -1239,7 +1239,7 @@ "bytes": 29815, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_sk_atomic_N1H8W8C16_K32Y3X3_t64x64x16_w2x2_a32x32x16_mem_default_sk_skratomic", - "sha256": "d9c46d394540a321408c425dc012480fd6a7688f52c697f0cb7c68f0f261e5bb", + "sha256": "c53bda427119ad0c08243e4610a194fd81157912d2dddb432204d91c2c0789c2", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3_sk_linear": { @@ -1247,7 +1247,7 @@ "bytes": 58937, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_sk_linear_N1H8W8C16_K32Y3X3_t64x64x16_w2x2_a32x32x16_mem_default_sk", - "sha256": "1819f9e2134239476b7315a2a059a7c2014004af3829fd5a764dbb06fea4b05d", + "sha256": "448f9b9541cba8f157274ec968215b3a7ebe2ec7e49a5ef65bc74da95bb4a4bc", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3_sk_tree": { @@ -1255,7 +1255,7 @@ "bytes": 60379, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_sk_tree_N1H8W8C16_K32Y3X3_t64x64x16_w2x2_a32x32x16_mem_default_sk_skrtree", - "sha256": "1d2674baa67db756ba52747582b5551d663aaf9e3d56b8069bd83665de9a5485", + "sha256": "467de800a91ff9e88e478536afc7ea9d09a8b46b2643648dd592fabb050968f3", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3_sk_workspace_pers": { @@ -1263,7 +1263,7 @@ "bytes": 31327, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_sk_workspace_pers_N1H8W8C16_K32Y3X3_t64x64x16_w2x2_a32x32x16_mem_default_sk_skpers_skrworkspace", - "sha256": "9d25da3d386646386e6e4075eec6794d76c3eda4da66121976d59a15ee4f0cde", + "sha256": "1d7cbbabfefe4ec75a18fa266c7342ee39f56332e8d83d82a7b31dd067d97618", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3_spk4": { @@ -1271,7 +1271,7 @@ "bytes": 26145, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_spk4_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_cshuffle_spk", - "sha256": "0d88a6ead61bfaafb1cbacf93e7927e5aa5e5a784a523d4781af30b3b661f7e2", + "sha256": "f88eb29eee495f0fa9ea2e0e1d3e543902a238589e117588cec1b0fbd2ffcf94", "status": "ok" }, "conv_wgrad/gfx950/n1h8c64k64r3_g4": { @@ -1279,7 +1279,7 @@ "bytes": 29930, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_g4_N1H8W8C64_K64Y3X3G4_t64x64x32_w2x2_a32x32x16_mem_cshuffle", - "sha256": "909036d510a3f694196117091bcf8ca52be3c767a3ec60da7519451c740fcae9", + "sha256": "288108b86bb115411fa60870d1246174789ef27f21beea603868acb2d830e57e", "status": "ok" }, "conv_wgrad/gfx950/n1h8c64k64r3_g4_cshuffle": { @@ -1287,7 +1287,7 @@ "bytes": 30209, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_g4_cshuffle_N1H8W8C64_K64Y3X3G4_t64x64x32_w2x2_a32x32x16_mem_cshuffle", - "sha256": "7ec4ccded2d060144039b3c5a644dcc8d4fd36c3f95c8a79384d205d687a5ba6", + "sha256": "572b642695d78b8f16cc0a69a4d94996a4c0a24af6329b9dfdc0a6b39459c8c3", "status": "ok" }, "conv_wgrad/gfx950/n1h8c64k64r3_g4_sk_linear_pers": { @@ -1295,7 +1295,7 @@ "bytes": 61458, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_g4_sk_linear_pers_N1H8W8C64_K64Y3X3G4_t64x64x16_w2x2_a32x32x16_mem_default_sk_skpers", - "sha256": "7e23d8219721828777ad6b20909fc105514a4304d1cb638f0779dadafa35eb7f", + "sha256": "94805ac108fd3af01ec24785879520d7671d0a329eb2ca5557da2d1fcb937910", "status": "ok" }, "conv_wgrad/gfx950/n1h8c64k64r3_g4_spk4": { @@ -1303,7 +1303,7 @@ "bytes": 38655, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_g4_spk4_N1H8W8C64_K64Y3X3G4_t64x64x32_w2x2_a32x32x16_mem_cshuffle_spk", - "sha256": "0d8e9e332da64db801bd8a9454a75983548949319ba945cb2f78fcb1b41277a2", + "sha256": "6e19701dff1ca74eefcaa75027a91e42eef9aa5b5c081b00987e5e0901de2a51", "status": "ok" }, "conv_wgrad/gfx950/n2h16c32k32r1": { @@ -1343,7 +1343,7 @@ "bytes": 22397, "family": "conv_wgrad_two_stage", "kernel_name": "irhash_wgrad_942_twostage_spk4_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_default_spk_twostage_wsr8", - "sha256": "1e019185a3ad4c9cecdd1aef2ee94218c7759f6b49d9a0c5c9520cdce47690e2", + "sha256": "cc4be08c887b128c272e9aacdb5002e2a872fb95655e0dfb9111bdd3c982a7ae", "status": "ok" }, "conv_wgrad_two_stage/gfx950/n1h8c16k32r3_spk4": { @@ -1351,7 +1351,7 @@ "bytes": 22397, "family": "conv_wgrad_two_stage", "kernel_name": "irhash_wgrad_950_twostage_spk4_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_default_spk_twostage_wsr8", - "sha256": "97e5d82a9b670492822cc4e88c8c71766d0269519cda08f4d75775010d33c4fa", + "sha256": "81209850b52419c54504906cea8ed6fe73e4729a82244f8485c65aa367a6b668", "status": "ok" }, "crosslane/gfx1201/warp_shuffle_xor_dpp_reduce_i32": { @@ -1399,7 +1399,7 @@ "bytes": 54706, "family": "deep_fused_conv", "kernel_name": "irhash_deep_gfx950_wt16_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "2fd3ad4bcac9092b856a16136af552af4badc006ba2c88d69133b09a5f772356", + "sha256": "70013b1dc855589f7fbd129f49e72d57e7e9df224b91843670c703de5bad8d4b", "status": "ok" }, "deep/common_gfx950_wt16/h24w24_k32": { @@ -1407,7 +1407,7 @@ "bytes": 54755, "family": "deep_fused_conv", "kernel_name": "irhash_deep_gfx950_wt16b_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "de7a67013aaae877931f42261ba345957751a43f2df93d968a037b4f00757c38", + "sha256": "169c79786205c721363f6b5813e82d9f60bdf41e0763dc8d681234a3585fc043", "status": "ok" }, "deep/gfx1151/native_h16w16": { @@ -1439,7 +1439,7 @@ "bytes": 73896, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "b673409ceddc0a1ff69a10b91d7d090db02d22818ee02bea9dbce572d87ad610", + "sha256": "55e483936686150548d5576bf0f3b800cef75f2fb22a6cebfd7358e1afb0f786", "status": "ok" }, "deep/gfx1201/h24w24_k32": { @@ -1447,7 +1447,7 @@ "bytes": 73897, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "c8f3bee16186da4bdc383c2eba992d26d348c3625dbd61e88bf58b3854b6660c", + "sha256": "e744fc8d617f429ecb84de674f8cd9961e260e7e7a99c63768fbcf2f5ea7374f", "status": "ok" }, "deep/gfx12_generic/h16w16_k32": { @@ -1455,7 +1455,7 @@ "bytes": 73896, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "b673409ceddc0a1ff69a10b91d7d090db02d22818ee02bea9dbce572d87ad610", + "sha256": "55e483936686150548d5576bf0f3b800cef75f2fb22a6cebfd7358e1afb0f786", "status": "ok" }, "deep/gfx950/h16w16_k32": { @@ -1463,7 +1463,7 @@ "bytes": 33559, "family": "deep_fused_conv", "kernel_name": "rocke_gfx950_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a32x32x16_mem_cshuffle_epi_relu", - "sha256": "c8596da316c20a71ea75d32d025f09a071d07b0aaf98c366379f4ca5fd9a3589", + "sha256": "f7474b1f9b064ae37b6680c91d2e17b4a6f365a9b47f74097c56163d4ee4c018", "status": "ok" }, "deep/gfx950/h24w24_k32": { @@ -1471,7 +1471,7 @@ "bytes": 33563, "family": "deep_fused_conv", "kernel_name": "rocke_gfx950_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a32x32x16_mem_cshuffle_epi_relu", - "sha256": "4243b5bde3de11deacc8c69f8a5cc83467742f646b8bc46e13a6db5ba5739746", + "sha256": "7a8717fa04474d2911134c644b971215637d3c744afad224c81b0c3c2ceb15bc", "status": "ok" }, "fused_moe/gfx950/gather_h128": { @@ -1839,7 +1839,7 @@ "bytes": 2883, "family": "moe", "kernel_name": "irhash_moe_sort_scatter_scatter_T64_K2_E16_b128", - "sha256": "42b46dbb2203f1d2b1a25918ee0a355be2bd4e049ed4091f265975c6905d5e12", + "sha256": "45fda4bf93270823802bc61f52756750722b658a82b9c8cd67a49e69f8e758d5", "status": "ok" }, "nontemporal/gfx942/copy_bf16x8": { @@ -2145,7 +2145,7 @@ "bytes": 46267, "family": "attention", "kernel_name": "rocke_unified_attention_2d_scalar_q64_h4_kv2_d128_b16_fp16_rbu", - "sha256": "7f66eed09c4e1b24451f2ca0924b4d38c94bce086e758084378f8cf1d70be569", + "sha256": "51fe41aaaa3623be88d35d880007c86f723d1821ae0d32a511a3bbe530c682c4", "status": "ok" }, "attention/gfx950/scalar2d_fp16_d128_sw16_rb8": { @@ -2153,7 +2153,7 @@ "bytes": 46601, "family": "attention", "kernel_name": "rocke_unified_attention_2d_scalar_q64_h4_kv2_d128_b16_fp16_sw_rb8", - "sha256": "2c09ce21c74489d0ee9f54469e3eb6223329af9115226605574befb837e570f0", + "sha256": "1acd98d1834d52cec79c2a70bd09572d96d168423fdc6a11a1deac4e95901916", "status": "ok" }, "attention/gfx950/scalar2d_fp16_d128_top_left": { @@ -2161,7 +2161,7 @@ "bytes": 46371, "family": "attention", "kernel_name": "rocke_unified_attention_2d_scalar_q64_h4_kv2_d128_b16_fp16_tl", - "sha256": "b8ecab004b5607bceaf40f52cf787b6e4f1922d6420f3777fabdaf80bd6ec96d", + "sha256": "56f3f2d5411ee663955e06e799c0be8770248af65df1d5dc7fede2ea8ff048be", "status": "ok" }, "attention/gfx950/scalar3d_fp16_d128_sw16_rb8": { @@ -2169,7 +2169,7 @@ "bytes": 49102, "family": "attention", "kernel_name": "rocke_unified_attention_3d_scalar_q64_h4_kv2_d128_b16_seg8_fp16_rb8_sw", - "sha256": "f8b29b9ed628a427e29ebf81e51e98836e3bd1fa6a2a3601413cdcab5b185224", + "sha256": "1e75fb5a2c80f393ec5687d943261ba4ee4afa9304e2fb6f143a1e1accc3fd07", "status": "ok" }, "attention/gfx950/scalar3d_fp16_d128_top_left": { @@ -2177,7 +2177,7 @@ "bytes": 48862, "family": "attention", "kernel_name": "rocke_unified_attention_3d_scalar_q64_h4_kv2_d128_b16_seg8_fp16_tl", - "sha256": "2b50b06455e4a0503c58a1dd87437327acd147ee17f19323753e95c9e4bd8b23", + "sha256": "99353476cc9677d60f117ba680c708c996663844c0bb4b51f2a16308948a65ba", "status": "ok" }, "attention_d128_swa/gfx942/4warp_gqa_fold": { @@ -2561,7 +2561,7 @@ "bytes": 30449, "family": "conv", "kernel_name": "irhash_conv_1151_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_cshuffle", - "sha256": "a9653258fb29f11c6b61932477974f87ec1be4173646254e45c6f505079bbdce", + "sha256": "cbfe03763ea9b4b5b7cc9f00864be5843387ae9c579acf2147faf422da4c150f", "status": "ok" }, "conv/gfx1151/n2h16c32k32r1": { @@ -2577,7 +2577,7 @@ "bytes": 24078, "family": "conv", "kernel_name": "irhash_conv_1201_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_cshuffle", - "sha256": "3f8821f5ec4613af77c134557832814ecd6262a503f242c37b31b46a2c19406d", + "sha256": "e958db8b566b938a9874b69ebdb6bbfd38c084cbeb82fc5841e37de81bb7d362", "status": "ok" }, "conv/gfx90a/n1h8c16k32r3": { @@ -2585,7 +2585,7 @@ "bytes": 23752, "family": "conv", "kernel_name": "irhash_conv_90a_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_cshuffle", - "sha256": "1c188a261159004377bc0e6f8c7f753e3b396c0448407799a05320e207754370", + "sha256": "2bd0f5ad4a451f6a060db25105c1093588241325c5e6c6f6a369eb9a6e9f60ee", "status": "ok" }, "conv/gfx942/n1h8c16k32r3": { @@ -2593,7 +2593,7 @@ "bytes": 23752, "family": "conv", "kernel_name": "irhash_conv_942_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_cshuffle", - "sha256": "07b605d68462a88d6acb81d655c8a08a3f9afa96e1f6d2a08236b86d3803df07", + "sha256": "266fb42458642fc89a184bd7bfaab09b7a6fb790dc98bf9c327c58575d97e0fa", "status": "ok" }, "conv/gfx942/n2h14c64k64r3g64/dw_gm8_cshuffle": { @@ -2601,7 +2601,7 @@ "bytes": 25019, "family": "conv", "kernel_name": "irhash_conv_942_dw_gm8_N2H14W14C64_K64Y3X3G64_t64x32x16_w2x2_a16x16x16_mem_cshuffle_gm8", - "sha256": "41e313b5734cb98db6c2c70751aa2429b15513519ab8192ba5f48731ad4c9e8b", + "sha256": "24641cddc14e01a6a4ec40c57ad6996d9f37d09828ec02e12605140a840b9e4d", "status": "ok" }, "conv/gfx950/n1h8c16k32r3": { @@ -2609,7 +2609,7 @@ "bytes": 25084, "family": "conv", "kernel_name": "irhash_conv_950_a_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_compv4_cshuffle", - "sha256": "32577594d636778b5b6713141f1c9f556f6b4b78e34500704c4757072205ab6b", + "sha256": "b8bf7a64f45334d0e6172c24469d170e2067a8f40a3c7afdf600fa30f328e449", "status": "ok" }, "conv/gfx950/n1h8c16k32r3/basic_cshuffle": { @@ -2617,7 +2617,7 @@ "bytes": 24764, "family": "conv", "kernel_name": "irhash_conv_950_basic_b_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_basic_cshuffle", - "sha256": "528575a912fbdfb4ec3abce37a49ec4a15d9371f8ce07b9e351985b1f72854ea", + "sha256": "3f019254b83ab74a52debc1ff464e02ede46ae052a50a5138eb9bb820b779217", "status": "ok" }, "conv/gfx950/n1h8c16k32r3/basic_default": { @@ -2625,7 +2625,7 @@ "bytes": 39984, "family": "conv", "kernel_name": "irhash_conv_950_basic_a_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_basic_default", - "sha256": "4a61265b0c4a2e5c44f69add57d452a2d656c7c017dacc84159745856d29f8fd", + "sha256": "8e75ad5ba97f3831841e8314cefb6a4e4d2467cf2a4e7d73e24172c5a8e98325", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_gm32_cshuffle": { @@ -2633,7 +2633,7 @@ "bytes": 55357, "family": "conv", "kernel_name": "irhash_conv_950_dw_gm32_cshuffle_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_cshuffle_gm32", - "sha256": "b6a87c5f276c62a6462fc21fe6231cd8a1f50d3673fc4fda276ec103392c35ba", + "sha256": "f453bbb552072886df6e781aa9e2efc4844e721b103c2fad460a7057b9b0ce40", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_gm4_direct": { @@ -2641,7 +2641,7 @@ "bytes": 78006, "family": "conv", "kernel_name": "irhash_conv_950_dw_gm4_direct_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_default_gm4", - "sha256": "6c32f84fd05eb53b1b391aad3f1efd26a7b732306450f1c6aaf6b04d29778c5c", + "sha256": "1d7aa5bbe8417c17bf6c1b4d746a9e22ddd6e56a524e018a0c9990b24b3524f3", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_gm64_cshuffle": { @@ -2649,7 +2649,7 @@ "bytes": 55357, "family": "conv", "kernel_name": "irhash_conv_950_dw_gm64_cshuffle_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_cshuffle_gm64", - "sha256": "78179627a5dc54af62acda8edce1cb54537fcd3ce3038a34189f1e606bab1061", + "sha256": "f42d24ede547a842608467a527f85ec96d0f997d65fe6cf6d639e63a9e733a59", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_gm8_cshuffle": { @@ -2657,7 +2657,7 @@ "bytes": 55279, "family": "conv", "kernel_name": "irhash_conv_950_dw_gm8_cshuffle_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_cshuffle_gm8", - "sha256": "e326130d3318ea638c74b86d7ebee3380e4c352398a491f63298d762c857f120", + "sha256": "d19a095f820cb1d16d74ab7040b41c3ab328f4d7c5d8633b4310a2cc12708afe", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_unmerged": { @@ -2665,7 +2665,7 @@ "bytes": 129182, "family": "conv", "kernel_name": "irhash_conv_950_dw_unmerged_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_default", - "sha256": "c1a39bc9502d02fa2d1011dd087550a1515b454282fe4bf1caf605756c6259a3", + "sha256": "c897e8c40a276697fa783e635bc76bd0753d41f723d00df8e7590e37d15387fb", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_unmerged_cshuffle": { @@ -2673,7 +2673,7 @@ "bytes": 143551, "family": "conv", "kernel_name": "irhash_conv_950_dw_unmerged_cshuffle_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_cshuffle", - "sha256": "ad098bd8250126b9183038455eeaec0bc81e0fcb8f5c8bc5edb2901b60d96e44", + "sha256": "b685d7905d70f7f28b93deb49da01e8e6320127a3e51cabd8340115e4e6dfc4e", "status": "ok" }, "conv/gfx950/n2h16c32k32r1": { @@ -2689,7 +2689,7 @@ "bytes": 56045, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_1151_s1_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "87624afac14b3b91743cd3a50c420c5892ab7fd44c7c5ea25920a01efa8c1f72", + "sha256": "ba43396ccae79ac531c41c800987818c142b4c69e5b2e33563e8c661f204b6fb", "status": "ok" }, "conv_dgrad/gfx1151/n1h8c64k64r3_g4": { @@ -2697,7 +2697,7 @@ "bytes": 56954, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_1151_g4_s1_N1H8W8C64_K64Y3X3G4_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "4b5b804f37d4e4f90d2efa0ced040e14c0366cd16137135cdb6171e42a85c7b4", + "sha256": "d042691dd4445125407b2f66404ea60d70a7eb812115ace4be05d8c341e812c3", "status": "ok" }, "conv_dgrad/gfx1250/n1h8c64k64r3_g4": { @@ -2705,7 +2705,7 @@ "bytes": 65446, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_1250_g4_s1_N1H8W8C64_K64Y3X3G4_t32x32x32_w2x1_a16x16x32_mem_default", - "sha256": "85bf214fcb8daf16998c22335d04d6ebe6e484e601e0de4caa7b2398129635df", + "sha256": "70aed4ee8cb829f2f4f2415a6f973fabc9585de87926eea3d70c49ac9f443775", "status": "ok" }, "conv_dgrad/gfx90a/n1h8c16k32r3_s1": { @@ -2713,7 +2713,7 @@ "bytes": 75574, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_90a_s1_N1H8W8C16_K32Y3X3_t64x64x64_w2x2_a32x32x8_mem_default", - "sha256": "c5da094684cbb0905876019342704a4aaaa1c5737d82393e50025444e59d6706", + "sha256": "8e45fcb041c2a103906ebe41e22a4390a2c603d41341402a13f3ec04516e89a3", "status": "ok" }, "conv_dgrad/gfx942/n1h8c16k32r3_s1": { @@ -2721,7 +2721,7 @@ "bytes": 75574, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_942_s1_N1H8W8C16_K32Y3X3_t64x64x64_w2x2_a32x32x8_mem_default", - "sha256": "6615217425866fb9c3257ea80e91cb86e90392bcdc51f58fc7c2215c53581c27", + "sha256": "00160f744fbd8d080e67693067673ec82dfe8e78225176c6125b8adc4cd4e788", "status": "ok" }, "conv_dgrad/gfx942/n1h8c64k64r3_g4": { @@ -2729,7 +2729,7 @@ "bytes": 76646, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_942_g4_s1_N1H8W8C64_K64Y3X3G4_t64x64x64_w2x2_a32x32x8_mem_default", - "sha256": "1a84139167a926ef7624739dd80266ea06e7f22c3955b6304b6857f282c5f85d", + "sha256": "abbb38b0778720ed1e09a9105c1314d6cb3eb40f97d0d1d3c3257344eaeddcd1", "status": "ok" }, "conv_dgrad/gfx942/n1h8c64k64r3_g4_s2": { @@ -2737,7 +2737,7 @@ "bytes": 65916, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_942_g4_s2_N1H8W8C64_K64Y3X3G4_t64x64x32_w2x2_a16x16x16_mem_default", - "sha256": "a9e933a04b40f246247e6c8696b9d1b880aa79fdbf01ecea381c645a35d8ae61", + "sha256": "ca2ae07b8e5d3b47de0eacc631772a07e5fe23fc7b9eef1d401c82b444589c6a", "status": "ok" }, "conv_dgrad/gfx942/n2h16c32k32r3_s2": { @@ -2745,7 +2745,7 @@ "bytes": 64973, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_942_s2_N2H16W16C32_K32Y3X3_t64x64x32_w2x2_a16x16x16_mem_default", - "sha256": "13a3bf2feb9e0a82066979784eededf8916f4ef26ba51a2af052b21ced82882a", + "sha256": "92bc7e7777698c71e2ad4f8a42685062f1b04abfea2730c07668ed03a0cde534", "status": "ok" }, "conv_dgrad/gfx950/n1h8c16k32r3_s1": { @@ -2753,7 +2753,7 @@ "bytes": 59012, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_950_s1_N1H8W8C16_K32Y3X3_t64x64x64_w2x2_a32x32x16_mem_default", - "sha256": "e450cfc86966a963049a573ce9877490c7803d2c98e48780a16a5b2910915ede", + "sha256": "e08ab65359c96cd2088c7393569b0d4636d0ca6a8aa8cbde0ed4e3ca16774067", "status": "ok" }, "conv_dgrad/gfx950/n1h8c64k64r3_g4": { @@ -2761,7 +2761,7 @@ "bytes": 60080, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_950_g4_s1_N1H8W8C64_K64Y3X3G4_t64x64x64_w2x2_a32x32x16_mem_default", - "sha256": "b06249b9029c92e36c7a5ad62d5e4eb73721574c432fb4849080b1ccc5d859bc", + "sha256": "90b0fcdc8768d41d5f57471bbda641c9e503166141f85645cf22c57f73bf87a4", "status": "ok" }, "conv_dgrad/gfx950/n2h16c32k32r3_s2": { @@ -2769,7 +2769,7 @@ "bytes": 64702, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_950_s2_N2H16W16C32_K32Y3X3_t64x64x64_w2x2_a32x32x16_mem_default", - "sha256": "889882855e0a065a7730d1eeaccd61b675924fbfbf8dfd7cfdc5bcece7d02de4", + "sha256": "702447cee30ec90a6276773d44ebd4a260597f0fcf77c01b79b040e9eca8f054", "status": "ok" }, "conv_direct/gfx942/16c_n1h8_fold_false": { @@ -2777,7 +2777,7 @@ "bytes": 53561, "family": "conv_direct", "kernel_name": "irhash_direct16c_942_a_N1H8W8_g8_c16k16_bq16_bg4_db", - "sha256": "3890d2eb4712a7a231238be776409ab782987f06aeb1f9d56b777ef3dd585160", + "sha256": "717aaffd0883140fd84b9f3ac0bd25c72a0dc0b8c0af60e087cc13b905c04c03", "status": "ok" }, "conv_direct/gfx942/32c_n2h8": { @@ -2785,7 +2785,7 @@ "bytes": 195502, "family": "conv_direct", "kernel_name": "irhash_direct32c_942_a_N2H8W8_g4_c32k32_bq32_bg4_db", - "sha256": "c58f91fd839d37204c0d1cd90b4342c5e34e18bc5b3a6f9a964576b7c7430912", + "sha256": "b2ea1dfe3ebefa4e9fce63d8def8d61dba08236268cd247b6bac03bf416f51dd", "status": "ok" }, "conv_direct/gfx942/8c_n4h16": { @@ -2793,7 +2793,7 @@ "bytes": 37154, "family": "conv_direct", "kernel_name": "irhash_direct8c_942_a_N4H16W16_g8_c8k8_bq16_bg8_db", - "sha256": "3d2ab3c5284e65e772e05b6d82fcff467eee009e96eae1e5ac39aca630a7a504", + "sha256": "e3aaac6128987b34be0cbcf3fe44f6f8661a40ade9b5ec18c4218b5e15f196c9", "status": "ok" }, "conv_direct/gfx942/dw_col_s1_fp16": { @@ -2817,7 +2817,7 @@ "bytes": 46916, "family": "conv_direct", "kernel_name": "irhash_direct16c_950_bg4_N32H8W8_g16_c16k16_bq16_bg4_db_k32", - "sha256": "68f6ad7b791c6f1cd21c519d67c745d27d1c7ef80602a1a5c19fc8d272e0a073", + "sha256": "5fe846edc37d4e7579b3e6791904cf61518685c41a1d3dc17b1409641b558c47", "status": "ok" }, "conv_direct/gfx950/16c_n32h8_bg8_fold": { @@ -2825,7 +2825,7 @@ "bytes": 46929, "family": "conv_direct", "kernel_name": "irhash_direct16c_950_bg8_N32H8W8_g16_c16k16_bq16_bg8_db_k32", - "sha256": "b248f79a644981627f246e397783934149a05a834ece7ee7872884c0f3acc2b6", + "sha256": "f0e1c7fd949348231a1a3c78f4c32f2ede172e4a64c9eb2734e6e0a453b53e8b", "status": "ok" }, "conv_direct/gfx950/32c_n2h8": { @@ -2833,7 +2833,7 @@ "bytes": 195502, "family": "conv_direct", "kernel_name": "irhash_direct32c_950_a_N2H8W8_g4_c32k32_bq32_bg4_db", - "sha256": "f07249a3203a6721c506748bcaaf012e0f58621e0c637a1adb967d785860aa29", + "sha256": "a0bb4e6df452a79a93972a630dc042633e6d1c52a33d96df1a15ebf06ec63916", "status": "ok" }, "conv_direct/gfx950/4c_n1h8_small": { @@ -2857,7 +2857,7 @@ "bytes": 37154, "family": "conv_direct", "kernel_name": "irhash_direct8c_950_a_N4H16W16_g8_c8k8_bq16_bg8_db", - "sha256": "08347260bfeb35e31ec3a06f35be99d8d915d364aad8868f75faddf43f19ded3", + "sha256": "540762918fb46d259972e7b67488754e195a3dc358fe9051a848e4f8fba2c0d7", "status": "ok" }, "conv_direct/gfx950/dw_col_bw1_tail": { @@ -3065,7 +3065,7 @@ "bytes": 53717, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_1151_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "10058468003c76cdefe5f8372d5c1720e752634c46ce0d595f3da72b6b903ec7", + "sha256": "8d76d1be0d93e52671118aa8b00201a8c0d87596366a83e31ddf841c433586a3", "status": "ok" }, "conv_wgrad/gfx1201/n1h8c16k32r3": { @@ -3073,7 +3073,7 @@ "bytes": 47292, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_1201_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "15b8ae7c47e5823ce43b5ecfff1049e199b052147b67de102fe766632c19580b", + "sha256": "b5f8f9f39f36b0de3a851c7e7b737085a51f70d5be9aef0fb1144ef35913f548", "status": "ok" }, "conv_wgrad/gfx1250/n1h8c64k64r3_g4": { @@ -3081,7 +3081,7 @@ "bytes": 66661, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_1250_g4_N1H8W8C64_K64Y3X3G4_t32x32x32_w2x1_a16x16x32_mem_default", - "sha256": "cb0d7860ca4eee3f55e95d320d8d3460262905044b7bc09d6d49c1449afcd2b6", + "sha256": "c6f7a637d3908a92a88ab55c66b3398a3492a4a13031e72fc16a9959091abad2", "status": "ok" }, "conv_wgrad/gfx90a/n1h8c16k32r3": { @@ -3089,7 +3089,7 @@ "bytes": 21999, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_90a_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_cshuffle", - "sha256": "eb652794e8d5e396e13669c66fcfdb08addee7e9b91c3bf2a39898dcacb183cf", + "sha256": "285284b903adc27f1832f6e2f86f5ab565e0fff6dd0216e41e97a6832cd12eb6", "status": "ok" }, "conv_wgrad/gfx942/n1h8c16k32r3": { @@ -3097,7 +3097,7 @@ "bytes": 21999, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_cshuffle", - "sha256": "a4d404a44b5f8a351ce2f2d24957c976c79d0ee13887ff15c7c7a47438575b82", + "sha256": "dfb2078fbe57dc66837b42b6c4bfcfab124ad5d892be7a8c83a2a8ed10416a7e", "status": "ok" }, "conv_wgrad/gfx942/n1h8c16k32r3_sk_atomic_pers": { @@ -3105,7 +3105,7 @@ "bytes": 25289, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_sk_atomic_pers_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_default_sk_skpers_skratomic", - "sha256": "f4fa2ad553aa4a1180c5a4a67d3d13cd86f0ee6671bfad4954dd55c93fe67e3d", + "sha256": "f33280523279deda1e7e4b38d559332cebec0ec482d1c2ae44547f771e809358", "status": "ok" }, "conv_wgrad/gfx942/n1h8c64k64r3_g4": { @@ -3113,7 +3113,7 @@ "bytes": 22599, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_g4_N1H8W8C64_K64Y3X3G4_t64x32x16_w2x2_a16x16x16_mem_cshuffle", - "sha256": "dbf4caa3716ae46dffa53f4ab0ec58b505d5196239c88d0e21216614d9141729", + "sha256": "4153f2777dc61c81d9d28fbc046ea489b16c1c9f9648b58a2fe1f2835cfa4d88", "status": "ok" }, "conv_wgrad/gfx942/n1h8c64k64r3_g4_sk_tree_pers": { @@ -3121,7 +3121,7 @@ "bytes": 44535, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_g4_sk_tree_pers_N1H8W8C64_K64Y3X3G4_t64x32x16_w2x2_a16x16x16_mem_default_sk_skpers_skrtree", - "sha256": "2ef49bc86988a2fe9177d29325c801bb9ec0e7631a3255cc5a6d0fc2c86179be", + "sha256": "961286137a6c5e019406679ae46e1a67ce50e60fba4d78dffb0a19a539aaa585", "status": "ok" }, "conv_wgrad/gfx942/n1h8c64k64r3_g4_sk_workspace": { @@ -3129,7 +3129,7 @@ "bytes": 25931, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_g4_sk_workspace_N1H8W8C64_K64Y3X3G4_t64x32x16_w2x2_a16x16x16_mem_default_sk_skrworkspace", - "sha256": "266d9829bcf2d91378ebbf7d2840fce771486b21d08c460fda3bbc1f206309e2", + "sha256": "2aa3add805ac1f34a8fca90dc9f4b62e778cd670404dab379a4ba28ede7db8d0", "status": "ok" }, "conv_wgrad/gfx942/n1h8c64k64r3_g4_spk4": { @@ -3137,7 +3137,7 @@ "bytes": 27035, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_g4_spk4_N1H8W8C64_K64Y3X3G4_t64x32x16_w2x2_a16x16x16_mem_cshuffle_spk", - "sha256": "446d68ff4338f00c758d5ff0ddd7febe5b516457b78851c1079949fc5cd17f5b", + "sha256": "7411ee05b8fc22b908f5910efb04b368a0471acf8c49a527507c833e536feb1c", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3": { @@ -3145,7 +3145,7 @@ "bytes": 29352, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_a_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_mem_cshuffle", - "sha256": "46e8d22587b5a0f9bb3a0a5fe7d351933bd089e7d2c263535cc7d760631c2886", + "sha256": "5a53cae4d689ff5d80debe771e647f1af05f7050828eaa8cc05821c89dd44538", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3_sk_atomic": { @@ -3153,7 +3153,7 @@ "bytes": 29948, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_sk_atomic_N1H8W8C16_K32Y3X3_t64x64x16_w2x2_a32x32x16_mem_default_sk_skratomic", - "sha256": "ed4b63d61ea4edfd5c4e4c2a3aaef71fb586c7c0ebd5441ea975d8dfa516ba0d", + "sha256": "2605f3a883464407189dc92a54ccab5816a03cbba6aa5054a06fc610a4556ea5", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3_sk_linear": { @@ -3161,7 +3161,7 @@ "bytes": 59076, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_sk_linear_N1H8W8C16_K32Y3X3_t64x64x16_w2x2_a32x32x16_mem_default_sk", - "sha256": "a18cb4d993ab0e837f5e4cbda76848fce46e76a8393900e755b3cdaed05235f8", + "sha256": "4d06d96b79d44148a72707f0bbe90506bb9fa5ca4f8c38e9aae2cb507ee5b735", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3_sk_tree": { @@ -3169,7 +3169,7 @@ "bytes": 60518, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_sk_tree_N1H8W8C16_K32Y3X3_t64x64x16_w2x2_a32x32x16_mem_default_sk_skrtree", - "sha256": "efe578b01d1d931ed9ee46a7974a5167cb5032222f20718d28c8e546046d577a", + "sha256": "28303862508b8fd94760c1ac4a584320c8dad9d11ca8c69bd68e09fb2d85058f", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3_sk_workspace_pers": { @@ -3177,7 +3177,7 @@ "bytes": 31460, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_sk_workspace_pers_N1H8W8C16_K32Y3X3_t64x64x16_w2x2_a32x32x16_mem_default_sk_skpers_skrworkspace", - "sha256": "f2b8b993253d99f9d3b796085d5ac5c6ef3d15ae453ec0bd0aee0d6397591c28", + "sha256": "c05cb50578c2c646afc464201e1d86f51e05054ea1f8b7eb6c6325c2734fa855", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3_spk4": { @@ -3185,7 +3185,7 @@ "bytes": 26284, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_spk4_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_cshuffle_spk", - "sha256": "83e4a27b2892d3322f771242a7eb6a0f8a5ede30efbee22f4195880c14ac8a5a", + "sha256": "da9cc8fd173e6bee98f0c3f9ea45d1b44283230d165ebedc25466431882a034f", "status": "ok" }, "conv_wgrad/gfx950/n1h8c64k64r3_g4": { @@ -3193,7 +3193,7 @@ "bytes": 30063, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_g4_N1H8W8C64_K64Y3X3G4_t64x64x32_w2x2_a32x32x16_mem_cshuffle", - "sha256": "247d66dbd22dea1001f55ca1f8082c03707062b84c5ab157a66abe677a81e333", + "sha256": "0d05c0caa1e4a92e0993d324795496fecb00f0545e3a0e91bf970c3ebbec3cf2", "status": "ok" }, "conv_wgrad/gfx950/n1h8c64k64r3_g4_cshuffle": { @@ -3201,7 +3201,7 @@ "bytes": 30342, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_g4_cshuffle_N1H8W8C64_K64Y3X3G4_t64x64x32_w2x2_a32x32x16_mem_cshuffle", - "sha256": "e24daa9ac08990da4b1d1ad4ccbeece698279dcda588cb28c785ee2d4e21cd38", + "sha256": "a5de3ed6566db66b683fce3b1fd3b17bbcbce02db21e96152fd660b82d980d90", "status": "ok" }, "conv_wgrad/gfx950/n1h8c64k64r3_g4_sk_linear_pers": { @@ -3209,7 +3209,7 @@ "bytes": 61597, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_g4_sk_linear_pers_N1H8W8C64_K64Y3X3G4_t64x64x16_w2x2_a32x32x16_mem_default_sk_skpers", - "sha256": "f978aa87c489a5989e562217a201cc15675aa1b8fd38a816846881a2697d55cb", + "sha256": "40c2f8ffd329a0b5e451de8caaf5e81a626d891b5439466cf7fef686fabd3725", "status": "ok" }, "conv_wgrad/gfx950/n1h8c64k64r3_g4_spk4": { @@ -3217,7 +3217,7 @@ "bytes": 38794, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_g4_spk4_N1H8W8C64_K64Y3X3G4_t64x64x32_w2x2_a32x32x16_mem_cshuffle_spk", - "sha256": "a1d0a8d71a336f4349d283db0235e11681c00af9d60aae80f6ddd30255f3c2b2", + "sha256": "baaf64e74b63f526c6981224bdcfcfae0b8b3e3dac755eac97977964943945a3", "status": "ok" }, "conv_wgrad/gfx950/n2h16c32k32r1": { @@ -3257,7 +3257,7 @@ "bytes": 22530, "family": "conv_wgrad_two_stage", "kernel_name": "irhash_wgrad_942_twostage_spk4_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_default_spk_twostage_wsr8", - "sha256": "8188f75754f950a6596857f24fd4d604ba70f7d93f963e61639585fcc0c04b42", + "sha256": "8f19fc6e3ee525fbec7909bc526883f13502bb1d6d7082bdc4bec1aa94409bd5", "status": "ok" }, "conv_wgrad_two_stage/gfx950/n1h8c16k32r3_spk4": { @@ -3265,7 +3265,7 @@ "bytes": 22530, "family": "conv_wgrad_two_stage", "kernel_name": "irhash_wgrad_950_twostage_spk4_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_default_spk_twostage_wsr8", - "sha256": "ba6a28520cba7ce7a5295016b0f8645ae0ef998574f396e5456e81206c04fc46", + "sha256": "56c89bada2d4a673fa446314f17cf522d6a2cd24e2e309c68a8546fae756bc13", "status": "ok" }, "crosslane/gfx1201/warp_shuffle_xor_dpp_reduce_i32": { @@ -3313,7 +3313,7 @@ "bytes": 54888, "family": "deep_fused_conv", "kernel_name": "irhash_deep_gfx950_wt16_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "dda5cf7edba50b6c18f473370dc3de5c9949024ea0315ab20a379a70f9d6d941", + "sha256": "e2a33dab8e1896fe2c749a0e1677d4e8fe5e934ed4356d03940676e569c6223e", "status": "ok" }, "deep/common_gfx950_wt16/h24w24_k32": { @@ -3321,7 +3321,7 @@ "bytes": 54937, "family": "deep_fused_conv", "kernel_name": "irhash_deep_gfx950_wt16b_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "a2d5205b298e2477b8c7684ad32e40734671210ed40e64171489062538fffdc9", + "sha256": "8c3513078def4905dc86e42a521de911c8722863fec86e21a88e207c328ae274", "status": "ok" }, "deep/gfx1151/native_h16w16": { @@ -3353,7 +3353,7 @@ "bytes": 74083, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "6b760b3a7608deb2267ff2a77daa33f8a6be5929040d75302cb07dff2aad5914", + "sha256": "acf72a0eba462e84e2317866f37d8b4a74667b5f8d3ceca92e60df6610fbd743", "status": "ok" }, "deep/gfx1201/h24w24_k32": { @@ -3361,7 +3361,7 @@ "bytes": 74084, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "05f72082ef9c67a1b041247e2306ebd21366b47e5422abd8d2e9933dd1bebeda", + "sha256": "43dc8adb89923691dbc3694700d1210eccb1081e06aad815a959d54d30c917b1", "status": "ok" }, "deep/gfx12_generic/h16w16_k32": { @@ -3369,7 +3369,7 @@ "bytes": 74083, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "6b760b3a7608deb2267ff2a77daa33f8a6be5929040d75302cb07dff2aad5914", + "sha256": "acf72a0eba462e84e2317866f37d8b4a74667b5f8d3ceca92e60df6610fbd743", "status": "ok" }, "deep/gfx950/h16w16_k32": { @@ -3377,7 +3377,7 @@ "bytes": 33733, "family": "deep_fused_conv", "kernel_name": "rocke_gfx950_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a32x32x16_mem_cshuffle_epi_relu", - "sha256": "26ec9c06b6f7955392a4215dbd652bc80086aafe922e096eebcf7b2acfd150b4", + "sha256": "0fb33bc08bdcb1c53bab30e590994287635d0fc06117a5a02dd00da9e98d5478", "status": "ok" }, "deep/gfx950/h24w24_k32": { @@ -3385,7 +3385,7 @@ "bytes": 33737, "family": "deep_fused_conv", "kernel_name": "rocke_gfx950_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a32x32x16_mem_cshuffle_epi_relu", - "sha256": "7f00891dec6d6bf94b10ca053b50796272457f3716dea62e1eac04db387714dc", + "sha256": "fd22d40a5a1a57b5ac718b6963a33483239f7923a9ae267f256b49e4db7072d8", "status": "ok" }, "fused_moe/gfx950/gather_h128": { @@ -3753,7 +3753,7 @@ "bytes": 2890, "family": "moe", "kernel_name": "irhash_moe_sort_scatter_scatter_T64_K2_E16_b128", - "sha256": "5a5f80209d788afa4a66e8cafe495e4d711115c79fabe6713b2ad3279489f0d1", + "sha256": "c66d3e71a8662b2bcb4e92951163fffb8f97e47fe25ab007bb5659e7c461f0f1", "status": "ok" }, "nontemporal/gfx942/copy_bf16x8": { @@ -4059,7 +4059,7 @@ "bytes": 46331, "family": "attention", "kernel_name": "rocke_unified_attention_2d_scalar_q64_h4_kv2_d128_b16_fp16_rbu", - "sha256": "9fab004c9363ef6579708232116d30838750141470dabf4082069ec57be108f4", + "sha256": "1439195339b0bf240984da6a0e706798211e6266b0effecd21c730e056221d71", "status": "ok" }, "attention/gfx950/scalar2d_fp16_d128_sw16_rb8": { @@ -4067,7 +4067,7 @@ "bytes": 46665, "family": "attention", "kernel_name": "rocke_unified_attention_2d_scalar_q64_h4_kv2_d128_b16_fp16_sw_rb8", - "sha256": "4a6cbf9f4a26c34329f72c51f6fd1eeb235bbc808abc5ae781bd42a98f300808", + "sha256": "06b90e3c577d968c02ac6171c6f0b7bda0d798b36c2059818111baa0b8994965", "status": "ok" }, "attention/gfx950/scalar2d_fp16_d128_top_left": { @@ -4075,7 +4075,7 @@ "bytes": 46435, "family": "attention", "kernel_name": "rocke_unified_attention_2d_scalar_q64_h4_kv2_d128_b16_fp16_tl", - "sha256": "f2658f2f978dcfdecb2bd639a502f0084756b3d8bc1a806f3c1eae29efc3845d", + "sha256": "076fcd4b4e1f618f5c75534cba8333098b03497e5393d157543433476bb3d158", "status": "ok" }, "attention/gfx950/scalar3d_fp16_d128_sw16_rb8": { @@ -4083,7 +4083,7 @@ "bytes": 49166, "family": "attention", "kernel_name": "rocke_unified_attention_3d_scalar_q64_h4_kv2_d128_b16_seg8_fp16_rb8_sw", - "sha256": "373addf7aa218e6610346c2be4fcc093c7c50d16cfd4cd37d8f2a16b598dd6fb", + "sha256": "0b5f59549c2dba463219d00aea1c016c547479fe1c5a3ea51daccdf3c8973ee1", "status": "ok" }, "attention/gfx950/scalar3d_fp16_d128_top_left": { @@ -4091,7 +4091,7 @@ "bytes": 48926, "family": "attention", "kernel_name": "rocke_unified_attention_3d_scalar_q64_h4_kv2_d128_b16_seg8_fp16_tl", - "sha256": "1a0b800bd292a966e1e0ba4a424cef2a5b60f8578c61117bc9228a5ad771492e", + "sha256": "1b55556afefa9291e98a2a07cea73a144f8f9be96fb1790e0d3d3ab2c690d7ad", "status": "ok" }, "attention_d128_swa/gfx942/4warp_gqa_fold": { @@ -4475,7 +4475,7 @@ "bytes": 30513, "family": "conv", "kernel_name": "irhash_conv_1151_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_cshuffle", - "sha256": "8dd49feced0002d4b6ef8453bb43062bc64ce1bc1059556b5147166f2d2c4452", + "sha256": "7e5e93ff71b54ffd408ea5a76f8e31a56d9c033cc4af238491681574cfcae008", "status": "ok" }, "conv/gfx1151/n2h16c32k32r1": { @@ -4491,7 +4491,7 @@ "bytes": 24142, "family": "conv", "kernel_name": "irhash_conv_1201_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_cshuffle", - "sha256": "e7b06ca0f2dd5d5594a25c4fdbdc7d79cf6350e737250c038a7125ce28cfcea6", + "sha256": "d94f658d1c5dc893acc5695ec58281b4d3d148ede0597402ede5c6a10e230e16", "status": "ok" }, "conv/gfx90a/n1h8c16k32r3": { @@ -4499,7 +4499,7 @@ "bytes": 23816, "family": "conv", "kernel_name": "irhash_conv_90a_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_cshuffle", - "sha256": "befc225eaabe178ea410d7e3c3ec80d04391d3995a54a5ee9ce83670a74946c0", + "sha256": "2b70b8aecd933fc43cc22878d09061f09547b7727683ca27000ce944f366221d", "status": "ok" }, "conv/gfx942/n1h8c16k32r3": { @@ -4507,7 +4507,7 @@ "bytes": 23816, "family": "conv", "kernel_name": "irhash_conv_942_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_cshuffle", - "sha256": "3fc87c28cb5df0c17a1d38cc1f45b512d00b22b3d5d84076f354ed4af977779c", + "sha256": "66688788978c70e80826a3a056f6ca5d40239fef4cef6abeb135792582d380d7", "status": "ok" }, "conv/gfx942/n2h14c64k64r3g64/dw_gm8_cshuffle": { @@ -4515,7 +4515,7 @@ "bytes": 25083, "family": "conv", "kernel_name": "irhash_conv_942_dw_gm8_N2H14W14C64_K64Y3X3G64_t64x32x16_w2x2_a16x16x16_mem_cshuffle_gm8", - "sha256": "6eb439d13a05f43a879f96eda6477df30ba3cc1ab19928857c163ce3d87e765a", + "sha256": "5bb8064acb504068d834f80a4ac0de93d24dc4e6a28128876321b7d51ad5c033", "status": "ok" }, "conv/gfx950/n1h8c16k32r3": { @@ -4523,7 +4523,7 @@ "bytes": 25148, "family": "conv", "kernel_name": "irhash_conv_950_a_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_compv4_cshuffle", - "sha256": "9aecee204488b91b9ab54b29d074d30be4a5f0aaccb31cbc9e89cb36fdd98800", + "sha256": "7482df1a6402f6e07267d1cb58ea622c445253ecf0390b61dceb75c7f3b08c14", "status": "ok" }, "conv/gfx950/n1h8c16k32r3/basic_cshuffle": { @@ -4531,7 +4531,7 @@ "bytes": 24828, "family": "conv", "kernel_name": "irhash_conv_950_basic_b_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_basic_cshuffle", - "sha256": "dda20c0d05117c490fe9a1ce36d1bafd5f32f95798475e0129741c919b108690", + "sha256": "c14e1bb010b29e4cf7f4d7bdbff2d389df5756f7eff258ccbff7eef7a2a75db0", "status": "ok" }, "conv/gfx950/n1h8c16k32r3/basic_default": { @@ -4539,7 +4539,7 @@ "bytes": 40048, "family": "conv", "kernel_name": "irhash_conv_950_basic_a_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_basic_default", - "sha256": "29e0bd92af052809bdfef9e7ffac2741038ee946d6c82ddce7739f8e1d5d3ffe", + "sha256": "1afbfb395dabdc61c02b2a5387fac5d9ec6b5d06f94b43fe468097b191fe12e4", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_gm32_cshuffle": { @@ -4547,7 +4547,7 @@ "bytes": 55421, "family": "conv", "kernel_name": "irhash_conv_950_dw_gm32_cshuffle_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_cshuffle_gm32", - "sha256": "c98d14e1839186e109c00e8fb32be2e929546122be8fc411940a4b75b8187325", + "sha256": "060f172ec60595345d18d84301e9d1f66dc25b011fcfad145654044f760bcb78", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_gm4_direct": { @@ -4555,7 +4555,7 @@ "bytes": 78070, "family": "conv", "kernel_name": "irhash_conv_950_dw_gm4_direct_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_default_gm4", - "sha256": "dfee79656d077dd2d5eb54ce377f5b97c88ea0e2eaab5338244e0faab9b27ea8", + "sha256": "1598ca38bc21f0f3be1c281e3517a6c1899b2f7bdf16d9692461cd3251aca45a", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_gm64_cshuffle": { @@ -4563,7 +4563,7 @@ "bytes": 55421, "family": "conv", "kernel_name": "irhash_conv_950_dw_gm64_cshuffle_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_cshuffle_gm64", - "sha256": "58138ad235344b905a809f2700c950b73884feffa38c784d94ad6f0ce755af84", + "sha256": "e76379c7420a89bac48f08cf0a50d7109225303fce1f56352374a0c38c4a1061", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_gm8_cshuffle": { @@ -4571,7 +4571,7 @@ "bytes": 55343, "family": "conv", "kernel_name": "irhash_conv_950_dw_gm8_cshuffle_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_cshuffle_gm8", - "sha256": "5e12e2ab3db2c264da2e84cd0de41befd8b24b7196af83b95cae76616093e42d", + "sha256": "a589c7af6abf5914b6a12167dc6d9d0b54f316b3984d754f962732e1e0d61f4c", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_unmerged": { @@ -4579,7 +4579,7 @@ "bytes": 129246, "family": "conv", "kernel_name": "irhash_conv_950_dw_unmerged_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_default", - "sha256": "f8f41cd628bb31c4b0b4dc7a4b7d090123dab477404689a6cba99f42e4c00ccc", + "sha256": "941703a5df4fe9933e50807ab6873e220a7f3d59b4089fb2d85b493b52c6b75b", "status": "ok" }, "conv/gfx950/n2h14c64k64r3g64/dw_unmerged_cshuffle": { @@ -4587,7 +4587,7 @@ "bytes": 143615, "family": "conv", "kernel_name": "irhash_conv_950_dw_unmerged_cshuffle_N2H14W14C64_K64Y3X3G64_t64x64x64_w2x2_a32x32x16_mem_cshuffle", - "sha256": "8da4d635d7a142be3e668903a4338f0b26049e1bcb045ec4b1b6e18f297ffe6d", + "sha256": "c293f5b43827b7dbf36202080b7879b7f8cf0e0ea1fcbf249fb2b9779b803a7c", "status": "ok" }, "conv/gfx950/n2h16c32k32r1": { @@ -4603,7 +4603,7 @@ "bytes": 56109, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_1151_s1_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "afc07dbcef5e972ca03a751508aaca3f9f10b921d257f79ced0861040b0e8729", + "sha256": "82e71d702e7ffcaf23e00914aee1c78e523a1c3e19642f66299e6fbbb48b3de3", "status": "ok" }, "conv_dgrad/gfx1151/n1h8c64k64r3_g4": { @@ -4611,7 +4611,7 @@ "bytes": 57018, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_1151_g4_s1_N1H8W8C64_K64Y3X3G4_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "3c63f4e79ac3e48ad82ddbe3d211eaea790a6f69ca44199583775935b975e2f8", + "sha256": "79624e9d3399f9a0f5f48472ff5522026b6526af9f9e74da1b14c9f3858af211", "status": "ok" }, "conv_dgrad/gfx1250/n1h8c64k64r3_g4": { @@ -4619,7 +4619,7 @@ "bytes": 65510, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_1250_g4_s1_N1H8W8C64_K64Y3X3G4_t32x32x32_w2x1_a16x16x32_mem_default", - "sha256": "b880e496fc5a624efb907767ab12e9c5e648cba6915d92246b67cd13091fe516", + "sha256": "dfe2eb706c1c1575035a662928a29af01da080c8c2f228ddff9af056c07c1f32", "status": "ok" }, "conv_dgrad/gfx90a/n1h8c16k32r3_s1": { @@ -4627,7 +4627,7 @@ "bytes": 75638, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_90a_s1_N1H8W8C16_K32Y3X3_t64x64x64_w2x2_a32x32x8_mem_default", - "sha256": "c4d4b3e119ec30ddf4baaeae89f9f4cef117b0b7898b57dc2b6681f4e90f5bdf", + "sha256": "35eeabda948da3be81c9005db67451ad0eec7f8e6870c33906bbdf6a1bc82c72", "status": "ok" }, "conv_dgrad/gfx942/n1h8c16k32r3_s1": { @@ -4635,7 +4635,7 @@ "bytes": 75638, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_942_s1_N1H8W8C16_K32Y3X3_t64x64x64_w2x2_a32x32x8_mem_default", - "sha256": "4aa0ca0d5a7629a4ccf3fe0a4bab7dc66461583782f3f8f397b789cfcbc04925", + "sha256": "0a07167818cebb8260a23ec2c8f2877bdea8c1e9bddb4ed02dc014287e2dc9b5", "status": "ok" }, "conv_dgrad/gfx942/n1h8c64k64r3_g4": { @@ -4643,7 +4643,7 @@ "bytes": 76710, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_942_g4_s1_N1H8W8C64_K64Y3X3G4_t64x64x64_w2x2_a32x32x8_mem_default", - "sha256": "456bcd7e85626aa4bc71ba9ee282ed688b3e3fb78c6938f60921508a167dfecd", + "sha256": "568b204dd8634414dfa106ea1c4cbeb5c77e5606ae44666116a78bd1c6a0976d", "status": "ok" }, "conv_dgrad/gfx942/n1h8c64k64r3_g4_s2": { @@ -4651,7 +4651,7 @@ "bytes": 65980, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_942_g4_s2_N1H8W8C64_K64Y3X3G4_t64x64x32_w2x2_a16x16x16_mem_default", - "sha256": "5b54c72804ffbf5b7f47e1c5385e21e2c03f9dd05d705fd6741a740aec23d981", + "sha256": "ee97f53028c8c8dba381786dcc9b3e2b25cdbcd3464f50857f0477bc0b285bdb", "status": "ok" }, "conv_dgrad/gfx942/n2h16c32k32r3_s2": { @@ -4659,7 +4659,7 @@ "bytes": 65037, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_942_s2_N2H16W16C32_K32Y3X3_t64x64x32_w2x2_a16x16x16_mem_default", - "sha256": "84ab525c16a90ec04716bdb0d6c0ee1e487670464f1eb0ed488e571950f2859d", + "sha256": "4009e9c42d29a95596acb7ad22dc9c4e0acefd92b076e64698e944432ece0c73", "status": "ok" }, "conv_dgrad/gfx950/n1h8c16k32r3_s1": { @@ -4667,7 +4667,7 @@ "bytes": 59076, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_950_s1_N1H8W8C16_K32Y3X3_t64x64x64_w2x2_a32x32x16_mem_default", - "sha256": "a482c289305c2b391005c48e84a61a3b4c8d8d058c046ae87f6d01fb7ba004a9", + "sha256": "22f460a1d1dc44abac341c6a0e79daf8558a86c6ad9ec8f78290805cccec0fb1", "status": "ok" }, "conv_dgrad/gfx950/n1h8c64k64r3_g4": { @@ -4675,7 +4675,7 @@ "bytes": 60144, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_950_g4_s1_N1H8W8C64_K64Y3X3G4_t64x64x64_w2x2_a32x32x16_mem_default", - "sha256": "c608d229ab25f25bbd7f79cd350b8de39cbe2b071d96e38d44b58e2287473e28", + "sha256": "19720368c5f1e71b6d7961066424ee647b5aafded042b1b4a08d78c7597c78e7", "status": "ok" }, "conv_dgrad/gfx950/n2h16c32k32r3_s2": { @@ -4683,7 +4683,7 @@ "bytes": 64766, "family": "conv_dgrad", "kernel_name": "irhash_dgrad_950_s2_N2H16W16C32_K32Y3X3_t64x64x64_w2x2_a32x32x16_mem_default", - "sha256": "ccc4718ce474d4950a22ab7bf00ca0552f76548f8db92ba7b57f2fea2ae2efee", + "sha256": "873d7313816660ef4b8b3eb3a3b6a370ca0d95b808d32e0c4a5b2408022d8491", "status": "ok" }, "conv_direct/gfx942/16c_n1h8_fold_false": { @@ -4691,7 +4691,7 @@ "bytes": 53625, "family": "conv_direct", "kernel_name": "irhash_direct16c_942_a_N1H8W8_g8_c16k16_bq16_bg4_db", - "sha256": "56dfbde907bafc4f436d7b7040e66bcaea119dd1e5a28738adac34aa5644ca9e", + "sha256": "6db586226d3385d5fee0f90b9846bd8656a6b7071de911487fc58459c8f25eb5", "status": "ok" }, "conv_direct/gfx942/32c_n2h8": { @@ -4699,7 +4699,7 @@ "bytes": 195566, "family": "conv_direct", "kernel_name": "irhash_direct32c_942_a_N2H8W8_g4_c32k32_bq32_bg4_db", - "sha256": "9912fa66bfe2545d41469609a789da934cb65f8eb210187561626fec0f21722c", + "sha256": "c955f86e525537399dd611bd8a665bcd54f7473da02105812d3e999150c375c1", "status": "ok" }, "conv_direct/gfx942/8c_n4h16": { @@ -4707,7 +4707,7 @@ "bytes": 37218, "family": "conv_direct", "kernel_name": "irhash_direct8c_942_a_N4H16W16_g8_c8k8_bq16_bg8_db", - "sha256": "201a25827148bb4e492e2e32a970013ff262d09a4a3fdfca497b206e19d91144", + "sha256": "045dbac1c884a291153c4fef54291c224d3aa19bfa635903a8f673c2f79d0a99", "status": "ok" }, "conv_direct/gfx942/dw_col_s1_fp16": { @@ -4731,7 +4731,7 @@ "bytes": 46980, "family": "conv_direct", "kernel_name": "irhash_direct16c_950_bg4_N32H8W8_g16_c16k16_bq16_bg4_db_k32", - "sha256": "d6a6c46e92a7fa599be9adb9cb39e01c4f71c86f6ed1fb052330a414eebc52db", + "sha256": "94b6312b4641e2b2b97682ee42a2edc3dd0d0b8584b9c77c425ca6e7e6ffdb47", "status": "ok" }, "conv_direct/gfx950/16c_n32h8_bg8_fold": { @@ -4739,7 +4739,7 @@ "bytes": 46993, "family": "conv_direct", "kernel_name": "irhash_direct16c_950_bg8_N32H8W8_g16_c16k16_bq16_bg8_db_k32", - "sha256": "5562052bf57e541a8fb5644fab63ae1039822b589581961e462f135bce38420b", + "sha256": "64c295c51a83bf4917913d3adff7c0ebe582222674619b77da7e1bd79e55461d", "status": "ok" }, "conv_direct/gfx950/32c_n2h8": { @@ -4747,7 +4747,7 @@ "bytes": 195566, "family": "conv_direct", "kernel_name": "irhash_direct32c_950_a_N2H8W8_g4_c32k32_bq32_bg4_db", - "sha256": "5c1e78662de3b7c75471fb27e753733fa59d52b0bea621c55d472b664e0945ae", + "sha256": "7e4ff9ec9f3e239b9bdd84677685242015ad75c9e94abb4c9f7865dfd2e474a9", "status": "ok" }, "conv_direct/gfx950/4c_n1h8_small": { @@ -4771,7 +4771,7 @@ "bytes": 37218, "family": "conv_direct", "kernel_name": "irhash_direct8c_950_a_N4H16W16_g8_c8k8_bq16_bg8_db", - "sha256": "dafaaa0cb8a06c30445b5e68d0358feff5baf7f4eca3b13d1753610528722935", + "sha256": "02d7dc7bad84cd0c9c5222e3a1f2f91ce7aa69c30e02c06bd5c29f31d391d04e", "status": "ok" }, "conv_direct/gfx950/dw_col_bw1_tail": { @@ -4979,7 +4979,7 @@ "bytes": 53781, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_1151_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "d6cf50ed9e9006d39e51677977530fb4c83c3b8a8271f3cca294d06643b58c7c", + "sha256": "252f3647dc6ee6b392c4e1cb687e361bf9bf25d7411e3ad31759ea9faeaa45d4", "status": "ok" }, "conv_wgrad/gfx1201/n1h8c16k32r3": { @@ -4987,7 +4987,7 @@ "bytes": 47356, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_1201_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "1584e04e874628bf5a576ac3be3cb50d8e3a65d37530e35276feb03c3b16e109", + "sha256": "9c0d6fe24be253ef58611d9337e993875db5e5f4c208b54294aff2b2084969b7", "status": "ok" }, "conv_wgrad/gfx1250/n1h8c64k64r3_g4": { @@ -4995,7 +4995,7 @@ "bytes": 66725, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_1250_g4_N1H8W8C64_K64Y3X3G4_t32x32x32_w2x1_a16x16x32_mem_default", - "sha256": "a313a50ed1b697584bc9a8ba36e094fe6dec6cd9cfdbed75cb3b9e2bfb8578d0", + "sha256": "fd54064e2d579970aa568b21c1ab2227ec1b14b728bf8d83bd48f1b31cc5c06d", "status": "ok" }, "conv_wgrad/gfx90a/n1h8c16k32r3": { @@ -5003,7 +5003,7 @@ "bytes": 22063, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_90a_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_cshuffle", - "sha256": "a96687d5c4d7b4349414331aecbdcb6018923a678c526da1fe3dc63069071463", + "sha256": "72626c63133204f5778134b029104d52e627696004c6df772d5c2682c237dcfe", "status": "ok" }, "conv_wgrad/gfx942/n1h8c16k32r3": { @@ -5011,7 +5011,7 @@ "bytes": 22063, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_cshuffle", - "sha256": "24c106cf812be8655f0e9ed1a272346ecccc29dd2e893342eada4b6dbe9dd227", + "sha256": "b873d1f4dafd73093fbc5bdc3ea54a13f9abacac2720298169b9953e08d44363", "status": "ok" }, "conv_wgrad/gfx942/n1h8c16k32r3_sk_atomic_pers": { @@ -5019,7 +5019,7 @@ "bytes": 25353, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_sk_atomic_pers_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_default_sk_skpers_skratomic", - "sha256": "2863b1a0e57941df6e773f9e981c40f1843856237554d7615e92731db6ff06c4", + "sha256": "820b4a37c70ef085abe3997ace59ce06517d10f1c74b9510211cf71c70cb30f8", "status": "ok" }, "conv_wgrad/gfx942/n1h8c64k64r3_g4": { @@ -5027,7 +5027,7 @@ "bytes": 22663, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_g4_N1H8W8C64_K64Y3X3G4_t64x32x16_w2x2_a16x16x16_mem_cshuffle", - "sha256": "6b7dca0251498621500e6c6978cc778ec3a4a49e3304631cd1ab1939ce06b6c4", + "sha256": "feb34316d53737cbc4c76d03c039ed6b85f646ed3eabb6ae9dff2815195456f9", "status": "ok" }, "conv_wgrad/gfx942/n1h8c64k64r3_g4_sk_tree_pers": { @@ -5035,7 +5035,7 @@ "bytes": 44599, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_g4_sk_tree_pers_N1H8W8C64_K64Y3X3G4_t64x32x16_w2x2_a16x16x16_mem_default_sk_skpers_skrtree", - "sha256": "2558f2ea4588e91e12e697b23a095805e6a83b20d9c01d9a7f78be69a7e688ce", + "sha256": "1322857aaacb2b0ce1b1cfef9476631403348a535e8bfec79b8a53e9aa7b62e2", "status": "ok" }, "conv_wgrad/gfx942/n1h8c64k64r3_g4_sk_workspace": { @@ -5043,7 +5043,7 @@ "bytes": 25995, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_g4_sk_workspace_N1H8W8C64_K64Y3X3G4_t64x32x16_w2x2_a16x16x16_mem_default_sk_skrworkspace", - "sha256": "7c6ce1e064d7d92ebcfb0110334c89e674a828bb65bd784d566e5e9b58129721", + "sha256": "ba8d9b23b3b325fe393537dea5aea5039e240d7a49be59e2146bc192fb9599a4", "status": "ok" }, "conv_wgrad/gfx942/n1h8c64k64r3_g4_spk4": { @@ -5051,7 +5051,7 @@ "bytes": 27099, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_942_g4_spk4_N1H8W8C64_K64Y3X3G4_t64x32x16_w2x2_a16x16x16_mem_cshuffle_spk", - "sha256": "806158ee5ea999f68626ff13e588dffc09d64058cef6fc4f5e235c63fc25c8f9", + "sha256": "004a3b6223e994093c87f281d71a3e166de55601d1f4f8854a9514371872ec14", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3": { @@ -5059,7 +5059,7 @@ "bytes": 29416, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_a_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_mem_cshuffle", - "sha256": "95493c1a74fc7fe192cbf74030f19bd0ecc57dfd28198ebdcb40eac381e134d5", + "sha256": "44a919c13e749fb18f6fbc2e32c837a1784554d4bb88bebdfbf47806f9b4cb43", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3_sk_atomic": { @@ -5067,7 +5067,7 @@ "bytes": 30012, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_sk_atomic_N1H8W8C16_K32Y3X3_t64x64x16_w2x2_a32x32x16_mem_default_sk_skratomic", - "sha256": "9fc28049b1eee41c32d58d6fe3fc2e4aa218e367d3ef90acaa97790dcb4d18a3", + "sha256": "423d0f0f854a7aa8c961b1dde808069af77f5c8785414a34361559b1df87eded", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3_sk_linear": { @@ -5075,7 +5075,7 @@ "bytes": 59140, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_sk_linear_N1H8W8C16_K32Y3X3_t64x64x16_w2x2_a32x32x16_mem_default_sk", - "sha256": "65640f53273c121fa0e1127e49cce824a2e7e4f7ec8a6ccb53725570d419b2e9", + "sha256": "832b41581679ed15ffca842ea3d613945882ff1ebbc93b6d96478d56584ab672", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3_sk_tree": { @@ -5083,7 +5083,7 @@ "bytes": 60582, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_sk_tree_N1H8W8C16_K32Y3X3_t64x64x16_w2x2_a32x32x16_mem_default_sk_skrtree", - "sha256": "e50b150b37785afabbc8c8f136c069bdd88ffbbff332d14dbe0f9db82b88c975", + "sha256": "990ef0e8e9e4c57e20ee1f8c1e2a8dcaece332b71a1a01051ac11c789330c499", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3_sk_workspace_pers": { @@ -5091,7 +5091,7 @@ "bytes": 31524, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_sk_workspace_pers_N1H8W8C16_K32Y3X3_t64x64x16_w2x2_a32x32x16_mem_default_sk_skpers_skrworkspace", - "sha256": "3dbc6ebcdf5be7498ae4f73eeb9b99932a9cea7fe80df7077e4d629959f137dc", + "sha256": "a5bd9436143311631f1321a6489bbb2bbb1b236cf520616377c421a906346c92", "status": "ok" }, "conv_wgrad/gfx950/n1h8c16k32r3_spk4": { @@ -5099,7 +5099,7 @@ "bytes": 26348, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_spk4_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_cshuffle_spk", - "sha256": "5016a3d28409565c1751bc9d2b9dc9559d1d7a59156ea6fd9ce4c12314618715", + "sha256": "c090de9315a4d8a019dc6fabb172961345779dc8cc4b9c8cba1d040d99c5ac2f", "status": "ok" }, "conv_wgrad/gfx950/n1h8c64k64r3_g4": { @@ -5107,7 +5107,7 @@ "bytes": 30127, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_g4_N1H8W8C64_K64Y3X3G4_t64x64x32_w2x2_a32x32x16_mem_cshuffle", - "sha256": "7eb4c4cb0225eeffeab3421ff4e0d6606515bb2a39fd7ef1bd4dfa3097546780", + "sha256": "e8ca8f6d48c7d77450f6d8ec06daec86ceffd7daab599536b2d51bb16f923cc4", "status": "ok" }, "conv_wgrad/gfx950/n1h8c64k64r3_g4_cshuffle": { @@ -5115,7 +5115,7 @@ "bytes": 30406, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_g4_cshuffle_N1H8W8C64_K64Y3X3G4_t64x64x32_w2x2_a32x32x16_mem_cshuffle", - "sha256": "822ab7ae23cca12ac3351b10bec5ee0cc5a18945c210f3f39e1c8f38b76252d0", + "sha256": "8d14bfe727fb9fad0779de296db999c8eef08c6e418c81ca6eaff25af8b6b8a7", "status": "ok" }, "conv_wgrad/gfx950/n1h8c64k64r3_g4_sk_linear_pers": { @@ -5123,7 +5123,7 @@ "bytes": 61661, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_g4_sk_linear_pers_N1H8W8C64_K64Y3X3G4_t64x64x16_w2x2_a32x32x16_mem_default_sk_skpers", - "sha256": "1b892d27711fae5606fa0bf1eca307a105ab3a9db98a9860d46e72db11937992", + "sha256": "853f5a2dd1df73c50da645b336d4d7a61d5b12e410fbd6d2439da279e708fe68", "status": "ok" }, "conv_wgrad/gfx950/n1h8c64k64r3_g4_spk4": { @@ -5131,7 +5131,7 @@ "bytes": 38858, "family": "conv_wgrad", "kernel_name": "irhash_wgrad_950_g4_spk4_N1H8W8C64_K64Y3X3G4_t64x64x32_w2x2_a32x32x16_mem_cshuffle_spk", - "sha256": "aa51b3ca7325f96c10473790c38da76f61520db4a3dc0d30e5d1ba74c1da822f", + "sha256": "81bbd79a8447cf36bb163a127d12aede645368914b75f2e289ad5af3a9476c41", "status": "ok" }, "conv_wgrad/gfx950/n2h16c32k32r1": { @@ -5171,7 +5171,7 @@ "bytes": 22594, "family": "conv_wgrad_two_stage", "kernel_name": "irhash_wgrad_942_twostage_spk4_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_default_spk_twostage_wsr8", - "sha256": "2f4f7e7b9f650077df6aa6dd114f49e31652433c982ab8d06326ad0fa9edb55a", + "sha256": "02752d5efe8ba7d3147cdb63d6a1452c85292e910f90be6f9bfe0876acbe273d", "status": "ok" }, "conv_wgrad_two_stage/gfx950/n1h8c16k32r3_spk4": { @@ -5179,7 +5179,7 @@ "bytes": 22594, "family": "conv_wgrad_two_stage", "kernel_name": "irhash_wgrad_950_twostage_spk4_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_default_spk_twostage_wsr8", - "sha256": "fbf1a2b7546a2c19465b79b7b1446012e23dfae365e5e2886c80618e94bf6c0f", + "sha256": "e28e5f36d68882c1bb44ecdeb5661e28c01ca196667528ed2d45ba29485f430c", "status": "ok" }, "crosslane/gfx1201/warp_shuffle_xor_dpp_reduce_i32": { @@ -5227,7 +5227,7 @@ "bytes": 54952, "family": "deep_fused_conv", "kernel_name": "irhash_deep_gfx950_wt16_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "e377236ffe61752e91e794bb9164ed0320e02401e678e27989132ba3c0b6a2b2", + "sha256": "895ba0a243f02524fa875213adc2a35159244334280959dca77c6bced25fd2ad", "status": "ok" }, "deep/common_gfx950_wt16/h24w24_k32": { @@ -5235,7 +5235,7 @@ "bytes": 55001, "family": "deep_fused_conv", "kernel_name": "irhash_deep_gfx950_wt16b_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "df80581d0b7c25ebb3a8a9bcf0da0b9f679fbf140b89d274a8f9dbdc2364c31a", + "sha256": "00f8a6a84dc8a74b77cf011c8bcecb93231db6a43c6b4a9838d6f450c18f9ac2", "status": "ok" }, "deep/gfx1151/native_h16w16": { @@ -5267,7 +5267,7 @@ "bytes": 74147, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "f56c31e8e7b5f54443c42d3de3318ec7bdccb4e7bc11f7058ef6dd2a4cac97a6", + "sha256": "beee81ce04ce9a7d4e89295b0c103f54037bd1f3e0ff4615cff61f9f3ab1d040", "status": "ok" }, "deep/gfx1201/h24w24_k32": { @@ -5275,7 +5275,7 @@ "bytes": 74148, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "0e86fb325744a215675a2f6052a537babc86749e8cdff3d70aa058f787b819f3", + "sha256": "4b5fc75c0809c625695608579e550f51e39f9c2457e17cb5a80815c8b2ead9d7", "status": "ok" }, "deep/gfx12_generic/h16w16_k32": { @@ -5283,7 +5283,7 @@ "bytes": 74147, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "f56c31e8e7b5f54443c42d3de3318ec7bdccb4e7bc11f7058ef6dd2a4cac97a6", + "sha256": "beee81ce04ce9a7d4e89295b0c103f54037bd1f3e0ff4615cff61f9f3ab1d040", "status": "ok" }, "deep/gfx950/h16w16_k32": { @@ -5291,7 +5291,7 @@ "bytes": 33797, "family": "deep_fused_conv", "kernel_name": "rocke_gfx950_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a32x32x16_mem_cshuffle_epi_relu", - "sha256": "a6bee49403622c94fea9e0381d958941a95306175e6a423d5e09acd2e2b19dda", + "sha256": "a8de6ef9f43590910d02f49c327e9be8f4ede6995d13dd652d8667f18d0ec4b0", "status": "ok" }, "deep/gfx950/h24w24_k32": { @@ -5299,7 +5299,7 @@ "bytes": 33801, "family": "deep_fused_conv", "kernel_name": "rocke_gfx950_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a32x32x16_mem_cshuffle_epi_relu", - "sha256": "ef24301f31eca300c6d6e27889cdb9fd76ee92a73c0899875768a1506b9e82bb", + "sha256": "dfeba2fa3baac6599a12352cc46fab56e691a190dc313f3778846d4580c89880", "status": "ok" }, "fused_moe/gfx950/gather_h128": { @@ -5699,7 +5699,7 @@ "bytes": 2954, "family": "moe", "kernel_name": "irhash_moe_sort_scatter_scatter_T64_K2_E16_b128", - "sha256": "176ae1d4c90b56ec669917bf0466d15994566ecbb9eb3a6392c6c42d85932785", + "sha256": "afcd65e526827b62538fe963e47f0cf11908ad079e3a810fe2d7098767040824", "status": "ok" }, "nontemporal/gfx942/copy_bf16x8": { From 52e5c5342a59ec0245778886bc11ba50812d734d Mon Sep 17 00:00:00 2001 From: Chris Millette Date: Thu, 15 Oct 2026 21:15:57 -0400 Subject: [PATCH 3/4] refactor(rocke): move magic-number division to its own module calculate_magic_numbers, do_magic_division and do_magic_division_dynamic move verbatim from helpers/transforms.py to the new helpers/magic_division.py. Magic-number division is a general integer primitive, not part of the coordinate-transform DAG; the DAG's magic unmerge transforms now call it through the new module. Every importer imports from rocke.helpers.magic_division: the conv, conv dgrad, conv wgrad, conv argument, unified attention, dense attention, split-KV decode, sage and sparse attention kernels, the fused MoE GEMM instance, and their tests. helpers/transforms.py no longer exports or re-exports the three names, so an old import fails instead of silently resolving. Comments and docs that named the old location now name the new one, including the C++ helper header and source, the recipe VM, the portable IR docs and the file index. No emitted IR changes. Co-Authored-By: Claude --- .../common/_conv_implicit_gemm_common.py | 2 +- .../kernels/common/attention_unified.py | 5 +- .../rocke/library/kernels/common/conv_args.py | 2 +- .../common/conv_implicit_gemm_dgrad.py | 6 +- .../common/conv_implicit_gemm_wgrad.py | 2 +- .../kernels/common/fmha_splitkv_decode.py | 2 +- .../library/kernels/common/sage_attention.py | 2 +- .../kernels/common/sparse_attention.py | 2 +- .../library/kernels/gfx942/attention_dense.py | 4 +- .../attention/test_gfx942_dense_wiring.py | 2 +- .../library/tests/test_attention_builds.py | 2 +- .../tests/test_attention_dense_gfx942.py | 4 +- .../rocke/platform/cpp/helpers/transforms.cpp | 7 +- .../rocke/helper_rocke.helpers.transforms.h | 8 +- .../platform/cpp/portable_ir/recipe_vm.cpp | 2 +- .../portable_ir_production_readiness.md | 2 +- .../architecture/portable_ir_schema.md | 2 +- .../platform/dsl_docs/reference/file_index.md | 1 + .../python/rocke/helpers/magic_division.py | 106 ++++++++++++++++++ .../python/rocke/helpers/transforms.py | 98 ++-------------- .../rocke/instances/common/moe_gemm_fused.py | 2 +- .../portable_ir/portable_ir_scaling_plan.md | 2 +- .../rocke/portable_ir/tests/test_roll_nd.py | 4 +- .../rocke/portable_ir/utils/recipe_expand.py | 2 +- .../tests/core/test_magic_division_sum.py | 2 +- .../tests/core/test_transforms_dynamic.py | 8 +- .../tests/portable_ir/test_recipe_roller.py | 2 +- 27 files changed, 159 insertions(+), 124 deletions(-) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/magic_division.py diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/_conv_implicit_gemm_common.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/_conv_implicit_gemm_common.py index 894156d4eaa5..3b10319c1cd4 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/_conv_implicit_gemm_common.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/_conv_implicit_gemm_common.py @@ -39,11 +39,11 @@ Type, Value, ) +from rocke.helpers.magic_division import do_magic_division_dynamic from rocke.helpers.spec import choose_load_vec from rocke.helpers.transforms import ( TensorDescriptor, DynamicTensorDescriptor, - do_magic_division_dynamic, embed, embed_dynamic, pad, diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_unified.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_unified.py index d838808b86e4..59dbaac2ca47 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_unified.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_unified.py @@ -53,10 +53,9 @@ use_2d_kernel, ) +from rocke.helpers.magic_division import calculate_magic_numbers, do_magic_division from rocke.helpers.transforms import ( TensorDescriptor, - calculate_magic_numbers, - do_magic_division, ) # Alias the promoted log2-domain softcap helper under the local name used by @@ -244,7 +243,7 @@ def _magic_div(b: IRBuilder, dividend: Value, divisor: int) -> Value: (a grid axis or a KV position), well inside the documented 31-bit unsigned range of the magic sequence -- so the unsigned magic quotient equals the signed ``b.div`` it replaces. This is the device lowering of - ``merge_v2_magic_division`` (``transforms.do_magic_division``). + ``merge_v2_magic_division`` (``magic_division.do_magic_division``). """ mult, shift = calculate_magic_numbers(divisor) return do_magic_division(b, dividend, mult, shift) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/conv_args.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/conv_args.py index 3d30ea686da6..c1b2e180b971 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/conv_args.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/conv_args.py @@ -32,8 +32,8 @@ conv_direct_arg_names, ) from rocke.helpers.spec import ceil_div +from rocke.helpers.magic_division import calculate_magic_numbers from rocke.helpers.streamk import StreamKIterPartition, streamk_iter_partition -from rocke.helpers.transforms import calculate_magic_numbers __all__ = [ "ConvArgs", diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/conv_implicit_gemm_dgrad.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/conv_implicit_gemm_dgrad.py index 48150bd67c35..2c86f0fe4212 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/conv_implicit_gemm_dgrad.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/conv_implicit_gemm_dgrad.py @@ -69,6 +69,10 @@ from rocke.helpers.geometry import WarpGrid from rocke.helpers.layouts import ConvKOuterFragmentReader, LdsLayout from rocke.helpers.loads import CoalescedTileLoader +from rocke.helpers.magic_division import ( + calculate_magic_numbers, + do_magic_division_dynamic, +) from rocke.helpers.mfma_gemm_inner import decode_mfma_lanes from rocke.helpers.schedule import SchedulePolicy from rocke.helpers.spec import ceil_div, kernel_name_join @@ -76,8 +80,6 @@ from rocke.helpers.transforms import ( DynamicTensorDescriptor, TensorDescriptor, - calculate_magic_numbers, - do_magic_division_dynamic, embed, pad, unmerge_magic, diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/conv_implicit_gemm_wgrad.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/conv_implicit_gemm_wgrad.py index 86c70b05ae3c..086af0ad306c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/conv_implicit_gemm_wgrad.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/conv_implicit_gemm_wgrad.py @@ -2501,7 +2501,7 @@ def _free_axis_vec(chan: int, dtype: str, override: "int | None" = None) -> int: # so that run is exactly Gm, a build-time constant: fold it and its # magic pair here instead of reading the kernargs. The epilogue # keeps the true cpg through ``params``. - from rocke.helpers.transforms import calculate_magic_numbers + from rocke.helpers.magic_division import calculate_magic_numbers _merged_cpg = p_load.cpg _mult, _shift = calculate_magic_numbers(_merged_cpg) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/fmha_splitkv_decode.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/fmha_splitkv_decode.py index 4be655fcd311..5d59932cdfcb 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/fmha_splitkv_decode.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/fmha_splitkv_decode.py @@ -40,8 +40,8 @@ store_scalar_from_f32, store_vec, ) +from rocke.helpers.magic_division import calculate_magic_numbers, do_magic_division from rocke.helpers.spec import kernel_name_join -from rocke.helpers.transforms import calculate_magic_numbers, do_magic_division from ._fmha_common import FmhaCommonSpec, FmhaKernelBuilder, validate_common_spec from ._fmha_warp_body import WARP_SIZE from .fmha_arch import validate_fmha_mfma_atom diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/sage_attention.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/sage_attention.py index c04dde0e2dbc..3461f1d31925 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/sage_attention.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/sage_attention.py @@ -67,6 +67,7 @@ ) from rocke.helpers.i4_dequant import unpack_i4_byte_to_pair_i32 from rocke.helpers.io import io_ir_type, load_scalar_as_f32 +from rocke.helpers.magic_division import calculate_magic_numbers, do_magic_division from rocke.helpers.mfma_attention import ( MFMA_ATTN_BLOCK_K, MFMA_ATTN_BLOCK_M, @@ -79,7 +80,6 @@ load_q_scale_for_block, ) from rocke.helpers.spec import kernel_name_join -from rocke.helpers.transforms import calculate_magic_numbers, do_magic_division from ._fmha_common import FmhaCommonSpec, FmhaKernelBuilder, validate_common_spec from ._fmha_warp_body import WARP_SIZE, fmha_warp_fwd_inner_body from .fmha_arch import validate_fmha_mfma_atom diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/sparse_attention.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/sparse_attention.py index 8cda42281c06..05b7e7a6eb7d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/sparse_attention.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/sparse_attention.py @@ -46,13 +46,13 @@ from typing import Tuple from rocke.core.ir import I8, IRBuilder, KernelDef, Value +from rocke.helpers.magic_division import calculate_magic_numbers, do_magic_division from rocke.helpers.mfma_attention import ( MFMA_ATTN_BLOCK_K, MFMA_ATTN_BLOCK_M, mfma_attention_fwd_inner_body, ) from rocke.helpers.spec import kernel_name_join -from rocke.helpers.transforms import calculate_magic_numbers, do_magic_division from ._fmha_common import FmhaCommonSpec, FmhaKernelBuilder, validate_common_spec from .fmha_arch import validate_fmha_mfma_atom diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_dense.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_dense.py index 49a857dee55d..266360cb8555 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_dense.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_dense.py @@ -2345,7 +2345,7 @@ def attention_dense_signature(spec: AttentionDenseSpec): def _fastdiv_kernargs(spec: Gfx942AttentionDenseSpec) -> dict[str, int]: """``{_magic, _shift}`` for every ``spec.fastdiv_divisors`` - entry, from :func:`rocke.helpers.transforms.calculate_magic_numbers`. The magic + entry, from :func:`rocke.helpers.magic_division.calculate_magic_numbers`. The magic is a uint32 passed as its two's-complement i32 bit pattern, which is what the kernel's unsigned mul-hi reads. @@ -2353,7 +2353,7 @@ def _fastdiv_kernargs(spec: Gfx942AttentionDenseSpec) -> dict[str, int]: decode dividend is below it, and past it the quotient is wrong with no fault.""" if not spec.fastdiv_divisors: return {} - from rocke.helpers.transforms import calculate_magic_numbers + from rocke.helpers.magic_division import calculate_magic_numbers nqb = spec.seqlen_q // spec.block_m work = nqb * spec.num_query_heads * spec.batch diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/attention/test_gfx942_dense_wiring.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/attention/test_gfx942_dense_wiring.py index 029adeb02f14..4ff01fda55fb 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/attention/test_gfx942_dense_wiring.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/attention/test_gfx942_dense_wiring.py @@ -46,7 +46,7 @@ build_attention_dense, supports_attention_dense, ) -from rocke.helpers.transforms import calculate_magic_numbers +from rocke.helpers.magic_division import calculate_magic_numbers _NAME = "attention_gfx942_dense" _SPEC_ID = "gfx942_dense" diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_builds.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_builds.py index ab2a84d1363d..290a5b81d912 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_builds.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_builds.py @@ -4220,7 +4220,7 @@ def test_persistent_kernargs_append_fastdiv_pairs_per_decode(self): divisor of the resolved decode, in the built kernel's param order; the launcher's values are ``calculate_magic_numbers`` of the divisors, packed as two's-complement i32, and divide every work-item index exactly.""" - from rocke.helpers.transforms import calculate_magic_numbers + from rocke.helpers.magic_division import calculate_magic_numbers from kernels.gfx942.attention_dense import ( attention_dense_runtime_args, attention_dense_signature, diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_dense_gfx942.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_dense_gfx942.py index a055bdfaec77..df84d57fc51e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_dense_gfx942.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_dense_gfx942.py @@ -2108,13 +2108,13 @@ def test_persistent_kernarg_abi_carries_the_fastdiv_pairs(decode): GPU only. The values are checked two ways: equal to - :func:`rocke.helpers.transforms.calculate_magic_numbers` for the divisor (magic + :func:`rocke.helpers.magic_division.calculate_magic_numbers` for the divisor (magic packed as its two's-complement i32), and -- through the host model of the in-kernel divide -- exact for dividends up to just under the 2**30 bound. Non-pow2 divisors throughout (B=3, NQB=13, Hq=40, gqa=5), so the magic is not the trivial pow2 one. """ - from rocke.helpers.transforms import calculate_magic_numbers + from rocke.helpers.magic_division import calculate_magic_numbers spec = _persistent_spec( batch=3, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/transforms.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/transforms.cpp index fcc1409c6ee3..b960a6ff4706 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/transforms.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/transforms.cpp @@ -2,10 +2,11 @@ // SPDX-License-Identifier: MIT /* * helper_rocke.helpers.transforms.c -- C99 port of a SUBSET of - * rocke.helpers.transforms (the CK Tile coordinate-transform DAG). + * rocke.helpers.transforms (the CK Tile coordinate-transform DAG) and of + * rocke.helpers.magic_division. * * Ported symbols (see the header for the exact list): calculate_magic_numbers, - * do_magic_division, CoordVar, Embed/PassThrough/Unmerge/UnmergeMagicDiv + * do_magic_division (both from magic_division), CoordVar, Embed/PassThrough/Unmerge/UnmergeMagicDiv * transforms + their constructors (embed/pass_through/unmerge/unmerge_magic), * and TensorDescriptor.{naive,transform,offset,unmerge_lower}. * @@ -791,7 +792,7 @@ static bool rocke_i_apply_pad_dynamic(rocke_ir_builder_t* b, } /* do_magic_division_dynamic: same as do_magic_division but mult/shift are SSA. - * Mirrors transforms.do_magic_division_dynamic. */ + * Mirrors magic_division.do_magic_division_dynamic. */ static rocke_value_t* rocke_i_do_magic_division_dynamic(rocke_ir_builder_t* b, rocke_value_t* dividend, rocke_value_t* mult_v, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.transforms.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.transforms.h index e717c8e3cae7..4798299c65e4 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.transforms.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.transforms.h @@ -2,11 +2,13 @@ * SPDX-License-Identifier: MIT * * rocke/helper_rocke.helpers.transforms.h -- C99 port of a SUBSET of - * rocke/helpers/transforms.py: the CK Tile-style coordinate-transform DAG. + * rocke/helpers/transforms.py: the CK Tile-style coordinate-transform DAG, + * plus the magic-number division of rocke/helpers/magic_division.py that the + * DAG's magic unmerge emits. * * SCOPE OF THIS PORT (this phase) -- exactly these Python symbols: - * calculate_magic_numbers rocke_calculate_magic_numbers - * do_magic_division rocke_do_magic_division + * magic_division.calculate_magic_numbers rocke_calculate_magic_numbers + * magic_division.do_magic_division rocke_do_magic_division * CoordVar rocke_coord_var_t * Embed (transform) rocke_embed() / ROCKE_XFORM_EMBED * PassThrough (via pass_through) rocke_pass_through() / ROCKE_XFORM_PASS_THROUGH diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/portable_ir/recipe_vm.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/portable_ir/recipe_vm.cpp index 27bed96970db..1c7af339daab 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/portable_ir/recipe_vm.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/portable_ir/recipe_vm.cpp @@ -349,7 +349,7 @@ static long rv_int(rvm_t* vm, const jd_val_t* e) the multiplier depends on its odd part, so neither is expressible as arithmetic on the axis -- the recipe has to regenerate them. Mirrors recipe_expand.py::magic_division_constants, and upstream of - both, helpers/transforms.py::calculate_magic_numbers. */ + both, helpers/magic_division.py::calculate_magic_numbers. */ const jd_val_t* mm = rocke_jget(e, "magic_multiplier"); const jd_val_t* ms = rocke_jget(e, "magic_shift"); if(mm || ms) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/portable_ir_production_readiness.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/portable_ir_production_readiness.md index 41104c2facc1..2b0d211362ef 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/portable_ir_production_readiness.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/portable_ir_production_readiness.md @@ -404,7 +404,7 @@ an instructive way. integers that move with `C` are plainly affine, *including* the spatial products (`9 = Y*X`, `3136 = Hi*Wi`). The refusal comes from the two operands of a strength-reduced `n // C` — `(umul_hi(n, M) + n) >> s` from -`helpers/transforms.py::do_magic_division` — where `s = ceil(log2 C)` and `M` +`helpers/magic_division.py::do_magic_division` — where `s = ceil(log2 C)` and `M` depends on `C`'s odd part (`1` for powers of two, `0x55555556` for 3, `0x9999999A` for 5, `0x24924925` for 7). One constant is logarithmic, the other number-theoretic; no polynomial fits either. The tractable fix is not a richer diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/portable_ir_schema.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/portable_ir_schema.md index 65660539b591..ec906bdea3fe 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/portable_ir_schema.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/architecture/portable_ir_schema.md @@ -170,7 +170,7 @@ take the operand directly. `magic_multiplier` / `magic_shift` return the two operands of a strength-reduced unsigned division — `n // d` compiled as `(umul_hi(n, M) + n) >> s`, which is what -`helpers/transforms.py::do_magic_division` emits. They exist because those two +`helpers/magic_division.py::do_magic_division` emits. They exist because those two integers are *not* arithmetic on the divisor: the shift is `ceil(log2 d)` and the multiplier depends on `d`'s odd part, so a recipe that stays parametric in a divisor has to regenerate them rather than carry a formula. The divisor must diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/file_index.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/file_index.md index a9bb8067a22f..4c80e3eb7aa6 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/file_index.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/file_index.md @@ -47,6 +47,7 @@ A by-file map of the `rocke` package. Symbols listed are the primary contents (p |---------------------------------------|-------------------------------------------------------------------------------------------------| | `helpers/__init__.py` | Re-exports for the helper layer. | | `helpers/transforms.py` | Coordinate-transform DAG: `CoordVar`, `Transform`, `PassThrough`, `Pad`, `PadDynamic`, `Embed`, `Merge`, `Unmerge`, `Indirect`, `TensorDescriptor`, constructors (`pass_through`, `pad`, `pad_dynamic`, `embed`, `merge`, `unmerge`, `indirect`). Re-exported at the top level (`from rocke import ...`) and via `from rocke.helpers.transforms import ...`. | +| `helpers/magic_division.py` | `calculate_magic_numbers`, `do_magic_division`, `do_magic_division_dynamic`. Unsigned integer divide as mul-hi, add, shift with a host-computed `(multiplier, shift)` pair (CK Tile `magic_div.hpp`). Import via `from rocke.helpers.magic_division import ...`. | | `helpers/atoms.py` | `MfmaAtom`, `WmmaAtom`, `mfma_atom(dtype, m, n, k)`, `wmma_atom(dtype, m, n, k)`. MFMA catalogs (wave64): `MFMA_F16_ATOMS`, `MFMA_BF16_ATOMS`, `MFMA_FP8_ATOMS` (fp8 / bf8, incl. the wide-K `fp8_16x16x128` hero), `MFMA_MX_ATOMS` (fp4 / fp6), and the unified `MFMA_ATOMS`. WMMA catalogs (RDNA wave32): `WMMA_F16_ATOMS`, `WMMA_BF16_ATOMS`, `WMMA_ATOMS`. | | `helpers/geometry.py` | `WarpGrid` — block/warp/lane decomposition. | | `helpers/loads.py` | `CoalescedTileLoader`, `AsyncTileLoader`, `AsyncTileLoaderSlot`, `DescriptorFn`, `lane_contiguous_descriptor`. | diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/magic_division.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/magic_division.py new file mode 100644 index 000000000000..122f60df753b --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/magic_division.py @@ -0,0 +1,106 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""Magic-number division: an unsigned integer divide as mul-hi, add, shift. + +The AMDGPU backend has no integer-division instruction, so a divide by a +divisor known on the host (at build time or at launch) is emitted as +``(umul_hi(n, multiplier) + n) >> shift`` with a ``(multiplier, shift)`` pair +computed on the host. This mirrors CK Tile's +``ck_tile/core/utility/magic_div.hpp``. + +- :func:`calculate_magic_numbers` computes the pair on the host. +- :func:`do_magic_division` emits the sequence with the pair baked in as + constants. +- :func:`do_magic_division_dynamic` emits it with the pair as runtime values + (for example kernel arguments the launcher fills from + :func:`calculate_magic_numbers`). + +The coordinate-transform DAG in ``rocke.helpers.transforms`` is one caller; +kernels that split a flat index call it directly. +""" + +from __future__ import annotations + +from typing import Tuple + +from ..core.ir import IRBuilder, Value + + +def calculate_magic_numbers(divisor: int) -> Tuple[int, int]: + """Compute the ``(multiplier, shift)`` pair for unsigned magic division. + + Direct port of CK Tile's + ``magic_division32_bit_range::calculate_magic_numbers`` (the + ``magic_division`` alias used by ``merge_v2_magic_division``). The + returned constants reproduce ``dividend // divisor`` via the mul-hi + sequence in :func:`do_magic_division` for any ``dividend`` that fits + in the 31-bit unsigned range (the same restriction CK Tile documents). + + The math: + + .. code-block:: text + + shift = ceil(log2(divisor)) (smallest s with 2**s >= divisor) + multiplier = ((2**shift - divisor) << 32) // divisor + 1 + + Both values are plain Python ``int`` (compile-time constants); they are + baked into the emitted IR as ``const_i32`` operands, exactly like the + C++ constexpr path. + """ + if divisor < 1: + raise ValueError(f"magic division requires divisor >= 1, got {divisor}") + shift = 0 + while (1 << shift) < divisor: + shift += 1 + multiplier = (((1 << shift) - divisor) << 32) // divisor + 1 + return multiplier, shift + + +def do_magic_division( + b: IRBuilder, dividend: Value, multiplier: int, shift: int +) -> Value: + """Emit ``dividend // divisor`` using the magic ``(multiplier, shift)``. + + Mirrors CK Tile's ``do_magic_division`` device path + (``tmp = __umulhi(dividend, multiplier); (tmp + dividend) >> shift``), + which the AMDGPU backend lowers without an integer-division instruction. + ``dividend`` is an i32 SSA value interpreted as unsigned; the result is + the i32 quotient. ``multiplier`` / ``shift`` come from + :func:`calculate_magic_numbers`. + + The sum ``tmp + dividend`` is emitted as a no-unsigned-wrap add: for a + dividend near 2**31 it passes the signed maximum (from 1,610,612,736 for + divisor 3) while staying below 2**32, so a no-signed-wrap add would make + the quotient poison inside the documented range. + """ + # ``multiplier`` is a uint32 that may exceed the i32 signed range; bake + # it into the i32 constant as its two's-complement bit pattern so the + # unsigned ``umul_hi`` sees the right bits (matches the C++ uint32 path). + mult_i32 = multiplier - (1 << 32) if multiplier >= (1 << 31) else multiplier + tmp = b.umul_hi_i32(dividend, b.const_i32(mult_i32)) + summed = b.add_nuw(tmp, dividend) + if shift == 0: + return summed + return b.lshr(summed, b.const_i32(shift)) + + +def do_magic_division_dynamic( + b: IRBuilder, dividend: Value, multiplier: Value, shift: Value +) -> Value: + """Like :func:`do_magic_division` but ``multiplier`` and ``shift`` are + runtime SSA ``Value`` objects (e.g. kernel arguments) instead of Python + ``int``. Emits the same ``umul_hi + add + lshr`` sequence; ``lshr`` is + always emitted (shift == 0 produces a no-op right-shift). The sum is a + no-unsigned-wrap add, as in :func:`do_magic_division`. + """ + tmp = b.umul_hi_i32(dividend, multiplier) + summed = b.add_nuw(tmp, dividend) + return b.lshr(summed, shift) + + +__all__ = [ + "calculate_magic_numbers", + "do_magic_division", + "do_magic_division_dynamic", +] diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/transforms.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/transforms.py index 355e52962587..bf4d34487075 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/transforms.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/transforms.py @@ -73,6 +73,7 @@ from typing import Any, Dict, List, Optional, Sequence, Tuple from ..core.ir import F16, I32, I64, IRBuilder, Type, Value +from . import magic_division # --------------------------------------------------------------------- @@ -112,83 +113,6 @@ def _lt(b: IRBuilder, lhs: Value, rhs: Value) -> Value: return b.cmp_lt(lhs, rhs) -# --------------------------------------------------------------------- -# Magic-number division (mirrors ck_tile/core/utility/magic_div.hpp) -# --------------------------------------------------------------------- - - -def calculate_magic_numbers(divisor: int) -> Tuple[int, int]: - """Compute the ``(multiplier, shift)`` pair for unsigned magic division. - - Direct port of CK Tile's - ``magic_division32_bit_range::calculate_magic_numbers`` (the - ``magic_division`` alias used by ``merge_v2_magic_division``). The - returned constants reproduce ``dividend // divisor`` via the mul-hi - sequence in :func:`do_magic_division` for any ``dividend`` that fits - in the 31-bit unsigned range (the same restriction CK Tile documents). - - The math: - - .. code-block:: text - - shift = ceil(log2(divisor)) (smallest s with 2**s >= divisor) - multiplier = ((2**shift - divisor) << 32) // divisor + 1 - - Both values are plain Python ``int`` (compile-time constants); they are - baked into the emitted IR as ``const_i32`` operands, exactly like the - C++ constexpr path. - """ - if divisor < 1: - raise ValueError(f"magic division requires divisor >= 1, got {divisor}") - shift = 0 - while (1 << shift) < divisor: - shift += 1 - multiplier = (((1 << shift) - divisor) << 32) // divisor + 1 - return multiplier, shift - - -def do_magic_division( - b: IRBuilder, dividend: Value, multiplier: int, shift: int -) -> Value: - """Emit ``dividend // divisor`` using the magic ``(multiplier, shift)``. - - Mirrors CK Tile's ``do_magic_division`` device path - (``tmp = __umulhi(dividend, multiplier); (tmp + dividend) >> shift``), - which the AMDGPU backend lowers without an integer-division instruction. - ``dividend`` is an i32 SSA value interpreted as unsigned; the result is - the i32 quotient. ``multiplier`` / ``shift`` come from - :func:`calculate_magic_numbers`. - - The sum ``tmp + dividend`` is emitted as a no-unsigned-wrap add: for a - dividend near 2**31 it passes the signed maximum (from 1,610,612,736 for - divisor 3) while staying below 2**32, so a no-signed-wrap add would make - the quotient poison inside the documented range. - """ - # ``multiplier`` is a uint32 that may exceed the i32 signed range; bake - # it into the i32 constant as its two's-complement bit pattern so the - # unsigned ``umul_hi`` sees the right bits (matches the C++ uint32 path). - mult_i32 = multiplier - (1 << 32) if multiplier >= (1 << 31) else multiplier - tmp = b.umul_hi_i32(dividend, b.const_i32(mult_i32)) - summed = b.add_nuw(tmp, dividend) - if shift == 0: - return summed - return b.lshr(summed, b.const_i32(shift)) - - -def do_magic_division_dynamic( - b: IRBuilder, dividend: Value, multiplier: Value, shift: Value -) -> Value: - """Like :func:`do_magic_division` but ``multiplier`` and ``shift`` are - runtime SSA ``Value`` objects (e.g. kernel arguments) instead of Python - ``int``. Emits the same ``umul_hi + add + lshr`` sequence; ``lshr`` is - always emitted (shift == 0 produces a no-op right-shift). The sum is a - no-unsigned-wrap add, as in :func:`do_magic_division`. - """ - tmp = b.umul_hi_i32(dividend, multiplier) - summed = b.add_nuw(tmp, dividend) - return b.lshr(summed, shift) - - # --------------------------------------------------------------------- # Transforms # --------------------------------------------------------------------- @@ -622,9 +546,9 @@ class UnmergeMagicDiv(Transform): Numerically identical to :class:`Unmerge` -- it splits one flat upper coord into ``N`` lower coords -- but emits the mul-hi magic-division - sequence (:func:`do_magic_division`) instead of literal ``b.div`` / - ``b.mod``. This mirrors CK Tile's ``merge_v2_magic_division`` (the - default ``make_merge_transform``), whose ``calculate_lower_index`` is + sequence (:func:`rocke.helpers.magic_division.do_magic_division`) + instead of literal ``b.div`` / ``b.mod``. This mirrors CK Tile's + ``merge_v2_magic_division`` (the default ``make_merge_transform``), whose ``calculate_lower_index`` is exactly this flat-to-multi split. It removes the integer-division / modulo ops from the runtime addressing path (the single biggest source of div/mod assembly in conv ``m -> (n, ho, wo)`` / ``k -> (r, s, c)`` @@ -666,8 +590,8 @@ def apply(self, b: IRBuilder, coords: Dict[str, CoordVar]) -> Dict[str, CoordVar rem = b.const_i32(0) quot = tmp else: - mult, shift = calculate_magic_numbers(d) - quot = do_magic_division(b, tmp, mult, shift) + mult, shift = magic_division.calculate_magic_numbers(d) + quot = magic_division.do_magic_division(b, tmp, mult, shift) rem = b.sub(tmp, b.mul(quot, b.const_i32(d))) name = self.lower[i] out[name] = CoordVar(name, rem, u.valid) @@ -682,7 +606,8 @@ class UnmergeMagicDynamic(Transform): Like :class:`UnmergeMagicDiv` but the ``(multiplier, shift, dim)`` triples are SSA ``Value`` objects passed as kernel arguments, enabling shape-generic (AOT) kernels. The host computes the magic constants via - :func:`calculate_magic_numbers` at launch time and passes them in. + :func:`rocke.helpers.magic_division.calculate_magic_numbers` at launch + time and passes them in. ``magic_triples`` is a sequence of ``(multiplier, shift, dim)`` tuples, one per lower coord except the leading one (which gets the final quotient). @@ -726,7 +651,7 @@ def apply(self, b: IRBuilder, coords: Dict[str, CoordVar]) -> Dict[str, CoordVar rem = b.const_i32(0) quot = tmp else: - quot = do_magic_division_dynamic(b, tmp, mult_v, shift_v) + quot = magic_division.do_magic_division_dynamic(b, tmp, mult_v, shift_v) rem = b.sub(tmp, b.mul(quot, dim_v)) name = self.lower[i] out[name] = CoordVar(name, rem, u.valid) @@ -749,7 +674,7 @@ def unmerge_magic_dynamic( Host-side code computes each triple via:: - mult, shift = calculate_magic_numbers(dim) + mult, shift = magic_division.calculate_magic_numbers(dim) # pass mult, shift, dim as kernel args """ return UnmergeMagicDynamic(upper, into, magic_triples) @@ -1920,9 +1845,6 @@ def create( "UnmergeMagicDiv", "UnmergeMagicDynamic", "XorT", - "calculate_magic_numbers", - "do_magic_division", - "do_magic_division_dynamic", "embed", "embed_dynamic", "freeze", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/moe_gemm_fused.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/moe_gemm_fused.py index c14017cd9586..a9b005f6f640 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/moe_gemm_fused.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/moe_gemm_fused.py @@ -45,7 +45,7 @@ make_global_view, make_tile_window, ) -from ...helpers.transforms import calculate_magic_numbers, do_magic_division +from ...helpers.magic_division import calculate_magic_numbers, do_magic_division from .gemm_universal import ( DataSpec, TileSpec, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/portable_ir/portable_ir_scaling_plan.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/portable_ir/portable_ir_scaling_plan.md index f4bbe1292894..68c69c26dd4a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/portable_ir/portable_ir_scaling_plan.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/portable_ir/portable_ir_scaling_plan.md @@ -293,7 +293,7 @@ either reproduces the data or it does not, so it cannot overfit. It exists becau polynomial fits either, but the *generating formula* does, so the recipe carries that and regenerates both at replay. `magic_multiplier`/`magic_shift` are now intexpr primitives in both VMs; the DSL helper -(`helpers/transforms.py::calculate_magic_numbers`) stays the single source of +(`helpers/magic_division.py::calculate_magic_numbers`) stays the single source of truth, with `recipe_expand.py` and `recipe_vm.cpp` as tested mirrors. Ordering is what keeps a candidate library from becoming "fit anything", and it is diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/portable_ir/tests/test_roll_nd.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/portable_ir/tests/test_roll_nd.py index c2e8095dff4d..530e46d35dc4 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/portable_ir/tests/test_roll_nd.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/portable_ir/tests/test_roll_nd.py @@ -70,7 +70,7 @@ def build_magic(C): The emitted constants are a `ceil(log2 C)` shift and a multiplier keyed on C's odd part, so no polynomial in C fits either. Uses the production helper rather than hand-written numbers, so the test tracks the real idiom.""" - from rocke.helpers.transforms import calculate_magic_numbers, do_magic_division + from rocke.helpers.magic_division import calculate_magic_numbers, do_magic_division b = IRBuilder(f"magic_C{C}") A = b.param("A", PtrType(F32, "global"), readonly=True, align=16) @@ -359,7 +359,7 @@ def test_magic_constants_match_the_dsl_helper(self): Python ones against each other here; the C VM is pinned by `test_recipe_roller.py::test_standalone_cli_regenerates_magic_division_constants`. """ - from rocke.helpers.transforms import calculate_magic_numbers + from rocke.helpers.magic_division import calculate_magic_numbers for d in [1, 2, 3, 5, 7, 8, 9, 64, 96, 128, 160, 192, 224, 256, 384, 1000]: mult, shift = calculate_magic_numbers(d) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/portable_ir/utils/recipe_expand.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/portable_ir/utils/recipe_expand.py index 70330efd65cb..04194ddca7a4 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/portable_ir/utils/recipe_expand.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/portable_ir/utils/recipe_expand.py @@ -64,7 +64,7 @@ class GuardRejected(ExpandError): def magic_division_constants(divisor: int) -> tuple: """`(multiplier_i32, shift)` for a strength-reduced unsigned `n // divisor`. - The canonical mirror of `helpers/transforms.py::calculate_magic_numbers` plus + The canonical mirror of `helpers/magic_division.py::calculate_magic_numbers` plus `do_magic_division`'s two's-complement wrap (itself a port of CK Tile's `magic_division32_bit_range`). Kernels bake these two integers in as `const_i32` operands of `(umul_hi(n, M) + n) >> s`, so a recipe that wants to diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_magic_division_sum.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_magic_division_sum.py index 58cf96fa2d2b..f9489dda5d64 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_magic_division_sum.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_magic_division_sum.py @@ -26,7 +26,7 @@ from rocke.core.ir import I32, IRBuilder, PtrType from rocke.core.ir_serialize import serialize from rocke.core.lower_llvm import _lower_kernel_to_llvm_python -from rocke.helpers.transforms import ( +from rocke.helpers.magic_division import ( calculate_magic_numbers, do_magic_division, do_magic_division_dynamic, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_transforms_dynamic.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_transforms_dynamic.py index 488070d0d01a..d0284324d7e7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_transforms_dynamic.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_transforms_dynamic.py @@ -30,15 +30,17 @@ import pytest from rocke.core.ir import I32, IRBuilder, Value +from rocke.helpers.magic_division import ( + calculate_magic_numbers, + do_magic_division, + do_magic_division_dynamic, +) from rocke.helpers.transforms import ( CoordVar, DynamicTensorDescriptor, EmbedDynamic, PadDynamic, UnmergeMagicDynamic, - calculate_magic_numbers, - do_magic_division, - do_magic_division_dynamic, embed_dynamic, pad_dynamic, unmerge_magic_dynamic, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/portable_ir/test_recipe_roller.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/portable_ir/test_recipe_roller.py index e1856c29729f..e14ee2d8bfd7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/portable_ir/test_recipe_roller.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/portable_ir/test_recipe_roller.py @@ -228,7 +228,7 @@ def test_standalone_cli_replays_rolled_recipe(tmp_path): _MAGIC_BUILDER = """ def _magic(D): from rocke.core.ir import I32, IRBuilder, PtrType - from rocke.helpers.transforms import calculate_magic_numbers, do_magic_division + from rocke.helpers.magic_division import calculate_magic_numbers, do_magic_division b = IRBuilder("magic_div") out = b.param("out", PtrType(I32, "global")) From 72dd1602a9badd4bdfa28900165ad27488b4717d Mon Sep 17 00:00:00 2001 From: Chris Millette Date: Mon, 19 Oct 2026 11:05:06 -0400 Subject: [PATCH 4/4] fix(rocke): fold signed div and mod the way sdiv and srem compute them arith.div and arith.mod lower to sdiv and srem, which round the quotient toward zero, but the constant folder used Python's floored modulo (mirrored in C++) and a float division. A folded mod of a negative dividend or divisor, such as 3 mod 0x80000000 read at i32, disagreed with the unfolded kernel, and large i64 quotients lost precision. Both engines now fold with integer division rounded toward zero, and leave a zero divisor or the minimum value divided by -1 unfolded, as LLVM leaves them undefined. Tests in both engines cover high-bit and negative divisors, exact i64 quotients, and the unfolded cases. Co-Authored-By: Claude --- .../rocke/platform/cpp/core/passes.cpp | 24 ++++----- .../platform/python/rocke/core/passes.py | 28 ++++++++-- .../tests/core/test_unsigned_int_ops.cpp | 53 ++++++++++++++++++- .../tests/core/test_unsigned_int_ops.py | 53 +++++++++++++++++++ 4 files changed, 141 insertions(+), 17 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/passes.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/passes.cpp index aaa457f0327d..cb6bd0938ff1 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/passes.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/passes.cpp @@ -21,6 +21,7 @@ * graph for its whole lifetime -- abandoned intermediate arrays are reclaimed in * bulk when the arena is destroyed, exactly mirroring Python GC lifetime. */ +#include #include #include @@ -556,21 +557,20 @@ static bool try_fold(const rocke_op_t* op, int64_t* out) *out = (int64_t)((uint64_t)ints[0] * (uint64_t)ints[1]); return true; case ROCKE_OP_ARITH_DIV: - if(op->num_operands != 2 || ints[1] == 0) - return false; - /* Python int(ints[0] / ints[1]): float division truncated toward 0. */ - *out = (int64_t)((double)ints[0] / (double)ints[1]); - return true; case ROCKE_OP_ARITH_MOD: - if(op->num_operands != 2 || ints[1] == 0) + if(op->num_operands != 2) return false; - /* Python % is floored; mirror it (C99 % truncates toward zero). */ + /* Python _signed_quotient: C's / and % round toward zero, as LLVM's + * sdiv and srem do. A zero divisor, or a quotient past the signed + * range (the minimum value divided by -1), is left to run. */ { - int64_t a = ints[0], bm = ints[1]; - int64_t r = a % bm; - if(r != 0 && ((r < 0) != (bm < 0))) - r += bm; - *out = r; + const int64_t a = ints[0], d = ints[1]; + if(d == 0 || (a == INT64_MIN && d == -1)) + return false; + const int64_t quotient = a / d; + if(quotient != wrap_to_width(quotient, op->results[0]->type)) + return false; + *out = op->opcode == ROCKE_OP_ARITH_DIV ? quotient : a % d; } return true; case ROCKE_OP_ARITH_AND: diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/passes.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/passes.py index 8b8e7bb860c5..192f8b9dbee4 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/passes.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/passes.py @@ -192,6 +192,25 @@ def _wrap_to_width(value: int, t: Type) -> int: return value - (1 << width) if value >> (width - 1) else value +def _signed_quotient(a: int, b: int, t: Type) -> Optional[int]: + """``a / b`` as LLVM ``sdiv`` computes it, or None when it cannot fold. + + ``arith.div`` and ``arith.mod`` lower to ``sdiv`` and ``srem``, which round + the quotient toward zero (so ``-7 / 2`` is -3 and ``-7 % 2`` is -1), unlike + Python's floored ``//`` and ``%``. Integer arithmetic keeps i64 values + exact. A zero divisor, or a quotient past the signed range (the minimum + value divided by -1), is undefined in LLVM and is left to run. + """ + if b == 0: + return None + quotient = abs(a) // abs(b) + if (a < 0) != (b < 0): + quotient = -quotient + if quotient != _wrap_to_width(quotient, t): + return None + return quotient + + def _try_fold(op: Op) -> Optional[int]: if len(op.results) != 1: return None @@ -209,10 +228,11 @@ def _try_fold(op: Op) -> Optional[int]: return ints[0] - ints[1] if op.name == "arith.mul": return ints[0] * ints[1] - if op.name == "arith.div" and ints[1] != 0: - return int(ints[0] / ints[1]) - if op.name == "arith.mod" and ints[1] != 0: - return ints[0] % ints[1] + if op.name in ("arith.div", "arith.mod"): + quotient = _signed_quotient(ints[0], ints[1], op.results[0].type) + if quotient is None: + return None + return quotient if op.name == "arith.div" else ints[0] - ints[1] * quotient if op.name == "arith.and": return ints[0] & ints[1] if op.name == "arith.or": diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.cpp index 0e114fac02d5..09f8a6342cfa 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.cpp @@ -9,7 +9,8 @@ * the new attrs and its refusals of bad marker attrs (texts identical to the * Python verifier), and the constant folder (unsigned compare at the operand * width; constants read at their width; a marker on a constant folds to the - * constant), and the verifier's i32 / i64 rule for the nuw and ult forms. + * constant; signed div / mod rounded toward zero, undefined ones left + * unfolded), and the verifier's i32 / i64 rule for the nuw and ult forms. * * Run with --print-llvm it prints the LLVM text of the two-op function built * through rocke_b_add_nuw / rocke_b_cmp_ult, so the Python test can compare it @@ -448,6 +449,56 @@ void case_constant_folding() rocke_ir_builder_free(&lb); } + /* A folded div / mod holds what sdiv / srem compute (quotient rounded + * toward zero); a zero divisor or a quotient past the signed range is not + * folded. Same cases as the Python test. */ + struct DivModCase + { + bool is_mod; + bool wide; + int64_t a, d; + bool folds; + int64_t want; + }; + const DivModCase div_mod_cases[] = { + {true, false, 3, 0x80000000LL, true, 3}, + {false, false, 3, 0x80000000LL, true, 0}, + {true, false, -7, 2, true, -1}, + {false, false, -7, 2, true, -3}, + {true, false, 7, -2, true, 1}, + {false, false, 7, -2, true, -3}, + {true, false, -7, -2, true, -1}, + {false, false, INT32_MIN, 3, true, -715827882}, + {false, true, (int64_t{1} << 62) + 1, 1, true, (int64_t{1} << 62) + 1}, + {true, true, INT64_MIN, 3, true, -2}, + {false, false, INT32_MIN, -1, false, 0}, + {true, false, INT32_MIN, -1, false, 0}, + {false, true, INT64_MIN, -1, false, 0}, + {true, true, INT64_MIN, -1, false, 0}, + {false, false, 5, 0, false, 0}, + {true, false, 5, 0, false, 0}, + }; + for(const DivModCase& c : div_mod_cases) + { + rocke_ir_builder_t db; + rocke_ir_builder_init(&db, "unsigned_int_ops_fold_div_mod"); + const rocke_type_t* type = c.wide ? rocke_i64() : rocke_i32(); + rocke_value_t* out + = rocke_b_param(&db, "out", rocke_ptr_type(&db, type, "global"), nullptr); + rocke_value_t* av = c.wide ? rocke_b_const_i64(&db, c.a) : rocke_b_const_i32(&db, c.a); + rocke_value_t* dv = c.wide ? rocke_b_const_i64(&db, c.d) : rocke_b_const_i32(&db, c.d); + rocke_value_t* q = c.is_mod ? rocke_b_mod(&db, av, dv) : rocke_b_div(&db, av, dv); + rocke_b_global_store(&db, out, rocke_b_const_i32(&db, 0), q, 1); + int64_t got = -2; + const bool folded = folded_store_value(&db, &got); + const std::string label = std::string(c.is_mod ? "mod " : "div ") + std::to_string(c.a) + + ", " + std::to_string(c.d); + EXPECT(folded == c.folds, label + (c.folds ? " did not fold" : " folded")); + EXPECT(!c.folds || got == c.want, + label + " gave " + std::to_string(got) + ", want " + std::to_string(c.want)); + rocke_ir_builder_free(&db); + } + /* A marker on a constant folds to the constant, even with a runtime end. */ rocke_ir_builder_t b; rocke_ir_builder_init(&b, "unsigned_int_ops_fold_marker"); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.py index eeae95c9f889..0c395cd338cb 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_unsigned_int_ops.py @@ -400,6 +400,59 @@ def test_constants_are_read_at_their_width(): assert _folded_store_value(b.kernel) == 1 +def _div_or_mod(op_name, int_type, a, d): + const = {I32: lambda b, v: b.const_i32(v), I64: lambda b, v: b.const_i64(v)} + return _store_one( + f"fold_{op_name}", + int_type, + lambda b: getattr(b, op_name)(const[int_type](b, a), const[int_type](b, d)), + ) + + +# (op, type, dividend, divisor, the value sdiv / srem give). Both round the +# quotient toward zero, so a remainder takes the dividend's sign. +_DIV_MOD_FOLD_CASES = [ + ("mod", I32, 3, 0x80000000, 3), # the divisor reads as -2**31 + ("div", I32, 3, 0x80000000, 0), + ("mod", I32, -7, 2, -1), + ("div", I32, -7, 2, -3), + ("mod", I32, 7, -2, 1), + ("div", I32, 7, -2, -3), + ("mod", I32, -7, -2, -1), + ("div", I32, -(2**31), 3, -715827882), + ("div", I64, 2**62 + 1, 1, 2**62 + 1), # exact past a double's precision + ("mod", I64, -(2**63), 3, -2), +] + + +@pytest.mark.parametrize("op_name,int_type,a,d,expected", _DIV_MOD_FOLD_CASES) +def test_folded_div_and_mod_round_toward_zero(op_name, int_type, a, d, expected): + """A folded ``div`` / ``mod`` holds what the ``sdiv`` / ``srem`` it lowers to + computes, including for divisors with the high bit set (the ctest checks the + C++ folder with the same cases).""" + unfolded = _lower_kernel_to_llvm_python( + _div_or_mod(op_name, int_type, a, d), arch="gfx942" + ) + assert {"div": "sdiv", "mod": "srem"}[op_name] in unfolded + assert _folded_store_value(_div_or_mod(op_name, int_type, a, d)) == expected + + +@pytest.mark.parametrize("op_name", ["div", "mod"]) +@pytest.mark.parametrize( + "int_type,a,d", + [(I32, -(2**31), -1), (I64, -(2**63), -1), (I32, 5, 0)], + ids=["i32_min_by_minus_one", "i64_min_by_minus_one", "zero_divisor"], +) +def test_undefined_division_is_not_folded(op_name, int_type, a, d): + """A zero divisor or a quotient past the signed range makes ``sdiv`` and + ``srem`` undefined in LLVM; the folder leaves the op to run instead of + inventing a value.""" + kernel = _div_or_mod(op_name, int_type, a, d) + optimize_kernel(kernel) + (store,) = [op for op in kernel.body.ops if op.name == "memref.global_store_typed"] + assert store.operands[2].op.name == f"arith.{op_name}" + + def test_marker_on_a_constant_folds_to_the_constant(): b = IRBuilder("fold_marker") out = b.param("out", PtrType(I32, "global"))