From 218a041840650627f5f9840ee79628c137afabc8 Mon Sep 17 00:00:00 2001 From: Aleksander Dudek Date: Wed, 22 Jul 2026 07:50:24 +0000 Subject: [PATCH 1/5] feat(rocke): common LDS allocation for A/B/CShuffle Rebase of PR #8844 (users/barkocot/rocke-common-lds) onto the reorganized develop (rocKE/ -> rocke/platform, Python/ -> platform/python, Cpp/ -> platform/cpp). Only the common-LDS-allocation half of the original PR is applied here: the fp32/bf16 cshuffle half the branch also carried is already upstream on develop (store ops, trunc_f32_to_bf16, dtype_d epilogues), so re-applying it was skipped to avoid duplicate definitions. Contents (net vs develop): a single liveness-packed addrspace(3) smem pool with per-allocation byte offsets, DFS-preorder live-interval analysis + greedy interval packing (non-interfering A/B and cshuffle-C share the pool: pool = max(overlapping) instead of sum), and a per-(block,allocation) base-pointer cache so a non-zero-offset segment emits one byte-GEP per block. Mirrored in both the Python (core/lower_llvm.py) and C++ (cpp/core/lower_llvm/*) engines; conv implicit-GEMM LDS budget switched to max(A/B, C); new test_smem_liveness_packing.py; golden re-blessed for both llvm flavors. Validation on the rebased tree: byte-identity GREEN 65/65 at llvm20 and llvm22; pytest core (55) + test_rocke (231) + smem liveness (13) + golden gate (3) pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../platform/cpp/core/lower_llvm/core.cpp | 412 +++++++++++++++++- .../cpp/core/lower_llvm/crosslane.cpp | 48 +- .../platform/cpp/core/lower_llvm/mem.cpp | 52 ++- .../cpp/include/rocke/lower_llvm_internal.h | 34 ++ ...nv_implicit_gemm_conv_spec_descriptors.cpp | 67 ++- .../platform/python/rocke/core/lower_llvm.py | 295 +++++++++++-- .../instances/common/conv_implicit_gemm.py | 2 +- .../tests/core/test_smem_liveness_packing.py | 308 +++++++++++++ .../rocke_representative_ir_sha256.json | 256 +++++------ 9 files changed, 1241 insertions(+), 233 deletions(-) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_smem_liveness_packing.py diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp index af524c33d900..5a120082cdbb 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp @@ -946,6 +946,390 @@ const char* rocke_ll_smem_global_name(rocke_lower_t* L, L, ROCKE_ERR_KEY, "smem value %s never allocated", smem->name ? smem->name : "(null)"); } +/* ====================================================================== */ +/* smem live-interval analysis + pool layout (_compute_smem_layout) */ +/* ====================================================================== */ + +/* Live-interval entry: (first_seq, last_seq) for one smem global. */ +typedef struct ll_live_interval +{ + const char* gname; + int first_seq; + int last_seq; +} ll_live_interval_t; + +/* Mutable state threaded through the DFS walk (mirrors Python closures). */ +typedef struct ll_liveness_walk_ctx +{ + rocke_lower_t* L; + ll_live_interval_t* intervals; /* flat array, len == smem_globals.len */ + size_t num_intervals; + int counter; /* preorder sequence counter */ +} ll_liveness_walk_ctx_t; + +/* Return the index into intervals[] for gname, or -1 if not found. */ +static int ll_interval_index(const ll_liveness_walk_ctx_t* ctx, const char* gname) +{ + for(size_t i = 0; i < ctx->num_intervals; i++) + { + if(strcmp(ctx->intervals[i].gname, gname) == 0) + return (int)i; + } + return -1; +} + +/* Look up the global name for an IR value name (%foo) via smem_names. */ +static const char* ll_val_to_gname(const rocke_lower_t* L, const char* value_name) +{ + if(!value_name) + return NULL; + for(size_t i = 0; i < L->smem_names.len; i++) + { + if(L->smem_names.data[i].value_name + && strcmp(L->smem_names.data[i].value_name, value_name) == 0) + return L->smem_names.data[i].gname; + } + return NULL; +} + +/* Number of preorder counter ticks for `op` and its descendants -- mirrors + * Python `_subtree_size`. Matches ll_liveness_walk exactly: one tick per op + * slot (a NULL op still ticks the counter but does not recurse), plus every + * op in each sub-region. The subtree rooted at index `idx` therefore occupies + * contiguous sequence indices [idx, idx + ll_subtree_size(op) - 1]. */ +static int + ll_subtree_size(const rocke_op_t* op) +{ + int n = 1; + if(!op) + return n; + for(int r = 0; r < op->num_regions; r++) + { + const rocke_region_t* reg = op->regions[r]; + if(!reg) + continue; + for(int i = 0; i < reg->num_ops; i++) + n += ll_subtree_size(reg->ops[i]); + } + return n; +} + +/* DFS preorder walk -- mirrors Python `walk(ops, loop_end)`. */ +static void + ll_liveness_walk(ll_liveness_walk_ctx_t* ctx, const rocke_region_t* region, int loop_end) +{ + if(!region) + return; + for(int i = 0; i < region->num_ops; i++) + { + const rocke_op_t* op = region->ops[i]; + if(!op) + { + ctx->counter++; + continue; + } + int idx = ctx->counter++; + + /* Definition point: tile.smem_alloc */ + if(op->opcode == ROCKE_OP_TILE_SMEM_ALLOC && op->num_results > 0) + { + const rocke_value_t* res = op->results[0]; + const char* gn = ll_val_to_gname(ctx->L, res ? res->name : NULL); + if(gn) + { + int ii = ll_interval_index(ctx, gn); + if(ii >= 0 && ctx->intervals[ii].first_seq < 0) + { + ctx->intervals[ii].first_seq = idx; + ctx->intervals[ii].last_seq = idx; + } + } + } + + /* Any operand that is an smem value extends its live range. */ + for(int j = 0; j < op->num_operands; j++) + { + const rocke_value_t* v = op->operands[j]; + if(!v || !v->name) + continue; + const char* gn = ll_val_to_gname(ctx->L, v->name); + if(!gn) + continue; + int ii = ll_interval_index(ctx, gn); + if(ii < 0) + continue; + int first = (ctx->intervals[ii].first_seq < 0) ? idx : ctx->intervals[ii].first_seq; + int new_last = (loop_end >= 0) ? loop_end : idx; + int last = ctx->intervals[ii].last_seq; + ctx->intervals[ii].first_seq = (first < idx) ? first : idx; + ctx->intervals[ii].last_seq = (last > new_last) ? last : new_last; + } + + /* Recurse into sub-regions. scf.for gets a conservative loop_end: the + * last preorder index of the loop subtree (idx + size - 1), not the + * for-op's own (earlier) index, so allocations defined inside the loop + * are extended across the whole loop and interfere as they must. */ + for(int r = 0; r < op->num_regions; r++) + { + int child_loop_end + = (op->opcode == ROCKE_OP_SCF_FOR) ? (idx + ll_subtree_size(op) - 1) : loop_end; + ll_liveness_walk(ctx, op->regions[r], child_loop_end); + } + } +} + +/* Size of one smem allocation in bytes (Python _seg_size). */ +static int ll_smem_seg_size(const rocke_type_t* stype) +{ + if(!stype || !stype->elem || !stype->elem->name) + return 0; + const char* n = stype->elem->name; + int eb; + if(strcmp(n, "i8") == 0 || strcmp(n, "fp8e4m3") == 0 || strcmp(n, "bf8e5m2") == 0) + eb = 1; + else if(strcmp(n, "f16") == 0 || strcmp(n, "bf16") == 0) + eb = 2; + else if(strcmp(n, "i32") == 0 || strcmp(n, "f32") == 0) + eb = 4; + else if(strcmp(n, "i64") == 0) + eb = 8; + else + eb = 2; /* default */ + int seg = eb; + for(int d = 0; d < stype->rank; d++) + seg *= stype->shape[d]; + return seg; +} + +/* Alignment for one smem allocation (Python _align). */ +static int ll_smem_align(const rocke_type_t* stype) +{ + if(!stype || !stype->elem || !stype->elem->name) + return 4; + const char* n = stype->elem->name; + if(strcmp(n, "i8") == 0 || strcmp(n, "fp8e4m3") == 0 || strcmp(n, "bf8e5m2") == 0) + return 16; + return 4; +} + +void rocke_ll_compute_smem_layout(rocke_lower_t* L) +{ + if(!L) + return; + + /* Build the pool name unconditionally (needed even when empty). */ + const char* kname = L->kernel ? L->kernel->name : ""; + L->smem_pool_name = rocke_arena_printf(&L->arena, "@smem_pool.%s", kname ? kname : ""); + L->smem_pool_size = 0; + + /* Initialize offset vector to zero. */ + rocke_vec_init(&L->smem_offsets); + + size_t n = L->smem_globals.len; + if(n == 0) + return; + + /* ---- live-interval analysis ---- */ + ll_live_interval_t* intervals + = (ll_live_interval_t*)rocke_arena_calloc(&L->arena, n * sizeof(*intervals)); + if(!intervals) + rocke_ll_fail(L, ROCKE_ERR_OOM, "smem_layout: intervals alloc"); + + for(size_t i = 0; i < n; i++) + { + intervals[i].gname = L->smem_globals.data[i].gname; + intervals[i].first_seq = -1; /* unset */ + intervals[i].last_seq = -1; + } + + ll_liveness_walk_ctx_t ctx; + ctx.L = L; + ctx.intervals = intervals; + ctx.num_intervals = n; + ctx.counter = 0; + ll_liveness_walk(&ctx, L->kernel ? L->kernel->body : NULL, -1); + + /* Fix up allocations with no uses: first_seq = last_seq = 0. */ + for(size_t i = 0; i < n; i++) + { + if(intervals[i].first_seq < 0) + { + intervals[i].first_seq = 0; + intervals[i].last_seq = 0; + } + } + + /* ---- sort by live-interval start (stable: preserve declaration order for + * ties, like Python's sorted key) ---- */ + /* Build an index array and sort it. Simple insertion sort (few allocs). */ + int* order = (int*)rocke_arena_calloc(&L->arena, n * sizeof(int)); + if(!order) + rocke_ll_fail(L, ROCKE_ERR_OOM, "smem_layout: order alloc"); + for(size_t i = 0; i < n; i++) + order[i] = (int)i; + /* Insertion sort by first_seq. */ + for(size_t i = 1; i < n; i++) + { + int key = order[i]; + int key_first = intervals[key].first_seq; + int j = (int)i - 1; + while(j >= 0 && intervals[order[j]].first_seq > key_first) + { + order[j + 1] = order[j]; + j--; + } + order[j + 1] = key; + } + + /* ---- greedy interval packing ---- */ + /* Each slot: (offset, size, last_seq). */ + typedef struct ll_slot + { + int offset; + int size; + int last_seq; + } ll_slot_t; + + ll_slot_t* slots = (ll_slot_t*)rocke_arena_calloc(&L->arena, n * sizeof(*slots)); + if(!slots) + rocke_ll_fail(L, ROCKE_ERR_OOM, "smem_layout: slots alloc"); + int num_slots = 0; + + /* Offset array in declaration order (parallel to smem_globals). */ + int* offsets = (int*)rocke_arena_calloc(&L->arena, n * sizeof(int)); + if(!offsets) + rocke_ll_fail(L, ROCKE_ERR_OOM, "smem_layout: offsets alloc"); + + for(size_t si = 0; si < n; si++) + { + int gi = order[si]; /* smem_globals index */ + const rocke_type_t* stype = L->smem_globals.data[gi].stype; + int seg = ll_smem_seg_size(stype); + int aln = ll_smem_align(stype); + int first_seq = intervals[gi].first_seq; + int last_seq = intervals[gi].last_seq; + + /* Find the best (lowest aligned-offset) free slot. */ + int best = -1; + int best_aligned = -1; + for(int k = 0; k < num_slots; k++) + { + if(slots[k].last_seq >= first_seq) + continue; /* still live, interference */ + int aligned_off = (slots[k].offset + aln - 1) & ~(aln - 1); + if(best < 0 || aligned_off < best_aligned) + { + best = k; + best_aligned = aligned_off; + } + } + + if(best >= 0) + { + int aligned_off = best_aligned; + offsets[gi] = aligned_off; + int new_size = slots[best].size; + int end_needed = aligned_off - slots[best].offset + seg; + if(end_needed > new_size) + new_size = end_needed; + slots[best].size = new_size; + slots[best].last_seq = last_seq; + } + else + { + /* No reusable slot; open a new one at the end of the pool. */ + int current_end = 0; + for(int k = 0; k < num_slots; k++) + { + int e = slots[k].offset + slots[k].size; + if(e > current_end) + current_end = e; + } + int aligned_off = (current_end + aln - 1) & ~(aln - 1); + offsets[gi] = aligned_off; + slots[num_slots].offset = aligned_off; + slots[num_slots].size = seg; + slots[num_slots].last_seq = last_seq; + num_slots++; + } + } + + /* Compute pool size (max slot end, rounded to 16). */ + int pool_size = 0; + for(int k = 0; k < num_slots; k++) + { + int e = slots[k].offset + slots[k].size; + if(e > pool_size) + pool_size = e; + } + L->smem_pool_size = (pool_size + 15) & ~15; + + /* Populate L->smem_offsets in declaration order. */ + for(size_t i = 0; i < n; i++) + { + int rc; + rocke_vec_push(&L->arena, &L->smem_offsets, offsets[i], rc); + if(rc != 0) + rocke_ll_fail(L, ROCKE_ERR_OOM, "smem_layout: offsets push"); + } +} + +const char* + rocke_ll_emit_smem_base_ptr(rocke_lower_t* L, const char* gname, const rocke_type_t* stype) +{ + if(!L) + return ""; + if(!gname) + return L->smem_pool_name ? L->smem_pool_name : ""; + + /* Find the offset for this global. */ + int offset = 0; + for(size_t i = 0; i < L->smem_globals.len; i++) + { + if(L->smem_globals.data[i].gname && strcmp(L->smem_globals.data[i].gname, gname) == 0) + { + if(i < L->smem_offsets.len) + offset = L->smem_offsets.data[i]; + break; + } + } + + if(offset == 0) + return L->smem_pool_name; + + /* Reuse a base pointer already computed for this allocation in the current + * block: the byte offset is a compile-time constant, so one GEP per + * (block, allocation) suffices and dominates all later same-block uses + * (instructions within a block execute sequentially). Mirrors Python + * _smem_base_cache. */ + const rocke_ll_block_t* blk = rocke_ll_current(L); + for(size_t i = 0; i < L->smem_base_cache.len; i++) + { + if(L->smem_base_cache.data[i].block == blk && L->smem_base_cache.data[i].gname + && strcmp(L->smem_base_cache.data[i].gname, gname) == 0) + return L->smem_base_cache.data[i].base; + } + + const char* agg_ty = rocke_ll_smem_storage_type(L, stype); + (void)agg_ty; /* not needed for the byte-GEP */ + const char* base = rocke_ll_fresh(L, "smem_base"); + rocke_ll_emitf(L, + " %s = getelementptr inbounds i8, ptr addrspace(3) %s, i32 %d", + base, + L->smem_pool_name, + offset); + + rocke_ll_smem_base_cache_t ent; + ent.block = blk; + ent.gname = gname; + ent.base = base; + int rc; + rocke_vec_push(&L->arena, &L->smem_base_cache, ent, rc); + if(rc != 0) + rocke_ll_fail(L, ROCKE_ERR_OOM, "smem_base_cache push"); + return base; +} + /* ====================================================================== */ /* yield-stack helpers DEFINED IN CONTROL bucket -- not here. */ /* ====================================================================== */ @@ -1159,24 +1543,16 @@ void rocke_ll_finalize(rocke_lower_t* L, rocke_strbuf_t* out) rocke_strbuf_appendf(out, "target triple = \"%s\"\n", tr ? tr : ""); rocke_strbuf_append(out, "\n"); - /* smem globals. */ - for(size_t i = 0; i < L->smem_globals.len; i++) + /* smem pool: a single unified addrspace(3) global backing all smem allocations. + * align 16 satisfies all segment alignments (strictest: 16 B for ds_read_b64_tr_b8 + * on i8/fp8 tiles). */ + if(L->smem_globals.len > 0 && L->smem_pool_name && L->smem_pool_size > 0) { - const rocke_ll_smem_global_t* g = &L->smem_globals.data[i]; - const char* agg = rocke_ll_smem_storage_type(L, g->stype); - const char* elem_name = (g->stype && g->stype->elem) ? g->stype->elem->name : NULL; - bool elem_is_byte = elem_name - && (strcmp(elem_name, "i8") == 0 || strcmp(elem_name, "fp8e4m3") == 0 - || strcmp(elem_name, "bf8e5m2") == 0); - int align = elem_is_byte ? 16 : 4; rocke_strbuf_appendf(out, - "%s = internal unnamed_addr addrspace(3) global %s poison, align %d\n", - g->gname, - agg, - align); - } - if(L->smem_globals.len > 0) - { + "%s = internal unnamed_addr addrspace(3) " + "global [%d x i8] poison, align 16\n", + L->smem_pool_name, + L->smem_pool_size); rocke_strbuf_append(out, "\n"); } @@ -1369,6 +1745,7 @@ static void ll_lower_into(rocke_lower_t* L, /* Pre-pass + lowering. */ rocke_ll_collect_smem(L, kernel->body); + rocke_ll_compute_smem_layout(L); rocke_ll_lower_region(L, kernel->body); rocke_strbuf_t sb; @@ -1446,6 +1823,9 @@ static rocke_status_t ll_lower_kernel_to_llvm_ex_impl(const rocke_kernel_def_t* rocke_vec_init(&L.dyn_decls); rocke_vec_init(&L.smem_globals); rocke_vec_init(&L.smem_names); + rocke_vec_init(&L.smem_offsets); + L.smem_pool_size = 0; + L.smem_pool_name = NULL; rocke_vec_init(&L.yield_stack); /* A failure anywhere in lowering raises a ckc::Error; catch it here so the diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/crosslane.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/crosslane.cpp index bc6bc5796b52..20c7327c1482 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/crosslane.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/crosslane.cpp @@ -428,19 +428,22 @@ static void ll_ds_read_tr16(rocke_lower_t* L, return; } agg_ty = rocke_ll_smem_storage_type(L, stype); - base = rocke_ll_fresh(L, base_hint); - - /* getelementptr inbounds , ptr addrspace(3) , i32 0, i32 ... */ - if(rocke_strbuf_init(&gep, 64) != 0) { - rocke_ll_fail(L, ROCKE_ERR_OOM, "ds_read_tr16: strbuf OOM"); + const char* pool_ptr = rocke_ll_emit_smem_base_ptr(L, gname, stype); + base = rocke_ll_fresh(L, base_hint); + + /* getelementptr inbounds , ptr addrspace(3) , i32 0, i32 ... */ + if(rocke_strbuf_init(&gep, 64) != 0) + { + rocke_ll_fail(L, ROCKE_ERR_OOM, "ds_read_tr16: strbuf OOM"); + } + rocke_strbuf_appendf(&gep, + " %s = getelementptr inbounds %s, ptr addrspace(3) %s, " + "i32 0", + base, + agg_ty, + pool_ptr); } - rocke_strbuf_appendf(&gep, - " %s = getelementptr inbounds %s, ptr addrspace(3) %s, " - "i32 0", - base, - agg_ty, - gname); for(i = 1; i < op->num_operands; ++i) { rocke_strbuf_appendf(&gep, ", i32 %s", rocke_ll_operand(L, op->operands[i])); @@ -497,18 +500,21 @@ static void _op_tile_ds_read_tr_b8(rocke_lower_t* L, const rocke_op_t* op) return; } agg_ty = rocke_ll_smem_storage_type(L, stype); - base = rocke_ll_fresh(L, "tr8.base"); - - if(rocke_strbuf_init(&gep, 64) != 0) { - rocke_ll_fail(L, ROCKE_ERR_OOM, "ds_read_tr_b8: strbuf OOM"); + const char* pool_ptr = rocke_ll_emit_smem_base_ptr(L, gname, stype); + base = rocke_ll_fresh(L, "tr8.base"); + + if(rocke_strbuf_init(&gep, 64) != 0) + { + rocke_ll_fail(L, ROCKE_ERR_OOM, "ds_read_tr_b8: strbuf OOM"); + } + rocke_strbuf_appendf(&gep, + " %s = getelementptr inbounds %s, ptr addrspace(3) %s, " + "i32 0", + base, + agg_ty, + pool_ptr); } - rocke_strbuf_appendf(&gep, - " %s = getelementptr inbounds %s, ptr addrspace(3) %s, " - "i32 0", - base, - agg_ty, - gname); for(i = 1; i < op->num_operands; ++i) { rocke_strbuf_appendf(&gep, ", i32 %s", rocke_ll_operand(L, op->operands[i])); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mem.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mem.cpp index 8a49810797e2..935397354087 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mem.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/mem.cpp @@ -383,12 +383,17 @@ static void op_tile_smem_store(rocke_lower_t* L, const rocke_op_t* op) const rocke_value_t* value = op->operands[op->num_operands - 1]; const rocke_type_t* stype = NULL; const char* gname = rocke_ll_smem_global_name(L, smem, &stype); + const char* base_ptr = rocke_ll_emit_smem_base_ptr(L, gname, stype); const char* gep = rocke_ll_fresh(L, "gep"); const char* gidx = ll_smem_gidx(L, op, 1, op->num_operands - 1); const char* agg_ty = rocke_ll_smem_storage_type(L, stype); int align = ll_elem_bytes(value->type->name); - rocke_ll_emitf( - L, " %s = getelementptr inbounds %s, ptr addrspace(3) %s, %s", gep, agg_ty, gname, gidx); + rocke_ll_emitf(L, + " %s = getelementptr inbounds %s, ptr addrspace(3) %s, %s", + gep, + agg_ty, + base_ptr, + gidx); rocke_ll_emitf(L, " store %s %s, ptr addrspace(3) %s, align %d", rocke_ll_llvm_type(L, value->type), @@ -404,13 +409,18 @@ static void op_tile_lds_atomic_add(rocke_lower_t* L, const rocke_op_t* op) const char* elem_ty = rocke_ll_llvm_type(L, val->type); const rocke_type_t* stype = NULL; const char* gname = rocke_ll_smem_global_name(L, smem, &stype); + const char* base_ptr = rocke_ll_emit_smem_base_ptr(L, gname, stype); const char* agg_ty = rocke_ll_smem_storage_type(L, stype); const char* gep = rocke_ll_fresh(L, "gep"); const char* gidx = ll_smem_gidx(L, op, 1, op->num_operands - 1); const char* ordering = ll_attr_str(op, "ordering", "monotonic"); const char* rmw_op = (val->type->name && strcmp(val->type->name, "f32") == 0) ? "fadd" : "add"; - rocke_ll_emitf( - L, " %s = getelementptr inbounds %s, ptr addrspace(3) %s, %s", gep, agg_ty, gname, gidx); + rocke_ll_emitf(L, + " %s = getelementptr inbounds %s, ptr addrspace(3) %s, %s", + gep, + agg_ty, + base_ptr, + gidx); rocke_ll_emitf(L, " %s = atomicrmw %s ptr addrspace(3) %s, %s %s %s", ll_res(op), @@ -428,14 +438,19 @@ static void op_tile_smem_store_vN(rocke_lower_t* L, const rocke_op_t* op) int64_t vec = ll_attr_int(op, "vec", 0); const rocke_type_t* stype = NULL; const char* gname = rocke_ll_smem_global_name(L, smem, &stype); + const char* base_ptr = rocke_ll_emit_smem_base_ptr(L, gname, stype); const char* agg_ty = rocke_ll_smem_storage_type(L, stype); const char* gep = rocke_ll_fresh(L, "gep"); const char* gidx = ll_smem_gidx(L, op, 1, op->num_operands - 1); const char* elem_ty = rocke_ll_llvm_type(L, value->type->elem); int elem_bytes = ll_elem_bytes(value->type->elem->name); int64_t align = ll_attr_int(op, "align", vec * elem_bytes); - rocke_ll_emitf( - L, " %s = getelementptr inbounds %s, ptr addrspace(3) %s, %s", gep, agg_ty, gname, gidx); + rocke_ll_emitf(L, + " %s = getelementptr inbounds %s, ptr addrspace(3) %s, %s", + gep, + agg_ty, + base_ptr, + gidx); rocke_ll_emitf(L, " store <%lld x %s> %s, ptr addrspace(3) %s, align %lld", (long long)vec, @@ -452,6 +467,7 @@ static void op_tile_smem_load_v4(rocke_lower_t* L, const rocke_op_t* op) const rocke_value_t* col = op->operands[2]; const rocke_type_t* stype = NULL; const char* gname = rocke_ll_smem_global_name(L, smem, &stype); + const char* base_ptr = rocke_ll_emit_smem_base_ptr(L, gname, stype); const char* agg_ty = rocke_ll_smem_storage_type(L, stype); const char* base = rocke_ll_fresh(L, "smem.base"); const char* elems[4]; @@ -462,7 +478,7 @@ static void op_tile_smem_load_v4(rocke_lower_t* L, const rocke_op_t* op) "i32 0, i32 %s, i32 %s", base, agg_ty, - gname, + base_ptr, rocke_ll_operand(L, row), rocke_ll_operand(L, col)); for(i = 0; i < 4; i++) @@ -491,6 +507,7 @@ static void op_tile_smem_load_vN(rocke_lower_t* L, const rocke_op_t* op) int64_t vec = ll_attr_int(op, "vec", 0); const rocke_type_t* stype = NULL; const char* gname = rocke_ll_smem_global_name(L, smem, &stype); + const char* base_ptr = rocke_ll_emit_smem_base_ptr(L, gname, stype); const char* agg_ty = rocke_ll_smem_storage_type(L, stype); const char* base = rocke_ll_fresh(L, "smem.base"); const char* idx_strs = ll_smem_gidx(L, op, 1, op->num_operands); @@ -527,7 +544,7 @@ static void op_tile_smem_load_vN(rocke_lower_t* L, const rocke_op_t* op) " %s = getelementptr inbounds %s, ptr addrspace(3) %s, %s", base, agg_ty, - gname, + base_ptr, idx_strs); if(vec == 1) { @@ -564,6 +581,7 @@ static void op_tile_smem_store_distributed(rocke_lower_t* L, const rocke_op_t* o int n = ll_is_vec(values->type) ? values->type->count : 1; const rocke_type_t* stype = NULL; const char* gname = rocke_ll_smem_global_name(L, smem, &stype); + const char* base_ptr = rocke_ll_emit_smem_base_ptr(L, gname, stype); const char* agg_ty = rocke_ll_smem_storage_type(L, stype); const char* elem_ty = ll_is_vec(values->type) ? rocke_ll_llvm_type(L, values->type->elem) : rocke_ll_llvm_type(L, values->type); @@ -583,7 +601,7 @@ static void op_tile_smem_store_distributed(rocke_lower_t* L, const rocke_op_t* o " %s = getelementptr inbounds %s, ptr addrspace(3) %s, i32 0, i32 %d", gep, agg_ty, - gname, + base_ptr, i); rocke_ll_emitf(L, " store %s %s, ptr addrspace(3) %s, align 2", elem_ty, ev, gep); } @@ -596,12 +614,17 @@ static void op_tile_smem_store_vN_f32(rocke_lower_t* L, const rocke_op_t* op) int64_t vec = ll_attr_int(op, "vec", 0); const rocke_type_t* stype = NULL; const char* gname = rocke_ll_smem_global_name(L, smem, &stype); + const char* base_ptr = rocke_ll_emit_smem_base_ptr(L, gname, stype); const char* agg_ty = rocke_ll_smem_storage_type(L, stype); const char* gep = rocke_ll_fresh(L, "gep"); const char* gidx = ll_smem_gidx(L, op, 1, op->num_operands - 1); int64_t align = vec * 4; - rocke_ll_emitf( - L, " %s = getelementptr inbounds %s, ptr addrspace(3) %s, %s", gep, agg_ty, gname, gidx); + rocke_ll_emitf(L, + " %s = getelementptr inbounds %s, ptr addrspace(3) %s, %s", + gep, + agg_ty, + base_ptr, + gidx); if(vec == 1) { if(ll_is_vec(value->type)) @@ -641,6 +664,7 @@ static void op_tile_smem_load_vN_f32(rocke_lower_t* L, const rocke_op_t* op) int64_t vec = ll_attr_int(op, "vec", 0); const rocke_type_t* stype = NULL; const char* gname = rocke_ll_smem_global_name(L, smem, &stype); + const char* base_ptr = rocke_ll_emit_smem_base_ptr(L, gname, stype); const char* agg_ty = rocke_ll_smem_storage_type(L, stype); const char* base = rocke_ll_fresh(L, "smem.base"); const char* idx_strs = ll_smem_gidx(L, op, 1, op->num_operands); @@ -649,7 +673,7 @@ static void op_tile_smem_load_vN_f32(rocke_lower_t* L, const rocke_op_t* op) " %s = getelementptr inbounds %s, ptr addrspace(3) %s, %s", base, agg_ty, - gname, + base_ptr, idx_strs); if(vec == 1) { @@ -680,8 +704,10 @@ static void op_tile_smem_load_vN_f32(rocke_lower_t* L, const rocke_op_t* op) static void op_tile_smem_addr_of(rocke_lower_t* L, const rocke_op_t* op) { const rocke_value_t* smem = op->operands[0]; + const rocke_type_t* stype = smem->type; const char* gname = rocke_ll_smem_global_name(L, smem, NULL); - rocke_ll_emitf(L, " %s = ptrtoint ptr addrspace(3) %s to i64", ll_res(op), gname); + const char* base_ptr = rocke_ll_emit_smem_base_ptr(L, gname, stype); + rocke_ll_emitf(L, " %s = ptrtoint ptr addrspace(3) %s to i64", ll_res(op), base_ptr); } static void op_tile_smem_ptr_add(rocke_lower_t* L, const rocke_op_t* op) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/lower_llvm_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/lower_llvm_internal.h index db20fb2396cc..acdf998e58fb 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/lower_llvm_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/lower_llvm_internal.h @@ -166,6 +166,19 @@ typedef struct rocke_ll_smem_name const char* gname; } rocke_ll_smem_name_t; +/* One cached smem base pointer: the byte-level GEP SSA name computed for + * `gname` inside basic block `block`. Reused for later accesses of the same + * allocation in the same block (which the GEP dominates), so a non-zero-offset + * allocation accessed many times emits just one base GEP. Python + * _smem_base_cache keyed by (block label, gname); here we key on the block + * pointer -- one distinct arena object per block -- which groups identically. */ +typedef struct rocke_ll_smem_base_cache +{ + const rocke_ll_block_t* block; + const char* gname; + const char* base; +} rocke_ll_smem_base_cache_t; + /* A "needed intrinsic" record: the decl key plus the resolved decl text. The * decl text is captured at _need() time so dynamically-built decls (Python * self._decls[intrin] = ... for vector smax) are preserved in emit order. */ @@ -203,6 +216,13 @@ typedef struct rocke_lower ROCKE_VEC(rocke_ll_smem_global_t) smem_globals; ROCKE_VEC(rocke_ll_smem_name_t) smem_names; + /* smem pool: one unified addrspace(3) buffer; per-allocation byte offsets. + * Populated by rocke_ll_compute_smem_layout() after _collect_smem. */ + ROCKE_VEC(int) smem_offsets; /* parallel to smem_globals: byte offset per alloc */ + int smem_pool_size; /* total pool size in bytes (rounded to 16) */ + const char* smem_pool_name; /* "@smem_pool." */ + ROCKE_VEC(rocke_ll_smem_base_cache_t) smem_base_cache; /* per-(block,alloc) base ptr */ + /* scf.for yield recording stack (Python _yield_stack: list of list[str]). * Each frame is a vector of operand strings. */ ROCKE_VEC(ROCKE_VEC(const char*) *) yield_stack; @@ -326,6 +346,20 @@ const char* rocke_ll_smem_global_name(rocke_lower_t* L, const rocke_value_t* smem, const rocke_type_t** out_stype); +/* Compute byte offsets for all smem allocations in a single pool, using + * live-interval analysis to allow non-interfering allocations to share space + * (Python _compute_smem_layout). Must be called after rocke_ll_collect_smem + * and before rocke_ll_lower_region. Populates L->smem_offsets, + * L->smem_pool_size, and L->smem_pool_name. */ +void rocke_ll_compute_smem_layout(rocke_lower_t* L); + +/* Emit a byte-level GEP to the start of the smem segment for `gname` inside + * the unified pool. Returns the pool name directly when offset == 0; + * otherwise emits one GEP and returns a fresh SSA name (Python + * _emit_smem_base_ptr). */ +const char* + rocke_ll_emit_smem_base_ptr(rocke_lower_t* L, const char* gname, const rocke_type_t* stype); + /* ====================================================================== */ /* yield-stack helpers (Python _yield_stack manipulation) */ /* ====================================================================== */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp index 2622f477120b..0a52f696890f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp @@ -731,32 +731,57 @@ bool rocke_implicit_gemm_conv_is_valid_spec(const rocke_implicit_gemm_conv_spec_ arch); } - /* LDS budget: must fit before we attempt codegen. - * A_smem + B_smem, each (tile_m or tile_n) × row_stride × 2 bytes (f16). - * row_stride = tile_k + k_pad (k_pad = 8 when tile_k >= 16, else 0). - * compv4 pipeline double-buffers A and B → ×2. - * This mirrors the smem_alloc calls in instance_conv_implicit_gemm_conv_build_glue.c - * and catches the overflow that would otherwise produce CODEGEN_BC_TO_RELOCATABLE. */ - { - /* k_pad and double_buffer mirror build_glue priority order exactly: - * k_pad: has_lds_k_pad → lds_k_pad; async_dma → 0; else (tile_k>=16)?8:0 - * double_buffer: compv4 || async_dma || unroll_k */ - int k_pad - = s->has_lds_k_pad ? s->lds_k_pad : (s->async_dma ? 0 : ((s->tile_k >= 16) ? 8 : 0)); - int row_stride = s->tile_k + k_pad; - int ab_single = (s->tile_m + s->tile_n) * row_stride * 2; /* f16 = 2 bytes */ - int double_buf + /* LDS budget: A/B staging (×2 for double-buffer) + optional cshuffle C. + * Mirrors the Python is_valid_spec LDS check (added on this branch): + * _ab_dtype_bytes = 4 if dtype_a == "fp32" else 2 + * _a_shape = effective_lds_layout().storage_shape(tile_m) = (tile_m, row_stride) + * _b_shape = effective_lds_layout().storage_shape(tile_n) = (tile_n, row_stride) + * _ab_bytes = (a_rows*a_cols + b_rows*b_cols) * _ab_dtype_bytes + * _ab_lds = _ab_bytes * (2 if double_buffer else 1) + * _c_lds = tile_m * tile_n * _c_dtype_bytes (if epilogue == "cshuffle") + * _total_lds = max(_ab_lds, _c_lds) (not sum -- they can overlap in pool) */ + { + rocke_conv_lds_layout_t lds_layout; + char lds_reason[256]; + int ab_dtype_bytes; + int ab_bytes; + int ab_lds; + int c_lds; + int total_lds; + int double_buf; + bool is_cshuffle; + + if(!rocke_implicit_gemm_conv_spec_effective_lds_layout( + s, &lds_layout, lds_reason, sizeof(lds_reason))) + { + ROCKE_CONVVS_REJECT("%s", lds_reason); + } + + ab_dtype_bytes = (s->dtype_a && strcmp(s->dtype_a, "fp32") == 0) ? 4 : 2; + /* storage_shape(rows) = (rows, row_stride) */ + ab_bytes = (s->tile_m * lds_layout.row_stride + s->tile_n * lds_layout.row_stride) + * ab_dtype_bytes; + + double_buf = ((s->pipeline && strcmp(s->pipeline, "compv4") == 0) || s->async_dma || s->unroll_k) ? 1 : 0; - int bytes_lds = ab_single * (double_buf ? 2 : 1); - if(!rocke_archtarget_fits_lds(target, (long)bytes_lds)) + ab_lds = ab_bytes * (double_buf ? 2 : 1); + + is_cshuffle = (s->epilogue && strcmp(s->epilogue, "cshuffle") == 0); + { + int c_dtype_bytes = (s->dtype_d && strcmp(s->dtype_d, "fp32") == 0) ? 4 : 2; + c_lds = is_cshuffle ? (s->tile_m * s->tile_n * c_dtype_bytes) : 0; + } + total_lds = (ab_lds > c_lds) ? ab_lds : c_lds; + + if(!rocke_archtarget_fits_lds(target, (long)total_lds)) { - ROCKE_CONVVS_REJECT("LDS budget %d > %d cap (AB=%d, double_buf=%d) on %s", - bytes_lds, + ROCKE_CONVVS_REJECT("LDS budget %d bytes (A/B=%d, C=%d) > %d cap on %s", + total_lds, + ab_lds, + c_lds, target->lds_capacity_bytes, - ab_single, - double_buf, arch); } } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py index ec7c3ec1f669..a00ca9fd5813 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py @@ -881,6 +881,15 @@ def __init__( self._needs_fp_atomic_md: bool = False self._smem_globals: List[Tuple[str, SmemType]] = [] self._smem_storage_name: Dict[str, str] = {} # IR value name -> @global name + # smem pool: one unified addrspace(3) buffer; per-allocation byte offsets. + self._smem_offsets: Dict[str, int] = {} + self._smem_pool_size: int = 0 + self._smem_pool_name: Optional[str] = None + # Cache of per-allocation base pointers already emitted, keyed by + # (block label, global name). Lets a non-zero-offset allocation that is + # accessed several times in the same block reuse a single byte-level + # GEP instead of re-emitting it per access. + self._smem_base_cache: Dict[Tuple[str, str], str] = {} self._blocks: List[_Block] = [_Block("entry")] self._block_counter = 0 self._tmp_counter = 0 @@ -969,6 +978,218 @@ def _collect_smem(self, region: Region) -> None: for r in op.regions: self._collect_smem(r) + def _collect_smem_liveness(self, region: Region) -> Dict[str, Tuple[int, int]]: + """Compute live intervals for smem allocations via a DFS preorder walk. + + Returns a dict mapping global-name -> (first_seq, last_seq) where + seq is the preorder index of the op that defines / last-uses the + allocation. Uses inside a loop body (``scf.for``) are conservatively + extended to the *last* sequence index of the enclosing ``scf.for`` + subtree so that two allocations that are both live inside the loop + always interfere (they may be read on any iteration). + """ + # Map from IR value name (%foo) -> global name (@foo.kernel) + val_to_gname: Dict[str, str] = { + v: g for v, g in self._smem_storage_name.items() + } + intervals: Dict[str, Tuple[int, int]] = {} # gname -> (first, last) + counter = [0] # mutable int for nested closures + + def _subtree_size(op) -> int: + # Number of ops in this op's DFS-preorder subtree (itself + all + # descendants). Preorder numbers an op *before* its body, so the + # subtree rooted at index ``idx`` occupies contiguous sequence + # indices ``[idx, idx + _subtree_size(op) - 1]``. + n = 1 + for r in op.regions: + for child in r.ops: + n += _subtree_size(child) + return n + + def walk(ops: List, loop_end: Optional[int]) -> None: + for op in ops: + idx = counter[0] + counter[0] += 1 + + # Definition point of an alloc + if op.name == "tile.smem_alloc": + gname = val_to_gname[op.result.name] + if gname not in intervals: + intervals[gname] = (idx, idx) + + # Any operand that is an smem value extends its live range + for v in op.operands: + gname = val_to_gname.get(v.name) + if gname is not None: + first, last = intervals.get(gname, (idx, idx)) + new_last = loop_end if loop_end is not None else idx + intervals[gname] = (min(first, idx), max(last, new_last)) + + # Recurse into sub-regions; scf.for gets a conservative loop_end + for r in op.regions: + if op.name == "scf.for": + # Conservative loop liveness: a value written on one + # iteration may be read on the next, so every allocation + # touched anywhere in the loop body must be treated as + # live for the whole loop and thus interfere with the + # others. Preorder numbers the for-op *before* its body, + # so the loop subtree ends at ``idx + size - 1``; extend + # uses to that last index (not the for-op's own, earlier + # index, which would under-extend allocations defined + # inside the loop and let them wrongly share LDS). + loop_last = idx + _subtree_size(op) - 1 + walk(r.ops, loop_end=loop_last) + else: + walk(r.ops, loop_end=loop_end) + + walk(region.ops, loop_end=None) + return intervals + + def _compute_smem_layout(self) -> None: + """Compute byte offsets for all smem allocations in a single pool. + + Called after ``_collect_smem`` and before ``lower_region``. + + Uses live-interval analysis to let non-interfering allocations share + the same LDS region. Two allocations *interfere* when their live + intervals overlap; overlapping allocations must occupy disjoint byte + ranges. Non-interfering allocations may reuse the same range, + reducing total LDS consumption. + + The packing algorithm is a greedy linear-scan: allocations are + processed in order of their live-interval start. For each allocation + we find the lowest free slot (a previously assigned range whose end + is before the current start and whose size is large enough), or open + a new slot at the end of the pool. + + Alignment is preserved: 16 bytes for byte-element types, 4 bytes + otherwise. The pool itself is rounded up to 16-byte alignment. + + Falls back to the original sequential packing when liveness analysis + yields no intervals (e.g. zero smem allocations). + """ + _elem_bytes = { + "i8": 1, + "fp8e4m3": 1, + "bf8e5m2": 1, + "f16": 2, + "bf16": 2, + "i32": 4, + "f32": 4, + "i64": 8, + } + + pool_name = f"@smem_pool.{self.kernel.name}" + self._smem_pool_name = pool_name + + if not self._smem_globals: + self._smem_pool_size = 0 + return + + # ---- compute per-allocation sizes and alignments ---- + def _seg_size(stype: "SmemType") -> int: + eb = _elem_bytes.get(stype.elem.name, 2) + seg = eb + for d in stype.shape: + seg *= d + return seg + + def _align(stype: "SmemType") -> int: + return 16 if stype.elem.name in ("i8", "fp8e4m3", "bf8e5m2") else 4 + + # ---- live intervals from the kernel body ---- + live = self._collect_smem_liveness(self.kernel.body) + + # Sort allocations by live-interval start (definition order is a good + # proxy; fall back to declaration order for allocations with no uses). + def _sort_key(item: Tuple[str, "SmemType"]) -> int: + gname, _ = item + return live.get(gname, (0, 0))[0] + + sorted_allocs = sorted(self._smem_globals, key=_sort_key) + + # ---- greedy interval packing ---- + # Each "slot" is (offset, size, last_seq) – the byte range it occupies + # and the latest sequence index at which it is still live. + slots: List[Tuple[int, int, int]] = [] # (offset, size, last_seq) + + for gname, stype in sorted_allocs: + seg = _seg_size(stype) + aln = _align(stype) + first_seq, last_seq = live.get(gname, (0, 0)) + + # Try to reuse any slot that is free before this allocation starts. + # A "free" slot (s_last < first_seq) provides a candidate base + # address: we place the new allocation at aligned(s_off), regardless + # of whether it fits within s_size. The allocation may extend + # beyond the slot's original footprint — that is intentional. + # + # Example: A (12 KB, live 0..3) and B (12 KB, live 1..3) are packed + # into slots [0,12K] and [12K,12K]. C (64 KB, live 5..10) is free + # to reuse slot A (starting at offset 0) even though 64 KB > 12 KB. + # The pool size becomes max(0+64K, 12K+12K) = 64 KB instead of + # 12K+12K+64K = 88 KB. + # + # Among free slots, prefer the one with the smallest aligned start + # (lowest address, cache-friendly, minimises pool fragmentation). + best: Optional[int] = None # index into slots[] + for i, (s_off, s_size, s_last) in enumerate(slots): + if s_last >= first_seq: + # Still live when we start – interference, skip. + continue + aligned_off = (s_off + aln - 1) & ~(aln - 1) + if best is None or aligned_off < (slots[best][0] + aln - 1) & ~( + aln - 1 + ): + best = i + + if best is not None: + s_off, s_size, _ = slots[best] + aligned_off = (s_off + aln - 1) & ~(aln - 1) + self._smem_offsets[gname] = aligned_off + # Expand the slot to cover the new allocation if it overflows. + new_size = max(s_size, aligned_off - s_off + seg) + slots[best] = (s_off, new_size, last_seq) + else: + # No reusable slot – open a new one at the end of the pool. + current_end = max( + (s_off + s_size for s_off, s_size, _ in slots), default=0 + ) + aligned_off = (current_end + aln - 1) & ~(aln - 1) + self._smem_offsets[gname] = aligned_off + slots.append((aligned_off, seg, last_seq)) + + pool_size = max(s_off + s_size for s_off, s_size, _ in slots) + self._smem_pool_size = (pool_size + 15) & ~15 + + def _emit_smem_base_ptr(self, gname: str, stype: SmemType) -> str: + """Return an addrspace(3) pointer to the start of the smem segment. + + When the segment sits at offset 0 in the pool, returns the pool name + directly (no extra GEP instruction). Otherwise emits one byte-level + GEP and returns the fresh SSA name. + + The byte offset is a compile-time constant, so the base pointer is + cached per (block, allocation): repeated accesses to the same segment + within one block reuse a single GEP. Keying on the current block keeps + every reuse dominance-safe -- instructions within a block execute + sequentially, so the cached value always dominates its later uses. + """ + offset = self._smem_offsets[gname] + if offset == 0: + return self._smem_pool_name + key = (self._current().label, gname) + cached = self._smem_base_cache.get(key) + if cached is not None: + return cached + base = self._fresh("smem_base") + self._current().emit( + f" {base} = getelementptr inbounds i8, ptr addrspace(3) " + f"{self._smem_pool_name}, i32 {offset}" + ) + self._smem_base_cache[key] = base + return base + # ----- per-op lowerings ----- def lower_op(self, op: Op) -> None: @@ -1881,11 +2102,12 @@ def _op_tile_smem_store(self, op: Op) -> None: indices = op.operands[1:-1] value = op.operands[-1] gname, stype = self._smem_global_name(smem) + base_ptr = self._emit_smem_base_ptr(gname, stype) gep = self._fresh("gep") gidx = ["i32 0"] + [f"i32 {self._operand(i)}" for i in indices] agg_ty = _smem_storage_type(stype) self._current().emit( - f" {gep} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {gname}, " + f" {gep} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {base_ptr}, " f"{', '.join(gidx)}" ) # Alignment is the element byte size: 1 for i8, 2 for f16/bf16, @@ -1919,11 +2141,12 @@ def _op_tile_lds_atomic_add(self, op: Op) -> None: val = op.operands[-1] elem_ty = _llvm_type(val.type) gname, stype = self._smem_global_name(smem) + base_ptr = self._emit_smem_base_ptr(gname, stype) agg_ty = _smem_storage_type(stype) gep = self._fresh("gep") gidx = ["i32 0"] + [f"i32 {self._operand(i)}" for i in indices] self._current().emit( - f" {gep} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {gname}, " + f" {gep} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {base_ptr}, " f"{', '.join(gidx)}" ) ordering = op.attrs.get("ordering", "monotonic") @@ -1945,11 +2168,12 @@ def _op_tile_smem_store_vN(self, op: Op) -> None: value = op.operands[-1] vec = int(op.attrs["vec"]) gname, stype = self._smem_global_name(smem) + base_ptr = self._emit_smem_base_ptr(gname, stype) agg_ty = _smem_storage_type(stype) gep = self._fresh("gep") gidx = ["i32 0"] + [f"i32 {self._operand(i)}" for i in indices] self._current().emit( - f" {gep} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {gname}, " + f" {gep} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {base_ptr}, " f"{', '.join(gidx)}" ) elem_ty = _llvm_type(value.type.elem) # type: ignore[attr-defined] @@ -1974,10 +2198,11 @@ def _op_tile_smem_store_vN(self, op: Op) -> None: def _op_tile_smem_load_v4(self, op: Op) -> None: smem, row, col = op.operands gname, stype = self._smem_global_name(smem) + base_ptr = self._emit_smem_base_ptr(gname, stype) agg_ty = _smem_storage_type(stype) base = self._fresh("smem.base") self._current().emit( - f" {base} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {gname}, " + f" {base} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {base_ptr}, " f"i32 0, i32 {self._operand(row)}, i32 {self._operand(col)}" ) # 4 contiguous fp16 loads + insertelement chain. We do separate @@ -2013,11 +2238,12 @@ def _op_tile_smem_load_vN(self, op: Op) -> None: indices = list(op.operands[1:]) vec = int(op.attrs["vec"]) gname, stype = self._smem_global_name(smem) + base_ptr = self._emit_smem_base_ptr(gname, stype) agg_ty = _smem_storage_type(stype) base = self._fresh("smem.base") idx_strs = ["i32 0"] + [f"i32 {self._operand(i)}" for i in indices] self._current().emit( - f" {base} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {gname}, " + f" {base} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {base_ptr}, " f"{', '.join(idx_strs)}" ) elem_ty = _llvm_type(op.result.type.elem) # type: ignore[attr-defined] @@ -2420,6 +2646,7 @@ def _op_tile_smem_store_distributed(self, op: Op) -> None: values = op.operands[1] n = values.type.count if isinstance(values.type, VectorType) else 1 gname, stype = self._smem_global_name(smem) + base_ptr = self._emit_smem_base_ptr(gname, stype) agg_ty = _smem_storage_type(stype) elem_ty = ( _llvm_type(values.type.elem) @@ -2435,7 +2662,7 @@ def _op_tile_smem_store_distributed(self, op: Op) -> None: ) self._current().emit( f" {gep} = getelementptr inbounds {agg_ty}, " - f"ptr addrspace(3) {gname}, i32 0, i32 {i}" + f"ptr addrspace(3) {base_ptr}, i32 0, i32 {i}" ) self._current().emit( f" store {elem_ty} {ev}, ptr addrspace(3) {gep}, align 2" @@ -2801,11 +3028,12 @@ def _op_tile_ds_read_tr16_b64(self, op: Op) -> None: smem = op.operands[0] indices = list(op.operands[1:]) gname, stype = self._smem_global_name(smem) + base_ptr = self._emit_smem_base_ptr(gname, stype) agg_ty = _smem_storage_type(stype) base = self._fresh("tr.base") idx_strs = ["i32 0"] + [f"i32 {self._operand(i)}" for i in indices] self._current().emit( - f" {base} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {gname}, " + f" {base} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {base_ptr}, " f"{', '.join(idx_strs)}" ) self._need("ds.read.tr16.b64") @@ -2830,11 +3058,12 @@ def _op_tile_ds_read_tr16_b128(self, op: Op) -> None: smem = op.operands[0] indices = list(op.operands[1:]) gname, stype = self._smem_global_name(smem) + base_ptr = self._emit_smem_base_ptr(gname, stype) agg_ty = _smem_storage_type(stype) base = self._fresh("trw.base") idx_strs = ["i32 0"] + [f"i32 {self._operand(i)}" for i in indices] self._current().emit( - f" {base} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {gname}, " + f" {base} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {base_ptr}, " f"{', '.join(idx_strs)}" ) elem_name = op.attrs.get("elem_type", "f16") @@ -2924,11 +3153,12 @@ def _op_tile_ds_read_tr_b8(self, op: Op) -> None: smem = op.operands[0] indices = list(op.operands[1:]) gname, stype = self._smem_global_name(smem) + base_ptr = self._emit_smem_base_ptr(gname, stype) agg_ty = _smem_storage_type(stype) base = self._fresh("tr8.base") idx_strs = ["i32 0"] + [f"i32 {self._operand(i)}" for i in indices] self._current().emit( - f" {base} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {gname}, " + f" {base} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {base_ptr}, " f"{', '.join(idx_strs)}" ) addr = self._fresh("tr8.addr") @@ -3000,9 +3230,11 @@ def _op_arith_umul_hi_i32(self, op: Op) -> None: def _op_tile_smem_addr_of(self, op: Op) -> None: (smem,) = op.operands gname = self._smem_storage_name[smem.name] + stype = smem.type + base_ptr = self._emit_smem_base_ptr(gname, stype) # The global is ptr addrspace(3); cast to i64 for arithmetic. self._current().emit( - f" {op.result.name} = ptrtoint ptr addrspace(3) {gname} to i64" + f" {op.result.name} = ptrtoint ptr addrspace(3) {base_ptr} to i64" ) def _op_tile_smem_ptr_add(self, op: Op) -> None: @@ -3079,11 +3311,12 @@ def _op_tile_global_load_async_to_lds(self, op: Op) -> None: ) # Per-lane LDS destination address (typed aggregate GEP). gname, stype = self._smem_global_name(lds_smem) + base_ptr = self._emit_smem_base_ptr(gname, stype) agg_ty = _smem_storage_type(stype) gidx = ["i32 0"] + [f"i32 {self._operand(i)}" for i in lds_indices] gep_l = self._fresh("async_dst") self._current().emit( - f" {gep_l} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {gname}, " + f" {gep_l} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {base_ptr}, " f"{', '.join(gidx)}" ) self._need(f"global.load.async.to.lds.{suffix}") @@ -3656,11 +3889,12 @@ def _op_tile_smem_store_vN_f32(self, op: Op) -> None: value = op.operands[-1] vec = int(op.attrs["vec"]) gname, stype = self._smem_global_name(smem) + base_ptr = self._emit_smem_base_ptr(gname, stype) agg_ty = _smem_storage_type(stype) gep = self._fresh("gep") gidx = ["i32 0"] + [f"i32 {self._operand(i)}" for i in indices] self._current().emit( - f" {gep} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {gname}, " + f" {gep} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {base_ptr}, " f"{', '.join(gidx)}" ) align = vec * 4 @@ -3690,11 +3924,12 @@ def _op_tile_smem_load_vN_f32(self, op: Op) -> None: indices = list(op.operands[1:]) vec = int(op.attrs["vec"]) gname, stype = self._smem_global_name(smem) + base_ptr = self._emit_smem_base_ptr(gname, stype) agg_ty = _smem_storage_type(stype) base = self._fresh("smem.base") idx_strs = ["i32 0"] + [f"i32 {self._operand(i)}" for i in indices] self._current().emit( - f" {base} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {gname}, " + f" {base} = getelementptr inbounds {agg_ty}, ptr addrspace(3) {base_ptr}, " f"{', '.join(idx_strs)}" ) align = vec * 4 @@ -4355,28 +4590,21 @@ def finalize(self) -> str: out.append(f'target triple = "{self._backend.triple}"') out.append("") - # smem globals. - # ``align 4`` matches the natural alignment of f16/bf16/f32/i32 - # LDS storage and is what every 16 B ``ds_read_b128`` / - # ``ds_write_b128`` issued against them needs (the runtime - # offset math handles the per-row 16 B stride). The exception - # is fp8/bf8/i8 storage paired with ``ds_read_b64_tr_b8``: that - # intrinsic packs 8 bytes per lane and the AMDGPU backend - # requires the load address to be 8 B aligned; landing the i8 - # global on a 4 B boundary silently corrupts the b64 - # transpose-read output. Bump only the i8/fp8 globals to 16 B - # so the b64 transpose-read is always safe; leave fp16/f32 - # globals at align 4 (raising them would inflate occupancy - # pressure on long-prefill 3D kernels). - for gname, stype in self._smem_globals: - agg = _smem_storage_type(stype) - elem_name = stype.elem.name - elem_is_byte = elem_name in ("i8", "fp8e4m3", "bf8e5m2") - align = 16 if elem_is_byte else 4 + # smem pool: a single unified addrspace(3) global backing all smem + # allocations. Segments are placed by liveness-guided interval packing + # (see _compute_smem_layout): non-interfering allocations reuse the same + # byte range, so the pool holds max(overlapping segments) rather than the + # sum. Per-allocation alignment is preserved (4 bytes for f16/bf16/f32/ + # i32, 16 bytes for i8/fp8). + # Each _op_tile_smem_* method emits a byte-level GEP to its segment + # base before the typed aggregate GEP, so the typed addressing is + # unchanged. align 16 satisfies all segment alignments (the strictest + # is the 16-byte requirement for ds_read_b64_tr_b8 on i8/fp8 tiles). + if self._smem_globals: out.append( - f"{gname} = internal unnamed_addr addrspace(3) global {agg} poison, align {align}" + f"{self._smem_pool_name} = internal unnamed_addr addrspace(3) " + f"global [{self._smem_pool_size} x i8] poison, align 16" ) - if self._smem_globals: out.append("") # Intrinsic declarations actually used. ``self._decls`` is @@ -4621,6 +4849,7 @@ def _lower_kernel_to_llvm_python( """ lowerer = _Lowerer(kernel, llvm_flavor=llvm_flavor, arch=arch) lowerer._collect_smem(kernel.body) + lowerer._compute_smem_layout() lowerer.lower_region(kernel.body) return lowerer.finalize() diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/conv_implicit_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/conv_implicit_gemm.py index 7f8bffdfddfe..5eef5d762f3b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/conv_implicit_gemm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/conv_implicit_gemm.py @@ -573,7 +573,7 @@ def is_valid_spec(spec: ImplicitGemmConvSpec, arch: str = "gfx950") -> Tuple[boo _c_lds = ( spec.tile_m * spec.tile_n * _c_dtype_bytes if spec.epilogue == "cshuffle" else 0 ) - _total_lds = _ab_lds + _c_lds + _total_lds = max(_ab_lds, _c_lds) if not target.fits_lds(_total_lds): return False, ( f"LDS budget {_total_lds} bytes " diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_smem_liveness_packing.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_smem_liveness_packing.py new file mode 100644 index 000000000000..d4485933cf0f --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_smem_liveness_packing.py @@ -0,0 +1,308 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""Unit tests for LDS liveness analysis and interval-based smem packing. + +Tests cover: + - _collect_smem_liveness: correct live-interval computation for sequential + ops, nested scf.for loops, and mixed regions. + - _compute_smem_layout: interval packing produces the correct byte offsets + and pool size for non-interfering, interfering, and mixed allocations. +""" + +from __future__ import annotations + +import unittest + +from rocke.core.ir import F16, F32, I8, KernelDef, Op, Region, SmemType, Value +from rocke.core.lower_llvm import _Lowerer + + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- + + +def _smem(elem, *shape: int) -> SmemType: + return SmemType(elem, list(shape)) + + +def _make_op(name: str, operands=(), results=()): + op = Op(name=name, operands=list(operands), results=list(results)) + for r in results: + r.op = op + return op + + +def _alloc(name: str, stype: SmemType) -> tuple[Op, Value]: + v = Value(name=name, type=stype) + op = _make_op("tile.smem_alloc", results=[v]) + return op, v + + +def _use(*vals: Value) -> Op: + return _make_op("tile.lds_store", operands=list(vals)) + + +def _lowerer(*ops: Op) -> _Lowerer: + body = Region(label="body", ops=list(ops)) + kernel = KernelDef(name="k", params=[], body=body) + low = _Lowerer(kernel) + low._collect_smem(body) + return low + + +# byte sizes +_KB = 1024 +_12KB = 12 * _KB # f32 [128x24] +_24KB = 24 * _KB # two 12KB tiles +_64KB = 64 * _KB # f32 [128x128] (CShuffle) +_8KB = 8 * _KB # f16 [64x64] +_4KB = 4 * _KB # f32 [32x32] + + +# --------------------------------------------------------------------------- +# liveness tests +# --------------------------------------------------------------------------- + + +class TestSmemLiveness(unittest.TestCase): + + def _live(self, *ops: Op) -> dict: + low = _lowerer(*ops) + raw = low._collect_smem_liveness(low.kernel.body) + # Strip the kernel-name suffix for readability: "@A.k" -> "A" + return {k.lstrip("@").split(".")[0]: v for k, v in raw.items()} + + def test_single_alloc_no_uses(self): + op_a, _ = _alloc("%A", _smem(F16, 64, 64)) + live = self._live(op_a) + # Defined at seq 0, no downstream uses → last = 0 + self.assertEqual(live["A"], (0, 0)) + + def test_alloc_with_single_use(self): + op_a, va = _alloc("%A", _smem(F16, 64, 64)) + op_u = _use(va) + live = self._live(op_a, op_u) + # alloc at 0, use at 1 → (0, 1) + self.assertEqual(live["A"], (0, 1)) + + def test_two_allocs_disjoint(self): + # A used only in first half, C used only in second half + op_a, va = _alloc("%A", _smem(F32, 128, 24)) + op_b, vb = _alloc("%B", _smem(F32, 128, 24)) + op_u1 = _use(va, vb) + op_u2 = _use(va, vb) + op_c, vc = _alloc("%C", _smem(F32, 128, 128)) + op_u3 = _use(vc) + live = self._live(op_a, op_b, op_u1, op_u2, op_c, op_u3) + # A: def=0, last use=3; B: def=1, last use=3; C: def=4, last use=5 + self.assertEqual(live["A"], (0, 3)) + self.assertEqual(live["B"], (1, 3)) + self.assertEqual(live["C"], (4, 5)) + # A and C must not interfere (C starts after A ends) + a_first, a_last = live["A"] + c_first, c_last = live["C"] + self.assertGreater(c_first, a_last) + + def test_scf_for_extends_last_to_loop_end(self): + # A used inside a loop → last_seq extended to the loop's *last* preorder + # index (end of the scf.for subtree), not the for-op's own index. + op_a, va = _alloc("%A", _smem(F16, 64, 64)) + inner_use = _use(va) + loop_body = Region(label="body", ops=[inner_use]) + for_op = Op( + name="scf.for", + operands=[], + results=[], + regions=[loop_body], + ) + low = _lowerer(op_a, for_op) + raw = low._collect_smem_liveness(low.kernel.body) + live = {k.lstrip("@").split(".")[0]: v for k, v in raw.items()} + # seq: op_a=0, for_op=1, inner_use=2. The loop subtree spans [1, 2], so + # the use inside extends A's last_seq to 2 (the loop end). + self.assertEqual(live["A"][0], 0) + self.assertEqual(live["A"][1], 2) + + def test_alloc_inside_non_loop_region(self): + # alloc inside an if-region (not scf.for) — no conservative extension + op_a, va = _alloc("%A", _smem(F16, 64, 64)) + inner_use = _use(va) + if_body = Region(label="then", ops=[inner_use]) + if_op = Op(name="scf.if", operands=[], results=[], regions=[if_body]) + low = _lowerer(op_a, if_op) + raw = low._collect_smem_liveness(low.kernel.body) + live = {k.lstrip("@").split(".")[0]: v for k, v in raw.items()} + # scf.if is not a loop → use inside extends last normally (to inner idx) + # The inner use is at seq 2 (after op_a=0, if_op=1); but since + # loop_end=None for scf.if, last = inner idx = 2. + self.assertGreaterEqual(live["A"][1], 1) + + +# --------------------------------------------------------------------------- +# packing tests +# --------------------------------------------------------------------------- + + +class TestSmemPacking(unittest.TestCase): + + def _pack(self, *ops: Op): + low = _lowerer(*ops) + low._compute_smem_layout() + offsets = { + k.lstrip("@").split(".")[0]: low._smem_offsets[k] for k in low._smem_offsets + } + return offsets, low._smem_pool_size + + @staticmethod + def _overlaps(o1, s1, o2, s2) -> bool: + return o1 < o2 + s2 and o2 < o1 + s1 + + # -- single alloc --------------------------------------------------------- + + def test_single_alloc(self): + op_a, _ = _alloc("%A", _smem(F32, 128, 24)) + offsets, pool = self._pack(op_a) + self.assertEqual(offsets["A"], 0) + self.assertEqual(pool, (_12KB + 15) & ~15) + + # -- interfering allocs must not overlap ---------------------------------- + + def test_two_interfering_no_overlap(self): + op_a, va = _alloc("%A", _smem(F32, 128, 24)) # 12KB + op_b, vb = _alloc("%B", _smem(F32, 128, 24)) # 12KB + op_u = _use(va, vb) + offsets, pool = self._pack(op_a, op_b, op_u) + self.assertFalse( + self._overlaps(offsets["A"], _12KB, offsets["B"], _12KB), + "A and B interfere and must not share memory", + ) + # Pool = A + B = 24KB + self.assertEqual(pool, (_24KB + 15) & ~15) + + # -- non-interfering: C smaller than A+B ---------------------------------- + + def test_non_interfering_c_smaller_reuses_a_slot(self): + op_a, va = _alloc("%A", _smem(F16, 64, 64)) # 8KB + op_b, vb = _alloc("%B", _smem(F16, 64, 64)) # 8KB + op_u1 = _use(va, vb) + op_u2 = _use(va, vb) + op_c, vc = _alloc("%C", _smem(F32, 32, 32)) # 4KB — fits in A's slot + op_u3 = _use(vc) + offsets, pool = self._pack(op_a, op_b, op_u1, op_u2, op_c, op_u3) + # C must reuse slot A (offset 0) + self.assertEqual(offsets["C"], 0, "C should reuse A's slot at offset 0") + # Pool = max(A+B, C@0) = 16KB + self.assertEqual(pool, (2 * _8KB + 15) & ~15) + # A and B must not overlap + self.assertFalse(self._overlaps(offsets["A"], _8KB, offsets["B"], _8KB)) + + # -- non-interfering: C larger than A+B (the CShuffle case) -------------- + + def test_non_interfering_c_larger_reuses_a_slot_expands_pool(self): + """CShuffle (64KB) reuses A's slot even though it's larger than A (12KB). + + Before the fix the packer opened a fresh slot at offset 24KB + producing pool=88KB. After the fix C starts at offset 0 and + pool = max(A+B=24KB, C=64KB) = 64KB. + """ + op_a, va = _alloc("%A", _smem(F32, 128, 24)) # 12KB + op_b, vb = _alloc("%B", _smem(F32, 128, 24)) # 12KB + op_u1 = _use(va, vb) + op_u2 = _use(va, vb) + op_c, vc = _alloc("%C", _smem(F32, 128, 128)) # 64KB + op_u3 = _use(vc) + offsets, pool = self._pack(op_a, op_b, op_u1, op_u2, op_c, op_u3) + # C must sit at offset 0 + self.assertEqual(offsets["C"], 0, "C should reuse A's slot at offset 0") + # Pool = 64KB (not 88KB) + self.assertEqual(pool, (_64KB + 15) & ~15) + # A and B must not overlap each other + self.assertFalse(self._overlaps(offsets["A"], _12KB, offsets["B"], _12KB)) + + # -- all three interfere -------------------------------------------------- + + def test_three_interfering_allocs(self): + op_a, va = _alloc("%A", _smem(F16, 64, 64)) # 8KB + op_b, vb = _alloc("%B", _smem(F16, 64, 64)) # 8KB + op_c, vc = _alloc("%C", _smem(F16, 64, 64)) # 8KB + # All three are defined before the single op that reads them, so their + # live intervals overlap at the use and none may share memory. (Ops are + # passed in valid SSA order: every %value is defined before it is used.) + op_u = _use(va, vb, vc) + offsets, pool = self._pack(op_a, op_b, op_c, op_u) + for n1, o1, n2, o2 in [ + ("A", offsets["A"], "B", offsets["B"]), + ("A", offsets["A"], "C", offsets["C"]), + ("B", offsets["B"], "C", offsets["C"]), + ]: + self.assertFalse( + self._overlaps(o1, _8KB, o2, _8KB), + f"{n1} and {n2} interfere and must not overlap", + ) + # Three mutually interfering 8KB tiles → pool = 24KB + self.assertEqual(pool, (3 * _8KB + 15) & ~15) + + # -- double-buffer: A and A2 interfere with each other -------------------- + + def test_double_buffer_ping_pong(self): + # compv4 allocates A_smem and A_smem2 for ping-pong; both live in loop + op_a, va = _alloc("%A", _smem(F16, 64, 64)) # 8KB + op_a2, va2 = _alloc("%A2", _smem(F16, 64, 64)) # 8KB + op_b, vb = _alloc("%B", _smem(F16, 64, 64)) # 8KB + op_b2, vb2 = _alloc("%B2", _smem(F16, 64, 64)) # 8KB + # All four live simultaneously in the loop body + loop_use = _use(va, va2, vb, vb2) + loop_body = Region(label="body", ops=[loop_use]) + for_op = Op(name="scf.for", operands=[], results=[], regions=[loop_body]) + # CShuffle after the loop — non-interfering + op_c, vc = _alloc("%C", _smem(F32, 32, 32)) # 4KB + op_cu = _use(vc) + low = _lowerer(op_a, op_a2, op_b, op_b2, for_op, op_c, op_cu) + low._compute_smem_layout() + oA = low._smem_offsets["@A.k"] + oA2 = low._smem_offsets["@A2.k"] + oB = low._smem_offsets["@B.k"] + oB2 = low._smem_offsets["@B2.k"] + oC = low._smem_offsets["@C.k"] + pool = low._smem_pool_size + # None of the four loop tiles may overlap + for n1, o1, s1, n2, o2, s2 in [ + ("A", oA, _8KB, "A2", oA2, _8KB), + ("A", oA, _8KB, "B", oB, _8KB), + ("A", oA, _8KB, "B2", oB2, _8KB), + ("A2", oA2, _8KB, "B", oB, _8KB), + ("A2", oA2, _8KB, "B2", oB2, _8KB), + ("B", oB, _8KB, "B2", oB2, _8KB), + ]: + self.assertFalse( + self._overlaps(o1, s1, o2, s2), + f"{n1} and {n2} must not overlap", + ) + # Pool must be exactly 4 × 8KB = 32KB; C reuses one of those slots + self.assertEqual(pool, (4 * _8KB + 15) & ~15) + # C must sit within the existing pool (reusing a freed slot) + self.assertLess(oC + _4KB, pool + 1) + + # -- byte-element alignment (i8 gets 16-byte align) ----------------------- + + def test_i8_alignment(self): + op_a, va = _alloc("%A", _smem(I8, 128, 128)) # 16KB, align=16 + op_b, vb = _alloc("%B", _smem(F16, 64, 64)) # 8KB, align=4 + op_u = _use(va, vb) + offsets, pool = self._pack(op_a, op_b, op_u) + self.assertEqual(offsets["A"] % 16, 0, "i8 alloc must be 16-byte aligned") + self.assertEqual(offsets["B"] % 4, 0, "f16 alloc must be 4-byte aligned") + + # -- empty kernel --------------------------------------------------------- + + def test_no_allocs(self): + op_u = _make_op("arith.constant") + low = _lowerer(op_u) + low._compute_smem_layout() + self.assertEqual(low._smem_pool_size, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json index 16748c2188ae..72477cee7685 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json @@ -4,130 +4,130 @@ "cases": { "conv/gfx1151/n1h8c16k32r3": { "arch": "gfx1151", - "bytes": 42207, + "bytes": 41851, "family": "conv", "kernel_name": "irhash_conv_1151_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "2935996d148ecd5ca81c26dabe1709fe3f2452cf0eb48f390b4a995644e960cc", + "sha256": "0623a37a96570f36d43d580d35dfa2e84dfa58d943d57a9a0c526aac15a8cd30", "status": "ok" }, "conv/gfx1151/n2h16c32k32r1": { "arch": "gfx1151", - "bytes": 41464, + "bytes": 41098, "family": "conv", "kernel_name": "irhash_conv_1151_b_N2H16W16C32_K32Y1X1_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "6e0c2e544cd07e14b965fb20f4d7abb50a453d83e557cad16c34d740e16b955e", + "sha256": "cf8847b6e5bf9f648b6bfba91bfd9d9abb04353e88d638d32a1da00e0005edd7", "status": "ok" }, "conv/gfx1201/n1h8c16k32r3": { "arch": "gfx1201", - "bytes": 35644, + "bytes": 35427, "family": "conv", "kernel_name": "irhash_conv_1201_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "caa3d3d0a89fb2bf017882c47fc399a29e5578ff4f95f262581dd83c722cd259", + "sha256": "eeb98b0b876cdf2f6053f655ab42c073b90667d87c15d9608368ba4812d3b575", "status": "ok" }, "conv/gfx90a/n1h8c16k32r3": { "arch": "gfx90a", - "bytes": 28268, + "bytes": 28125, "family": "conv", "kernel_name": "irhash_conv_90a_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_default", - "sha256": "3b1d948fec0a4f85cbd3de335c67da69e3bfd083352c6d6e0735a570be699472", + "sha256": "898affe0899bcf215e9cbb897651592e8daa98a0e6039cddb637435ce89fdaca", "status": "ok" }, "conv/gfx942/n1h8c16k32r3": { "arch": "gfx942", - "bytes": 28268, + "bytes": 28125, "family": "conv", "kernel_name": "irhash_conv_942_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_default", - "sha256": "1bf6bcc471b265c4d5d77e3336b8af08c2fb43d5eecd3c4688b03cc2d025f241", + "sha256": "e762a81eff84ecf415016414d7aab5f8dbe12ac0bdc07921638d833dcfa56c48", "status": "ok" }, "conv/gfx950/n1h8c16k32r3": { "arch": "gfx950", - "bytes": 23809, + "bytes": 23082, "family": "conv", "kernel_name": "irhash_conv_950_a_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_compv4_cshuffle", - "sha256": "3c54d531e8a37aacfbe8892d2a1681dee2d3dc4e9e726b730d9cd121c5e1001c", + "sha256": "65521c817d5bcad307cd1afdeadfa706513097c0ff8c7073c71b2230f77ce6b9", "status": "ok" }, "conv/gfx950/n2h16c32k32r1": { "arch": "gfx950", - "bytes": 27055, + "bytes": 26908, "family": "conv", "kernel_name": "irhash_conv_950_b_N2H16W16C32_K32Y1X1_t64x32x16_w2x2_a16x16x16_mem_default", - "sha256": "7a7185fb2e138254bb76984776df6af35776dd39ee17cadf17732c2b148ee640", + "sha256": "62813fc954066fb34edaa921c39f26248fb7a865af2f4d66695af7af1f0c40e9", "status": "ok" }, "deep/common_gfx950_wt16/h16w16_k32": { "arch": "gfx950", - "bytes": 54745, + "bytes": 53250, "family": "deep_fused_conv", "kernel_name": "irhash_deep_gfx950_wt16_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "33a5419beb872edd0893d7a0b95bddf3b68901bd2c3d08d9ab0948a1f3b1f21c", + "sha256": "056511051ae3d983c326f353e61cecb8f521c673bcaa0fd6814b5e5205db529a", "status": "ok" }, "deep/common_gfx950_wt16/h24w24_k32": { "arch": "gfx950", - "bytes": 54804, + "bytes": 53299, "family": "deep_fused_conv", "kernel_name": "irhash_deep_gfx950_wt16b_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "73512001004a9b070c7b83396458b613db3e21aa757bff4a274cbf0e047ee169", + "sha256": "cf0354729d53507b82aac23b0d97618f1fa4921aa31bf2cc8a87c368f749edf3", "status": "ok" }, "deep/gfx1151/native_h16w16": { "arch": "gfx1151", - "bytes": 117991, + "bytes": 112141, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1151_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_K132_pool2x2_s2x2_t64x32_pt4x4_w4x2_wmma16x16x16_directa_nativeiu8_i8i4_realquant", - "sha256": "10f91836433e0572c673d15e32fb4992d7aa368d5833bb33b956a1534e0581db", + "sha256": "5b8932a7f5e08ab5d983a4adbfd1b8272902ddbd0c9d32db142283e58a983a14", "status": "ok" }, "deep/gfx1151/native_h24w24": { "arch": "gfx1151", - "bytes": 117992, + "bytes": 112142, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1151_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_K132_pool2x2_s2x2_t64x32_pt4x4_w4x2_wmma16x16x16_directa_nativeiu8_i8i4_realquant", - "sha256": "2af06dfaa5b094805d0879112638e66ca583ab4344bf65adcd0d8e4a1e002df4", + "sha256": "ec326f771c868dd240f27d645056eed846ffe4355ae049a9c248f95335836396", "status": "ok" }, "deep/gfx11_generic/native_h16w16": { "arch": "gfx11-generic", - "bytes": 117991, + "bytes": 112141, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1151_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_K132_pool2x2_s2x2_t64x32_pt4x4_w4x2_wmma16x16x16_directa_nativeiu8_i8i4_realquant", - "sha256": "10f91836433e0572c673d15e32fb4992d7aa368d5833bb33b956a1534e0581db", + "sha256": "5b8932a7f5e08ab5d983a4adbfd1b8272902ddbd0c9d32db142283e58a983a14", "status": "ok" }, "deep/gfx1201/h16w16_k32": { "arch": "gfx1201", - "bytes": 74424, + "bytes": 72406, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_default_epi_relu", - "sha256": "f36fa0e17a2caaa1167143c0b02ebf5ff80c5e737753b5cd4136193629f2a60d", + "sha256": "e04852bbbcfd125ae3060f5c876366a662ceffda5f151e5cdf0f7c4ed5f2eb57", "status": "ok" }, "deep/gfx1201/h24w24_k32": { "arch": "gfx1201", - "bytes": 74425, + "bytes": 72407, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_default_epi_relu", - "sha256": "9b4a1cb81cae2ace6beb4a92b7490c002204a69d4d9466d5bc3459bceb4b7dc7", + "sha256": "71c6b87694919c000e9d96f1767c86669d9b58da80b70eaa2f4934477bb94f24", "status": "ok" }, "deep/gfx950/h16w16_k32": { "arch": "gfx950", - "bytes": 32995, + "bytes": 32107, "family": "deep_fused_conv", "kernel_name": "rocke_gfx950_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a32x32x16_mem_cshuffle_epi_relu", - "sha256": "320af416a9311ccdf470006f7258a8b21fbb1edf5c9834a2366c01f83d6c23df", + "sha256": "64cafece518e47ef2ef57a11673aead362059e0bccab16ff37c2c36ddae5af95", "status": "ok" }, "deep/gfx950/h24w24_k32": { "arch": "gfx950", - "bytes": 32999, + "bytes": 32111, "family": "deep_fused_conv", "kernel_name": "rocke_gfx950_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a32x32x16_mem_cshuffle_epi_relu", - "sha256": "e770c496c2c0b90b8ba7cde6ccc837464c4d1dbbc3bc23ebe5ce6886dee043c3", + "sha256": "37a5f88d3876f8f67f70b5340bea6b63c412f36aa6217a6cc677da3e9e271943", "status": "ok" }, "fused_moe/gfx950/gather_h128": { @@ -156,130 +156,130 @@ }, "gemm/gfx1151/t32x32x16": { "arch": "gfx1151", - "bytes": 20529, + "bytes": 20299, "family": "gemm", "kernel_name": "irhash_gemm_1151_a_fp16_t32x32x16_w2x2x1_wt16x16x16_mem_intrawave_default_pad", - "sha256": "271414bd4e5f53cad2671356845140f60ca26698128b39138bbdb4378b2a27a0", + "sha256": "b7af0afe67afaa6906172a7224367ed6e47ea234499e65fbe7747f4a1e9a5283", "status": "ok" }, "gemm/gfx1151/t64x32x16": { "arch": "gfx1151", - "bytes": 55402, + "bytes": 55029, "family": "gemm", "kernel_name": "irhash_gemm_1151_b_fp16_t64x32x16_w2x1x1_wt16x16x16_mem_intrawave_default_pad", - "sha256": "6655ed483bb86db72ae7986c05535ec003d0850cb23252f1f47281ca1e6f3fc9", + "sha256": "75f5790686710edf7aa8f6bcc7667d84e54f4d66d99c38bf7b3e0f432fbc31c5", "status": "ok" }, "gemm/gfx1201/t32x32x16": { "arch": "gfx1201", - "bytes": 16250, + "bytes": 16093, "family": "gemm", "kernel_name": "irhash_gemm_1201_a_fp16_t32x32x16_w2x2x1_wt16x16x16_mem_intrawave_default_pad", - "sha256": "1787651c0418d42f3f367d0e90080cc8c49f8f7dabba3411f35a58589f79c5b8", + "sha256": "8f9486e13d6da94d1d07fbf181b257f6fe16f1296909475b24817502d6cc4ef5", "status": "ok" }, "gemm/gfx1201/t64x32x16": { "arch": "gfx1201", - "bytes": 47045, + "bytes": 46816, "family": "gemm", "kernel_name": "irhash_gemm_1201_b_fp16_t64x32x16_w2x1x1_wt16x16x16_mem_intrawave_default_pad", - "sha256": "5330a7de1b72649799d3f252c587377c4e2fccdeb691cc4a7a00ce7fc0d37e5b", + "sha256": "7e4d31fe3b8a0a45f6f6196c9967be4a4a98af3ce9c6592eecbfdace6e8114a6", "status": "ok" }, "gemm/gfx1250/t32x32x32": { "arch": "gfx1250", - "bytes": 26791, + "bytes": 26563, "family": "gemm", "kernel_name": "irhash_gemm_1250_a_fp16_t32x32x32_w2x2x1_wt16x16x32_mem_intrawave_default_pad", - "sha256": "4c80bff2b4831a5b2b02494618cdfbc9080b8e9d04e695492f151aaa90f1213a", + "sha256": "5880def25dd5bf9841b74cc10bdbf4a54b6df88049b98f7874996a936024f7c8", "status": "ok" }, "gemm/gfx1250/t64x32x32": { "arch": "gfx1250", - "bytes": 76044, + "bytes": 75599, "family": "gemm", "kernel_name": "irhash_gemm_1250_b_fp16_t64x32x32_w2x1x1_wt16x16x32_mem_intrawave_default_pad", - "sha256": "7c447fc5bb3fd15fd8d6dd21caf83433e95c82cc5c7d2dc8c3d51626cabd0cac", + "sha256": "cfae1d3c9cbd049460ec97617e33664b960f68cb2e146e31ef2a6d237cd2cc1e", "status": "ok" }, "gemm/gfx90a/t128x128x16": { "arch": "gfx90a", - "bytes": 89792, + "bytes": 89223, "family": "gemm", "kernel_name": "irhash_gemm_90a_b_fp16_t128x128x16_w2x2x1_wt32x32x8_compv4_intrawave_cshuffle_pad", - "sha256": "0e8a9f3d06d1a38b2797a84a0cd09c3ff3b0a2f93ec6d813e8a58379e1e0c154", + "sha256": "580f9b19159acb58b4f4d6b8be80273cc00568156403f8c45c59a484e35ea987", "status": "ok" }, "gemm/gfx90a/t64x64x16": { "arch": "gfx90a", - "bytes": 25482, + "bytes": 25255, "family": "gemm", "kernel_name": "irhash_gemm_90a_a_fp16_t64x64x16_w2x2x1_wt16x16x16_mem_intrawave_default_pad", - "sha256": "c329c4b8a220883c0d5cc8591b3890379f9885254467ddc9a718dd7d59684cf0", + "sha256": "cac01baf9cf21bf33f1f7f4edc1bd1f516238209d836e6808fdf8c1d46a497de", "status": "ok" }, "gemm/gfx942/t128x128x16": { "arch": "gfx942", - "bytes": 89792, + "bytes": 89223, "family": "gemm", "kernel_name": "irhash_gemm_942_b_fp16_t128x128x16_w2x2x1_wt32x32x8_compv4_intrawave_cshuffle_pad", - "sha256": "b78c84e8861f63fa1349d72734f821906dd67425ac870b9c85bcbffad7cc2748", + "sha256": "46245c54e3f449c1e981655942bc90dc26a4d30fefeb14e0dd5c790c9b006bd7", "status": "ok" }, "gemm/gfx942/t64x64x16": { "arch": "gfx942", - "bytes": 25482, + "bytes": 25255, "family": "gemm", "kernel_name": "irhash_gemm_942_a_fp16_t64x64x16_w2x2x1_wt16x16x16_mem_intrawave_default_pad", - "sha256": "828a5c0129f043e9a874ffbe264a2314c47fd70620bf9c985582ba92ae1c9ff0", + "sha256": "1fead39b4190fcce18ceb0291fe40a352416607b75060a76a19578afaf756554", "status": "ok" }, "gemm/gfx950/t128x128x32": { "arch": "gfx950", - "bytes": 94865, + "bytes": 94212, "family": "gemm", "kernel_name": "irhash_gemm_950_a_fp16_t128x128x32_w2x2x1_wt32x32x16_compv4_intrawave_cshuffle_pad", - "sha256": "70734674e75049eeb9dd4eb88fd4bfc48a82b1d292032020e71d083c124f120b", + "sha256": "4b80dfe052894057034bda57a68ee237504d1a79916e86cb52969a882d5b4a88", "status": "ok" }, "gemm/gfx950/t64x128x32": { "arch": "gfx950", - "bytes": 45184, + "bytes": 44732, "family": "gemm", "kernel_name": "irhash_gemm_950_b_fp16_t64x128x32_w2x2x1_wt16x16x32_mem_intrawave_default_pad", - "sha256": "19c9640bad632bfd5e74a605662d6474d2df1649094b0e21ff32c501552fed0e", + "sha256": "d4e20421032922c0cdd0b88c1339758e1341de8897e8e4a54d119498b5a1119e", "status": "ok" }, "moe_sort/gfx1151/hist_t32_k2_e8": { "arch": "gfx1151", - "bytes": 3010, + "bytes": 3011, "family": "moe", "kernel_name": "irhash_moe_sort_histogram_hist_T32_K2_E8_b128", - "sha256": "2a345008540988c9341e01fe3fd20efbbf9548805f1918fc4e8626f88840fc69", + "sha256": "95b941eedb11d21181a97a5250fa44a219e1f2ad53351ef7f49714ed80327c9b", "status": "ok" }, "moe_sort/gfx1201/hist_t32_k2_e8": { "arch": "gfx1201", - "bytes": 3010, + "bytes": 3011, "family": "moe", "kernel_name": "irhash_moe_sort_histogram_hist_T32_K2_E8_b128", - "sha256": "2a345008540988c9341e01fe3fd20efbbf9548805f1918fc4e8626f88840fc69", + "sha256": "95b941eedb11d21181a97a5250fa44a219e1f2ad53351ef7f49714ed80327c9b", "status": "ok" }, "moe_sort/gfx942/hist_t32_k2_e8": { "arch": "gfx942", - "bytes": 3014, + "bytes": 3015, "family": "moe", "kernel_name": "irhash_moe_sort_histogram_hist_T32_K2_E8_b128", - "sha256": "1e7c3b23fe17eef71537b7c6855a700ecc77b21e5bfe1f0ed5450241760e4e69", + "sha256": "064bde15679ca808fff12e8f0e87c8bc6fb0f504203063a8087d3737f5aa3189", "status": "ok" }, "moe_sort/gfx950/hist_t32_k2_e8": { "arch": "gfx950", - "bytes": 3014, + "bytes": 3015, "family": "moe", "kernel_name": "irhash_moe_sort_histogram_hist_T32_K2_E8_b128", - "sha256": "1e7c3b23fe17eef71537b7c6855a700ecc77b21e5bfe1f0ed5450241760e4e69", + "sha256": "064bde15679ca808fff12e8f0e87c8bc6fb0f504203063a8087d3737f5aa3189", "status": "ok" }, "moe_sort/gfx950/scan_t64_k2_e16": { @@ -324,130 +324,130 @@ "cases": { "conv/gfx1151/n1h8c16k32r3": { "arch": "gfx1151", - "bytes": 42350, + "bytes": 41994, "family": "conv", "kernel_name": "irhash_conv_1151_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "bad2268874f192e079cad477b9b2c8efea79cbd094a8793d3c5ab700727ab5d2", + "sha256": "ce1c975f22e27bc0087ad54cd6ef72ee99f72d2acfc2642196ce766cfe7e23dc", "status": "ok" }, "conv/gfx1151/n2h16c32k32r1": { "arch": "gfx1151", - "bytes": 41607, + "bytes": 41241, "family": "conv", "kernel_name": "irhash_conv_1151_b_N2H16W16C32_K32Y1X1_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "3e74b7a7a8f4986c711ba4fdbb0015a2bb3541feb0610773b9fc1cbb3df60993", + "sha256": "e5d18c62b8b8ead0c460cb67546e3f9c85e04a50484ab0f054d87fae739d0aa3", "status": "ok" }, "conv/gfx1201/n1h8c16k32r3": { "arch": "gfx1201", - "bytes": 35787, + "bytes": 35570, "family": "conv", "kernel_name": "irhash_conv_1201_a_N1H8W8C16_K32Y3X3_t32x32x16_w2x1_a16x16x16_mem_default", - "sha256": "0b31e14adb6a7a4ab232132de703148aec6eda32992a3dadb80279c3de6135dd", + "sha256": "9f792368e733062bd8907c8d16f7f38e6b47fd0b4524bdf3aa92653e107d3c86", "status": "ok" }, "conv/gfx90a/n1h8c16k32r3": { "arch": "gfx90a", - "bytes": 28411, + "bytes": 28268, "family": "conv", "kernel_name": "irhash_conv_90a_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_default", - "sha256": "5472ca6bb7df916f445f61e30197e44f5426bfb5253634f3c01bb61a10f1d33c", + "sha256": "02edec2f222500f680576a275acf9ed89af7a7784763312e8427aea4197f5c8e", "status": "ok" }, "conv/gfx942/n1h8c16k32r3": { "arch": "gfx942", - "bytes": 28411, + "bytes": 28268, "family": "conv", "kernel_name": "irhash_conv_942_a_N1H8W8C16_K32Y3X3_t64x32x16_w2x2_a16x16x16_mem_default", - "sha256": "62a87c4bf5df27dda905b5c6902cd4f3d81c0c93eb4fd47438166564b7263b80", + "sha256": "46c25343a58ddfc5652ecc198be65bc6469e1700eceacea5af7756ee21e9db8f", "status": "ok" }, "conv/gfx950/n1h8c16k32r3": { "arch": "gfx950", - "bytes": 23946, + "bytes": 23219, "family": "conv", "kernel_name": "irhash_conv_950_a_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_compv4_cshuffle", - "sha256": "8823525336481fc1c2271f4d883c127654a25c9a5cf5c52a4aede5be7b50abd6", + "sha256": "1fe15959b07de36cfb8bd47f8e2d9a5047b50c48368ec221b365d78f4c0b922b", "status": "ok" }, "conv/gfx950/n2h16c32k32r1": { "arch": "gfx950", - "bytes": 27198, + "bytes": 27051, "family": "conv", "kernel_name": "irhash_conv_950_b_N2H16W16C32_K32Y1X1_t64x32x16_w2x2_a16x16x16_mem_default", - "sha256": "73cc4739fc0d9f1e278a3ee55e22a70ed433aca8f9c0e2876908d121d9e52720", + "sha256": "5a11a7d4cb0c58289e579662429dbcb9cf87217fabd408f621ccaeb7255b3a24", "status": "ok" }, "deep/common_gfx950_wt16/h16w16_k32": { "arch": "gfx950", - "bytes": 54931, + "bytes": 53436, "family": "deep_fused_conv", "kernel_name": "irhash_deep_gfx950_wt16_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "ee35714d4d223e5ffb61b754481dcbd4ffe74b3318582ecc5e3f9ae7b619ab53", + "sha256": "8903d08364a127d00d15a43d0a55e798d3f74fb37bcb8dd13800fe72582c1beb", "status": "ok" }, "deep/common_gfx950_wt16/h24w24_k32": { "arch": "gfx950", - "bytes": 54990, + "bytes": 53485, "family": "deep_fused_conv", "kernel_name": "irhash_deep_gfx950_wt16b_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "8be6a606506510cbf7e2b3862bb61e5afba32236e0696c83a27dbd6966398e2c", + "sha256": "605a94a47f380c3e7bf4546f5e5f57dc9f7d733feb8f87dd9b525d13d39e670a", "status": "ok" }, "deep/gfx1151/native_h16w16": { "arch": "gfx1151", - "bytes": 118002, + "bytes": 112152, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1151_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_K132_pool2x2_s2x2_t64x32_pt4x4_w4x2_wmma16x16x16_directa_nativeiu8_i8i4_realquant", - "sha256": "8a9d52e573063c3ebb31d1a96ffdb3c3745428796a5368624b0a9d307893a048", + "sha256": "fcf612ac9a9ea72bc36e7f9f0cb0a0f72f0b6f9a2b8341ce25e8f1306a40a039", "status": "ok" }, "deep/gfx1151/native_h24w24": { "arch": "gfx1151", - "bytes": 118003, + "bytes": 112153, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1151_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_K132_pool2x2_s2x2_t64x32_pt4x4_w4x2_wmma16x16x16_directa_nativeiu8_i8i4_realquant", - "sha256": "734f018aca5c15c64ba11e4cc38264154438adff59261f9d6c41df6e97434142", + "sha256": "b9633b0f9451a24bcc8726b0e8ea5fd853414631c769ba12afe5f1e5e9238169", "status": "ok" }, "deep/gfx11_generic/native_h16w16": { "arch": "gfx11-generic", - "bytes": 118002, + "bytes": 112152, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1151_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_K132_pool2x2_s2x2_t64x32_pt4x4_w4x2_wmma16x16x16_directa_nativeiu8_i8i4_realquant", - "sha256": "8a9d52e573063c3ebb31d1a96ffdb3c3745428796a5368624b0a9d307893a048", + "sha256": "fcf612ac9a9ea72bc36e7f9f0cb0a0f72f0b6f9a2b8341ce25e8f1306a40a039", "status": "ok" }, "deep/gfx1201/h16w16_k32": { "arch": "gfx1201", - "bytes": 74610, + "bytes": 72597, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_default_epi_relu", - "sha256": "7093bcdfc900c36ab5962fcb759f189c6ff81d7ce15374a22dec07a9424811c0", + "sha256": "9fc4c18f497636e1c23f6f2a8cbf8ba9fa8b386c987215db965bfba79d59db98", "status": "ok" }, "deep/gfx1201/h24w24_k32": { "arch": "gfx1201", - "bytes": 74611, + "bytes": 72598, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_default_epi_relu", - "sha256": "f46ab0ab42d17f37bf84b7b0a85c88abe5e886d09a4f21b2b31f879f020da4f8", + "sha256": "ac8d72a94bd835b0dc1accb893bd15da45ca353da0e06b1e181b65cdb8cf680d", "status": "ok" }, "deep/gfx950/h16w16_k32": { "arch": "gfx950", - "bytes": 33173, + "bytes": 32285, "family": "deep_fused_conv", "kernel_name": "rocke_gfx950_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a32x32x16_mem_cshuffle_epi_relu", - "sha256": "8e5c17e4eee71c67a85d20f92b7317f44aa90b8c71676219b32a71728020a919", + "sha256": "dd878e5603c4c851e46e2c3d44f71737b4d77aac33ad2ead4137655a1319c4a4", "status": "ok" }, "deep/gfx950/h24w24_k32": { "arch": "gfx950", - "bytes": 33177, + "bytes": 32289, "family": "deep_fused_conv", "kernel_name": "rocke_gfx950_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a32x32x16_mem_cshuffle_epi_relu", - "sha256": "d1c7bd604f14da107ee471a424b7b5fc4230b539b41e53ddd8875a479b71c49c", + "sha256": "17c58019efdde001e5d85ab6c528aca4a22267c7a45e67b1ca9854436dfcde8e", "status": "ok" }, "fused_moe/gfx950/gather_h128": { @@ -476,130 +476,130 @@ }, "gemm/gfx1151/t32x32x16": { "arch": "gfx1151", - "bytes": 20540, + "bytes": 20310, "family": "gemm", "kernel_name": "irhash_gemm_1151_a_fp16_t32x32x16_w2x2x1_wt16x16x16_mem_intrawave_default_pad", - "sha256": "43940c721b65ac62c85f3a01ecdd27b1dde7c391f048a2d5e757a1a03b7b3f6a", + "sha256": "9de9a2b40af45bd2097a9b648d76f3641161b83e38d53a83682f9ffabf202015", "status": "ok" }, "gemm/gfx1151/t64x32x16": { "arch": "gfx1151", - "bytes": 55413, + "bytes": 55040, "family": "gemm", "kernel_name": "irhash_gemm_1151_b_fp16_t64x32x16_w2x1x1_wt16x16x16_mem_intrawave_default_pad", - "sha256": "b470acf2e9505c28ef3fe1dbd4be1487bc859a821f8cc067d03ba155ae434b9d", + "sha256": "7118754b22d6eeb1eefeeeaae4c7a9912fb6ea0eb82fe47f35123aca5ceeaa2b", "status": "ok" }, "gemm/gfx1201/t32x32x16": { "arch": "gfx1201", - "bytes": 16261, + "bytes": 16104, "family": "gemm", "kernel_name": "irhash_gemm_1201_a_fp16_t32x32x16_w2x2x1_wt16x16x16_mem_intrawave_default_pad", - "sha256": "185b583668a60a7acb5bc123e71090b02fa1de733c8ec8cf5f8f9ad2275cf234", + "sha256": "704cfc383b7531093c1638258552af60ae282abc4b3c0e462a45f4628bb26397", "status": "ok" }, "gemm/gfx1201/t64x32x16": { "arch": "gfx1201", - "bytes": 47056, + "bytes": 46827, "family": "gemm", "kernel_name": "irhash_gemm_1201_b_fp16_t64x32x16_w2x1x1_wt16x16x16_mem_intrawave_default_pad", - "sha256": "2dd0ae7e294e2806d52a4aa43e8d3b2289a73dd273e92054c4dc1f1be4118d8f", + "sha256": "4602cb57a8b57916659d91de9dd76756616541649600d7c8596d4073c39f48c4", "status": "ok" }, "gemm/gfx1250/t32x32x32": { "arch": "gfx1250", - "bytes": 26802, + "bytes": 26574, "family": "gemm", "kernel_name": "irhash_gemm_1250_a_fp16_t32x32x32_w2x2x1_wt16x16x32_mem_intrawave_default_pad", - "sha256": "599664a4331022ce8899ab09f8bf5138371c7fdb986d1a4819973272879d0b5c", + "sha256": "ae485165ef0f4b6061839695443568567b958a9644f17b5937772e4eba7bd2c0", "status": "ok" }, "gemm/gfx1250/t64x32x32": { "arch": "gfx1250", - "bytes": 76055, + "bytes": 75610, "family": "gemm", "kernel_name": "irhash_gemm_1250_b_fp16_t64x32x32_w2x1x1_wt16x16x32_mem_intrawave_default_pad", - "sha256": "75b55ca4f50b6f52e9f52198fa8ebd57abab846d61fd03d19d398a41d511e8bf", + "sha256": "1bc884131b61b51b7b19806ef8ab9ca8dd7d82901c1d0abe9242671c93aaa5f3", "status": "ok" }, "gemm/gfx90a/t128x128x16": { "arch": "gfx90a", - "bytes": 89803, + "bytes": 89234, "family": "gemm", "kernel_name": "irhash_gemm_90a_b_fp16_t128x128x16_w2x2x1_wt32x32x8_compv4_intrawave_cshuffle_pad", - "sha256": "1c729eed6beb990f37ba598031be384a02d2bcf25d8c0447e45fb331d1a70c1d", + "sha256": "d2dc79ab69083cae0dac3b088c8b78f2a571c69e08c22bf034a3f232328fc50a", "status": "ok" }, "gemm/gfx90a/t64x64x16": { "arch": "gfx90a", - "bytes": 25493, + "bytes": 25266, "family": "gemm", "kernel_name": "irhash_gemm_90a_a_fp16_t64x64x16_w2x2x1_wt16x16x16_mem_intrawave_default_pad", - "sha256": "8e546f0adf374c1ca392f9dfca430c264e0a79ecde42f1816af6cf1becf24e93", + "sha256": "24e533210b8c4491712e900406c281f7eb8fe15dc387d7e12b00009d56d5c8ff", "status": "ok" }, "gemm/gfx942/t128x128x16": { "arch": "gfx942", - "bytes": 89803, + "bytes": 89234, "family": "gemm", "kernel_name": "irhash_gemm_942_b_fp16_t128x128x16_w2x2x1_wt32x32x8_compv4_intrawave_cshuffle_pad", - "sha256": "0371d377e4ae958c1cfbaaa42d2b1cae9a9808c9aa29d54a50638dba3eb27e5c", + "sha256": "56730e8a4790816a0219a9cbd59b017240641e9ab030aefded6da4d3298d35e1", "status": "ok" }, "gemm/gfx942/t64x64x16": { "arch": "gfx942", - "bytes": 25493, + "bytes": 25266, "family": "gemm", "kernel_name": "irhash_gemm_942_a_fp16_t64x64x16_w2x2x1_wt16x16x16_mem_intrawave_default_pad", - "sha256": "e394e329db1cc020826aec93896a4af6c6606d0112a5355c7569e9cb59052a72", + "sha256": "9140d3a20af35dd082903dfe1db8b7b58ff906a2aa1dc077cb5a589767fe6566", "status": "ok" }, "gemm/gfx950/t128x128x32": { "arch": "gfx950", - "bytes": 94876, + "bytes": 94223, "family": "gemm", "kernel_name": "irhash_gemm_950_a_fp16_t128x128x32_w2x2x1_wt32x32x16_compv4_intrawave_cshuffle_pad", - "sha256": "f34a9df726fb25388f20f93cf418de9d0fdf2ea688c2c99d3a501e9599ca9b3b", + "sha256": "3cf0a4dc05cd0de38d85454301636a3b4e494283150925fa8918c87990b3ceae", "status": "ok" }, "gemm/gfx950/t64x128x32": { "arch": "gfx950", - "bytes": 45195, + "bytes": 44743, "family": "gemm", "kernel_name": "irhash_gemm_950_b_fp16_t64x128x32_w2x2x1_wt16x16x32_mem_intrawave_default_pad", - "sha256": "59c7c2c2b8e96e76da5536bc430aadd082b5c49677ec61c84b86e676de0228bb", + "sha256": "f6f9ba500d6695ca21a1711c63e6cea4ad98875d8a478adfac96269c7005cee1", "status": "ok" }, "moe_sort/gfx1151/hist_t32_k2_e8": { "arch": "gfx1151", - "bytes": 3021, + "bytes": 3022, "family": "moe", "kernel_name": "irhash_moe_sort_histogram_hist_T32_K2_E8_b128", - "sha256": "0d5b4af3958ccb5439edb2630748e7d8537eeba297a35fc89296e8c97c1cfbe7", + "sha256": "32da3faf5c92a075224f1328ff83369eb7deef99dee64de27d9506ff671cdf68", "status": "ok" }, "moe_sort/gfx1201/hist_t32_k2_e8": { "arch": "gfx1201", - "bytes": 3021, + "bytes": 3022, "family": "moe", "kernel_name": "irhash_moe_sort_histogram_hist_T32_K2_E8_b128", - "sha256": "0d5b4af3958ccb5439edb2630748e7d8537eeba297a35fc89296e8c97c1cfbe7", + "sha256": "32da3faf5c92a075224f1328ff83369eb7deef99dee64de27d9506ff671cdf68", "status": "ok" }, "moe_sort/gfx942/hist_t32_k2_e8": { "arch": "gfx942", - "bytes": 3025, + "bytes": 3026, "family": "moe", "kernel_name": "irhash_moe_sort_histogram_hist_T32_K2_E8_b128", - "sha256": "7ce004168512067a87a975caf1117d7ff5e2e06aac9a56834b7032eb1532d2f5", + "sha256": "d4e0cba2f4ae527b14f144c57833521bb5b4b46ab8034a6acb239776eaa1077e", "status": "ok" }, "moe_sort/gfx950/hist_t32_k2_e8": { "arch": "gfx950", - "bytes": 3025, + "bytes": 3026, "family": "moe", "kernel_name": "irhash_moe_sort_histogram_hist_T32_K2_E8_b128", - "sha256": "7ce004168512067a87a975caf1117d7ff5e2e06aac9a56834b7032eb1532d2f5", + "sha256": "d4e0cba2f4ae527b14f144c57833521bb5b4b46ab8034a6acb239776eaa1077e", "status": "ok" }, "moe_sort/gfx950/scan_t64_k2_e16": { From 960fae0ea4ba79823555a8bf888b1f106658a2c2 Mon Sep 17 00:00:00 2001 From: Aleksander Dudek Date: Wed, 22 Jul 2026 09:07:31 +0000 Subject: [PATCH 2/5] run pre-commit hook --- .../rocke/platform/cpp/core/lower_llvm/core.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp index 5a120082cdbb..576aa5d0c005 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp @@ -997,8 +997,7 @@ static const char* ll_val_to_gname(const rocke_lower_t* L, const char* value_nam * slot (a NULL op still ticks the counter but does not recurse), plus every * op in each sub-region. The subtree rooted at index `idx` therefore occupies * contiguous sequence indices [idx, idx + ll_subtree_size(op) - 1]. */ -static int - ll_subtree_size(const rocke_op_t* op) +static int ll_subtree_size(const rocke_op_t* op) { int n = 1; if(!op) @@ -1322,7 +1321,7 @@ const char* rocke_ll_smem_base_cache_t ent; ent.block = blk; ent.gname = gname; - ent.base = base; + ent.base = base; int rc; rocke_vec_push(&L->arena, &L->smem_base_cache, ent, rc); if(rc != 0) From c4420c4e26606b0506eca5497bb38613c38fbbe3 Mon Sep 17 00:00:00 2001 From: Aleksander Dudek Date: Fri, 24 Jul 2026 17:01:27 +0000 Subject: [PATCH 3/5] fix(rocke): fix cshuffle LDS-reuse race and packer aliasing bugs Add a workgroup barrier before the first cshuffle C store (C aliases A/B in the shared pool but the double-buffered/prefetched tail MFMA had no barrier before the epilogue), guard the packer against expanding a reused slot over a still-live slot, and extend inner-loop liveness to the enclosing loop. Both engines; golden re-blessed. --- .../platform/cpp/core/lower_llvm/core.cpp | 41 +- .../rocke/platform/cpp/helpers/epilogues.cpp | 10 + .../cpp/instances/common/gemm_epilogue.cpp | 10 + .../platform/python/rocke/core/lower_llvm.py | 40 +- .../python/rocke/helpers/epilogues.py | 10 + .../rocke/instances/common/gemm_universal.py | 13 + .../tests/core/test_cshuffle_reuse_barrier.py | 131 ++++++ .../core/test_smem_liveness_packing_extra.py | 407 ++++++++++++++++++ .../rocke_representative_ir_sha256.json | 56 +-- 9 files changed, 682 insertions(+), 36 deletions(-) create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_cshuffle_reuse_barrier.py create mode 100644 dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_smem_liveness_packing_extra.py diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp index 576aa5d0c005..a7c29ae7a387 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp @@ -1067,11 +1067,25 @@ static void /* Recurse into sub-regions. scf.for gets a conservative loop_end: the * last preorder index of the loop subtree (idx + size - 1), not the * for-op's own (earlier) index, so allocations defined inside the loop - * are extended across the whole loop and interfere as they must. */ + * are extended across the whole loop and interfere as they must. + * + * For a nested loop, an allocation used only in the inner loop can be + * re-read on a later outer iteration, so its live range must reach the + * enclosing loop's end too -- take the max with any enclosing loop_end + * (loop_end < 0 means "no enclosing loop", i.e. Python's None). */ for(int r = 0; r < op->num_regions; r++) { - int child_loop_end - = (op->opcode == ROCKE_OP_SCF_FOR) ? (idx + ll_subtree_size(op) - 1) : loop_end; + int child_loop_end; + if(op->opcode == ROCKE_OP_SCF_FOR) + { + int own_last = idx + ll_subtree_size(op) - 1; + child_loop_end + = (loop_end < 0) ? own_last : (own_last > loop_end ? own_last : loop_end); + } + else + { + child_loop_end = loop_end; + } ll_liveness_walk(ctx, op->regions[r], child_loop_end); } } @@ -1216,6 +1230,27 @@ void rocke_ll_compute_smem_layout(rocke_lower_t* L) if(slots[k].last_seq >= first_seq) continue; /* still live, interference */ int aligned_off = (slots[k].offset + aln - 1) & ~(aln - 1); + /* Reusing slot k places this allocation at [aligned_off, + * aligned_off+seg). Because it may be larger than slot k's original + * footprint, that range can spill upward into a DIFFERENT slot that + * is still live while this allocation is live -- which would alias + * two simultaneously-live allocations and corrupt data. Reject any + * candidate whose placed range overlaps a still-live slot; slots + * already dead before first_seq are safe to overlap. */ + int placed_end = aligned_off + seg; + int conflict = 0; + for(int j = 0; j < num_slots; j++) + { + if(j == k || slots[j].last_seq < first_seq) + continue; + if(aligned_off < slots[j].offset + slots[j].size && slots[j].offset < placed_end) + { + conflict = 1; + break; + } + } + if(conflict) + continue; if(best < 0 || aligned_off < best_aligned) { best = k; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp index 279449d7f69d..cf4d508ba91b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp @@ -518,6 +518,16 @@ void rocke_cshuffle_epilogue_store(rocke_ir_builder_t* b, (void)rocke_b_const_i32(b, 0); (void)rocke_b_const_i32(b, 0); + /* ---- step 0: reuse barrier. ---- + * The common-LDS packer aliases this C staging tile onto the A/B staging + * bytes (non-interfering in program order). Double-buffered / prefetched + * (compv4 / async_dma) mainloops end with the tail-tile MFMA reading A/B + * from LDS *after* their last drain barrier and emit no trailing barrier, so + * without a barrier here a fast wave's first C ds_write would clobber A/B + * bytes a slow wave is still reading for its tail MFMA -- a cross-wave WAR on + * the aliased pool. Mirrors the Python CShuffleEpilogue.store. */ + rocke_b_sync(b); + for(mi = 0; mi < mfmas_m; ++mi) { for(ni = 0; ni < mfmas_n; ++ni) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_epilogue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_epilogue.cpp index ea5652cedbc2..7a2fd5eaefc5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_epilogue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_epilogue.cpp @@ -732,6 +732,16 @@ void rocke_gemm_emit_epilogue_cshuffle(rocke_ir_builder_t* b, Cs = rocke_b_smem_alloc(b, storage_dtype, shape, 2, "C_smem"); } + /* ---- step 0: reuse barrier. ---- + * The common-LDS packer aliases this C staging tile onto the A/B staging + * bytes (non-interfering in program order). The double-buffered / prefetched + * mainloops end with the tail-tile MFMA reading A/B from LDS *after* their + * last drain barrier and emit no trailing barrier, so without a barrier here + * a fast wave's first C ds_write would clobber A/B bytes a slow wave is still + * reading for its tail MFMA -- a cross-wave WAR on the aliased pool. Mirrors + * the Python _emit_epilogue_cshuffle. */ + rocke_b_sync(b); + warp_m_off = rocke_b_mul(b, warp_m_idx, rocke_b_const_i32(b, mfmas_m * t->warp_tile_m)); warp_n_off = rocke_b_mul(b, warp_n_idx, rocke_b_const_i32(b, mfmas_n * t->warp_tile_n)); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py index a00ca9fd5813..eb2c3164be2a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py @@ -1037,7 +1037,17 @@ def walk(ops: List, loop_end: Optional[int]) -> None: # uses to that last index (not the for-op's own, earlier # index, which would under-extend allocations defined # inside the loop and let them wrongly share LDS). - loop_last = idx + _subtree_size(op) - 1 + # + # For a nested loop, an allocation used only in the inner + # loop can still be re-read on a later *outer* iteration, + # so its live range must reach the enclosing loop's end + # too -- take the max with any enclosing ``loop_end``, + # otherwise the inner alloc could wrongly share LDS with + # an allocation used later in the outer body. + own_last = idx + _subtree_size(op) - 1 + loop_last = ( + own_last if loop_end is None else max(own_last, loop_end) + ) walk(r.ops, loop_end=loop_last) else: walk(r.ops, loop_end=loop_end) @@ -1133,21 +1143,41 @@ def _sort_key(item: Tuple[str, "SmemType"]) -> int: # Among free slots, prefer the one with the smallest aligned start # (lowest address, cache-friendly, minimises pool fragmentation). best: Optional[int] = None # index into slots[] + best_aligned = 0 for i, (s_off, s_size, s_last) in enumerate(slots): if s_last >= first_seq: # Still live when we start – interference, skip. continue aligned_off = (s_off + aln - 1) & ~(aln - 1) - if best is None or aligned_off < (slots[best][0] + aln - 1) & ~( - aln - 1 - ): + # Reusing slot i places this allocation at [aligned_off, + # aligned_off + seg). Because it may be larger than slot i's + # original footprint, that range can spill upward into a + # DIFFERENT slot that is still live while this allocation is + # live -- which would alias two simultaneously-live allocations + # and corrupt data. Reject any candidate whose placed range + # overlaps a still-live slot; slots already dead before + # first_seq are safe to overlap. + placed_end = aligned_off + seg + conflict = False + for j, (o2, sz2, last2) in enumerate(slots): + if j == i or last2 < first_seq: + continue + if aligned_off < o2 + sz2 and o2 < placed_end: + conflict = True + break + if conflict: + continue + if best is None or aligned_off < best_aligned: best = i + best_aligned = aligned_off if best is not None: s_off, s_size, _ = slots[best] - aligned_off = (s_off + aln - 1) & ~(aln - 1) + aligned_off = best_aligned self._smem_offsets[gname] = aligned_off # Expand the slot to cover the new allocation if it overflows. + # The guard above proved the placed range does not overlap any + # live slot, so the expansion is safe. new_size = max(s_size, aligned_off - s_off + seg) slots[best] = (s_off, new_size, last_seq) else: diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/epilogues.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/epilogues.py index 158c9323ffc7..af662acf247c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/epilogues.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/epilogues.py @@ -460,6 +460,16 @@ def store( dist = _cshuffle_acc_distribution(atom.c_per_lane) traits = LoadStoreTraits(distribution=dist, vector_dim_y=1, scalar_per_vector=1) + # ---- step 0: reuse barrier. ---- + # The common-LDS packer aliases this C staging tile onto the A/B + # staging bytes (non-interfering in program order). Double-buffered / + # prefetched (compv4 / async_dma) mainloops end with the tail-tile MFMA + # reading A/B from LDS *after* their last drain barrier and emit no + # trailing barrier, so without a barrier here a fast wave's first C + # ``ds_write`` would clobber A/B bytes a slow wave is still reading for + # its tail MFMA -- a cross-wave WAR on the aliased pool region. + b.sync() + for mi in range(mfmas_m): for ni in range(mfmas_n): acc = accs[mi * mfmas_n + ni] diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/gemm_universal.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/gemm_universal.py index 357ab54fcce4..0f67982d3fc6 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/gemm_universal.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/gemm_universal.py @@ -2457,6 +2457,19 @@ def _emit_epilogue_cshuffle( # LDS staging tile: tile_m x tile_n of output storage dtype. Cs = b.smem_alloc(storage_dtype, [t.tile_m, t.tile_n], name_hint="C_smem") + # ---- step 0: reuse barrier. ---- + # The common-LDS packer aliases this C staging tile onto the A/B + # staging bytes (they are non-interfering in program order, so the + # liveness packer places C at the A/B pool offset). The double-buffered + # and prefetched mainloops (``_emit_kloop_db`` / ``_emit_kloop_prefetch``) + # end with the tail-tile MFMA reading A/B from LDS *after* their last + # drain barrier and emit no trailing barrier, so without a barrier here a + # fast wave's first C ``ds_write`` would clobber A/B bytes a slow wave is + # still reading for its tail MFMA -- a cross-wave WAR on the aliased pool. + # (The single-buffer ``_emit_kloop_simple`` already has a trailing in-loop + # barrier, which makes this one redundant-but-harmless there.) + b.sync() + warp_m_off = b.mul(warp_m_idx, b.const_i32(mfmas_m * t.warp_tile_m)) warp_n_off = b.mul(warp_n_idx, b.const_i32(mfmas_n * t.warp_tile_n)) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_cshuffle_reuse_barrier.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_cshuffle_reuse_barrier.py new file mode 100644 index 000000000000..adaae7b7d742 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_cshuffle_reuse_barrier.py @@ -0,0 +1,131 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""Regression guard for the CShuffle LDS-reuse barrier (PR #8844). + +The common-LDS packer aliases the CShuffle C staging tile onto the A/B staging +bytes (they are non-interfering in program order). The double-buffered and +prefetched mainloops end with a *tail* MFMA that reads A/B from LDS after their +last drain barrier and emit no trailing barrier, so once C aliases A/B the first +C ``ds_write`` in the epilogue can clobber A/B bytes a slow wave is still reading +for its tail MFMA -- a cross-wave WAR hazard on the shared pool. + +The fix adds a workgroup barrier at the very start of the CShuffle epilogue, +before the first C store, in both epilogue emitters: + * ``gemm_universal._emit_epilogue_cshuffle`` (inline), and + * ``helpers.epilogues.CShuffleEpilogue.store`` (used by conv). + +These tests assert, at the KernelDef IR level (CPU-only, no compile), that a +barrier separates the C staging allocation from the first store into it. +""" + +from __future__ import annotations + +import pytest + +_BARRIERS = ("tile.sync", "tile.sync_lds_only", "tile.s_barrier_bare") + + +def _flatten(region, out): + for op in region.ops: + out.append(op) + for r in op.regions: + _flatten(r, out) + return out + + +def _assert_barrier_before_first_c_store(kernel) -> None: + """The first smem store after the C_smem allocation must be preceded by a + workgroup barrier that itself follows the allocation (i.e. no C store may + race the just-completed A/B reads on the aliased pool).""" + ops = _flatten(kernel.body, []) + + c_allocs = [ + i + for i, o in enumerate(ops) + if o.name == "tile.smem_alloc" and "C_smem" in o.results[0].name + ] + assert c_allocs, "expected a C_smem allocation in a cshuffle kernel" + c_idx = c_allocs[0] + + later_stores = [ + i + for i, o in enumerate(ops) + if o.name.startswith("tile.smem_store") and i > c_idx + ] + assert later_stores, "expected at least one store into the C staging tile" + first_store = later_stores[0] + + barriers = [i for i in range(c_idx + 1, first_store) if ops[i].name in _BARRIERS] + assert barriers, ( + "no workgroup barrier between the C_smem allocation and the first C " + "store: a fast wave's C write could clobber A/B bytes a slow wave is " + "still reading for its tail MFMA (cross-wave WAR on the aliased pool)" + ) + # And no C store may sneak in before that barrier. + assert not any( + ops[i].name.startswith("tile.smem_store") for i in range(c_idx + 1, barriers[0]) + ), "a C store precedes the reuse barrier" + + +@pytest.mark.parametrize("pipeline", ["mem", "compv3", "compv4"]) +def test_gemm_cshuffle_has_reuse_barrier(pipeline): + """Every cshuffle GEMM pipeline emits the reuse barrier before the first C + store (covers ``_emit_epilogue_cshuffle`` for single-buffer, double-buffer + and prefetched mainloops).""" + from rocke.instances.common.gemm_universal import ( + TileSpec, + TraitSpec, + UniversalGemmSpec, + build_universal_gemm, + ) + + spec = UniversalGemmSpec( + name="vg", + tile=TileSpec( + tile_m=128, + tile_n=128, + tile_k=32, + warp_m=2, + warp_n=2, + warp_tile_m=32, + warp_tile_n=32, + warp_tile_k=16, + ), + trait=TraitSpec(pipeline=pipeline, scheduler="intrawave", epilogue="cshuffle"), + ) + _assert_barrier_before_first_c_store(build_universal_gemm(spec)) + + +def test_conv_cshuffle_has_reuse_barrier(): + """The conv implicit-GEMM cshuffle path (double-buffered compv4, via the + shared ``CShuffleEpilogue`` helper) emits the reuse barrier before the first + C store.""" + from rocke.instances.common.conv_implicit_gemm import ( + ConvProblem, + ImplicitGemmConvSpec, + build_implicit_gemm_conv, + ) + + spec = ImplicitGemmConvSpec( + problem=ConvProblem(N=8, Hi=56, Wi=56, C=64, K=64, Y=3, X=3, pH=1, pW=1), + name="c", + tile_m=64, + tile_n=64, + tile_k=64, + warp_m=2, + warp_n=2, + warp_tile_m=32, + warp_tile_n=32, + warp_tile_k=16, + wave_size=64, + pipeline="compv4", + epilogue="cshuffle", + ) + _assert_barrier_before_first_c_store(build_implicit_gemm_conv(spec, arch="gfx950")) + + +if __name__ == "__main__": # pragma: no cover + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_smem_liveness_packing_extra.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_smem_liveness_packing_extra.py new file mode 100644 index 000000000000..f7912db96f1f --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_smem_liveness_packing_extra.py @@ -0,0 +1,407 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""Adversarial tests for LDS liveness analysis and interval-based smem packing. + +Companion to ``test_smem_liveness_packing.py`` for PR #8844 ("common LDS +allocation for A/B/CShuffle"). Where the sibling file checks the intended happy +paths, this file stress-tests the *safety* invariant that actually protects the +kernel from data corruption: + + Two allocations whose live intervals OVERLAP must be assigned DISJOINT byte + ranges in the shared @smem_pool. If overlapping allocations ever share a + byte, one kernel write silently clobbers another's live data. + +Everything here is CPU-only: it drives ``_Lowerer._collect_smem`` / +``_collect_smem_liveness`` / ``_compute_smem_layout`` directly (no GPU, no +compilation), matching the style of the sibling test file. + +Tests 5 and 7 originally exposed two real packer bugs (nested-loop liveness +under-extension and the slot-expansion collision). Both are now fixed +(``_collect_smem_liveness`` extends inner-loop uses to the enclosing loop end, +and ``_compute_smem_layout`` rejects a slot-reuse candidate whose expanded range +would overlap a still-live slot), so these tests are now hard regression guards. +""" + +from __future__ import annotations + +import random + +import pytest + +from rocke.core.ir import ( + BF16, + BF8E5M2, + F16, + F32, + FP8E4M3, + I8, + KernelDef, + Op, + Region, + SmemType, + Value, +) +from rocke.core.lower_llvm import _Lowerer + + +# --------------------------------------------------------------------------- +# helpers (mirrors test_smem_liveness_packing.py) +# --------------------------------------------------------------------------- + + +def _smem(elem, *shape: int) -> SmemType: + return SmemType(elem, list(shape)) + + +def _make_op(name: str, operands=(), results=()): + op = Op(name=name, operands=list(operands), results=list(results)) + for r in results: + r.op = op + return op + + +def _alloc(name: str, stype: SmemType): + v = Value(name=name, type=stype) + op = _make_op("tile.smem_alloc", results=[v]) + return op, v + + +def _use(*vals: Value) -> Op: + return _make_op("tile.lds_store", operands=list(vals)) + + +def _for(*ops: Op) -> Op: + """A minimal scf.for whose body is the given ops.""" + return Op( + name="scf.for", + operands=[], + results=[], + regions=[Region(label="body", ops=list(ops))], + ) + + +def _lowerer(*ops: Op) -> _Lowerer: + body = Region(label="body", ops=list(ops)) + kernel = KernelDef(name="k", params=[], body=body) + low = _Lowerer(kernel) + low._collect_smem(body) + return low + + +def _pack(*ops: Op): + """Run the full layout pass; return ({short_name: offset}, pool_size).""" + low = _lowerer(*ops) + low._compute_smem_layout() + offsets = { + g.lstrip("@").split(".")[0]: low._smem_offsets[g] for g in low._smem_offsets + } + return offsets, low._smem_pool_size + + +def _overlaps(o1: int, s1: int, o2: int, s2: int) -> bool: + """True iff byte ranges [o1,o1+s1) and [o2,o2+s2) intersect.""" + return o1 < o2 + s2 and o2 < o1 + s1 + + +def _intervals_interfere(a, b) -> bool: + """Closed-interval interference, matching the packer's reuse rule. + + The packer reuses a slot only when ``slot_last < new_first`` (strict), so + two allocations interfere (cannot share a byte) exactly when their closed + live intervals [first, last] intersect. + """ + return a[0] <= b[1] and b[0] <= a[1] + + +# byte sizes +_KB = 1024 +_4KB = 4 * _KB # f32 [32x32] +_8KB = 8 * _KB # f16 [64x64] +_12KB = 12 * _KB # f32 [128x24] +_16KB = 16 * _KB # i8 [128x128] +_64KB = 64 * _KB # f32 [128x128] (CShuffle) + + +def _round16(n: int) -> int: + return (n + 15) & ~15 + + +# =========================================================================== +# 1. Non-interference safety — the core anti-corruption invariant +# =========================================================================== + + +def test_overlapping_allocs_get_disjoint_byte_ranges(): + """CORE INVARIANT: allocations with overlapping live intervals never + share a byte. + + Two allocations that are both read by the same op are simultaneously live; + the packer must give them disjoint byte ranges, otherwise one kernel store + corrupts the other's data. This is the single most important safety + property of the shared-LDS packer. + """ + # A and B are both live at the shared use -> must not overlap. + op_a, va = _alloc("%A", _smem(F32, 128, 24)) # 12KB + op_b, vb = _alloc("%B", _smem(F32, 128, 24)) # 12KB + op_u = _use(va, vb) + offsets, pool = _pack(op_a, op_b, op_u) + assert not _overlaps( + offsets["A"], _12KB, offsets["B"], _12KB + ), "A and B are simultaneously live and MUST occupy disjoint byte ranges" + # With two mutually interfering 12KB tiles the pool holds both. + assert pool == _round16(2 * _12KB) + + # Three mutually-interfering tiles: every pair must be disjoint. + op_a, va = _alloc("%A", _smem(F16, 64, 64)) # 8KB + op_b, vb = _alloc("%B", _smem(F16, 64, 64)) # 8KB + op_c, vc = _alloc("%C", _smem(F16, 64, 64)) # 8KB + op_u = _use(va, vb, vc) + offsets, pool = _pack(op_a, op_b, op_c, op_u) + for n1, n2 in (("A", "B"), ("A", "C"), ("B", "C")): + assert not _overlaps( + offsets[n1], _8KB, offsets[n2], _8KB + ), f"{n1} and {n2} are simultaneously live and must be disjoint" + assert pool == _round16(3 * _8KB) + + +# =========================================================================== +# 2. Reuse actually happens for non-overlapping intervals +# =========================================================================== + + +def test_nonoverlapping_allocs_reuse_same_offset(): + """Allocations whose live ranges are strictly ordered (one fully before the + other) share the SAME base offset, and the pool size collapses to + max(size), not sum(size).""" + # A is used only in the first phase; C only in the second phase. + op_a, va = _alloc("%A", _smem(F16, 64, 64)) # 8KB, live [0,1] + op_u1 = _use(va) + op_c, vc = _alloc("%C", _smem(F16, 64, 64)) # 8KB, live [2,3] + op_u2 = _use(vc) + offsets, pool = _pack(op_a, op_u1, op_c, op_u2) + assert offsets["A"] == 0 + assert offsets["C"] == 0, "C's interval is after A's -> C must reuse A's slot" + assert pool == _round16(_8KB), "pool must collapse to max(size), not sum(size)" + + +# =========================================================================== +# 3. A/B/C GEMM shape — the pattern the PR actually optimizes +# =========================================================================== + + +def test_abc_gemm_shape_pool_is_max_not_sum(): + """Model the real GEMM: A and B live inside the K-loop, CShuffle live only + after the loop. + + Expected: pool == max(A+B, C) (A and B coexist in the loop, C reuses the + freed A/B region), and A and B — both live in the loop — never alias. + """ + op_a, va = _alloc("%A", _smem(F32, 128, 24)) # 12KB + op_b, vb = _alloc("%B", _smem(F32, 128, 24)) # 12KB + # K-loop reads both A and B every iteration. + loop = _for(_use(va, vb)) + # CShuffle after the loop. + op_c, vc = _alloc("%C", _smem(F32, 128, 128)) # 64KB + op_cu = _use(vc) + offsets, pool = _pack(op_a, op_b, loop, op_c, op_cu) + + ab_total = _12KB + _12KB + assert not _overlaps( + offsets["A"], _12KB, offsets["B"], _12KB + ), "A and B are both live inside the K-loop and must never alias" + assert pool == _round16(max(ab_total, _64KB)), "pool must be max(A+B, C), not A+B+C" + assert pool == _round16(_64KB) + + +# =========================================================================== +# 4. Alignment +# =========================================================================== + + +@pytest.mark.parametrize("byte_elem", [I8, FP8E4M3, BF8E5M2]) +def test_byte_elem_allocs_are_16b_aligned(byte_elem): + """Byte-element (i8/fp8/bf8) allocations get 16-byte-aligned offsets; + 16-bit (f16/bf16) allocations get >=4-byte-aligned offsets; and the whole + pool size is a multiple of 16.""" + op_a, va = _alloc("%A", _smem(byte_elem, 128, 128)) # 16KB, align 16 + op_b, vb = _alloc("%B", _smem(F16, 64, 64)) # 8KB, align 4 + op_u = _use(va, vb) + offsets, pool = _pack(op_a, op_b, op_u) + assert offsets["A"] % 16 == 0, f"{byte_elem.name} alloc must be 16-byte aligned" + assert offsets["B"] % 4 == 0, "f16 alloc must be >=4-byte aligned" + assert pool % 16 == 0, "pool size must be a multiple of 16" + + +@pytest.mark.parametrize("half_elem", [F16, BF16]) +def test_byte_alloc_after_odd_half_alloc_is_16b_aligned(half_elem): + """Mixed-dtype pool: a 16-bit alloc whose size is NOT a multiple of 16 + precedes an interfering byte alloc. The byte alloc must still land on a + 16-byte boundary (padding is inserted), never on the raw end of the f16 + region.""" + # f16 [130] = 260 bytes -> 260 % 16 == 4, so a naive bump would misalign. + op_h, vh = _alloc("%H", _smem(half_elem, 130)) # 260 bytes, align 4 + op_b, vb = _alloc("%B", _smem(I8, 4096)) # 4KB, align 16 + op_u = _use(vh, vb) # both live -> byte alloc opens a fresh, aligned slot + offsets, pool = _pack(op_h, op_b, op_u) + assert offsets["H"] % 4 == 0 + assert offsets["B"] % 16 == 0, "byte alloc must be 16-byte aligned even after f16" + assert offsets["B"] >= 260, "byte alloc must sit past the live f16 region" + assert pool % 16 == 0 + + +# =========================================================================== +# 5. Nested-loop conservative liveness (adversarial) — XFAIL, real bug +# =========================================================================== + + +def test_nested_loop_inner_alloc_interferes_with_outer_body_alloc(): + """Outer scf.for { inner scf.for { use X } ; use Y }. + + X is used only in the inner loop; Y only in the outer body after the inner + loop. Within a single outer iteration X's use precedes Y's, so naively they + do not overlap. But because the outer loop repeats, on iteration N+1 the + inner loop re-reads (re-produces/consumes) X while Y from iteration N may + still be live -> X and Y are loop-carried interfering and MUST get disjoint + byte ranges. + + Regression guard for the nested-loop liveness fix: ``_collect_smem_liveness`` + now extends an inner-loop use to the enclosing loop's end, so X's live range + reaches the outer loop end and interferes with Y. + """ + op_x, vx = _alloc("%X", _smem(F16, 64, 64)) # 8KB + inner = _for(op_x, _use(vx)) + op_y, vy = _alloc("%Y", _smem(F16, 64, 64)) # 8KB + outer = _for(inner, op_y, _use(vy)) + offsets, pool = _pack(outer) + # Correct (conservative) behavior: X and Y interfere -> disjoint. + assert not _overlaps(offsets["X"], _8KB, offsets["Y"], _8KB), ( + "X (inner-loop) and Y (outer-body) are loop-carried interfering and " + "must not share LDS" + ) + + +# =========================================================================== +# 6. Fallback — single alloc, zero allocs +# =========================================================================== + + +def test_single_alloc_offset_zero(): + """A lone allocation lands at offset 0 with a 16-rounded pool.""" + op_a, _ = _alloc("%A", _smem(F32, 128, 24)) # 12KB + offsets, pool = _pack(op_a) + assert offsets["A"] == 0 + assert pool == _round16(_12KB) + + +def test_zero_allocs_pool_is_zero(): + """A kernel with no smem allocations produces a zero-size pool.""" + low = _lowerer(_make_op("arith.constant")) + low._compute_smem_layout() + assert low._smem_pool_size == 0 + + +# =========================================================================== +# 7. Property / fuzz — the strongest anti-corruption guarantee +# =========================================================================== + + +def _pack_from_intervals(sizes, intervals, elems=None): + """Drive the REAL packer with caller-supplied sizes and live intervals. + + Builds one smem_alloc per entry (so ``_smem_globals`` is populated the same + way real IR would), then monkeypatches ``_collect_smem_liveness`` to return + the requested intervals. ``_compute_smem_layout`` then runs unchanged. + Returns the list of assigned byte offsets in input order. + """ + n = len(sizes) + if elems is None: + elems = [I8] * n # i8 -> 1 byte/elem, exact byte size via shape [size] + ops = [] + gnames = [] + for i, (elem, sz) in enumerate(zip(elems, sizes)): + eb = {"i8": 1, "f16": 2, "f32": 4}[elem.name] + assert sz % eb == 0 + op, _ = _alloc(f"%a{i}", _smem(elem, sz // eb)) + ops.append(op) + gnames.append(f"@a{i}.k") + low = _lowerer(*ops) + live = {gnames[i]: tuple(intervals[i]) for i in range(n)} + low._collect_smem_liveness = lambda region, _live=live: dict(_live) + low._compute_smem_layout() + return [low._smem_offsets[g] for g in gnames] + + +def test_fuzz_deterministic_expansion_collision(): + """Deterministic regression guard for the slot-expansion collision. + + A(12KB, live[0,3]), B(12KB, live[1,10]), C(64KB, live[4,10]). + When C is packed, A's slot (offset 0) is free (A dead at seq>3). The old + packer reused it and expanded the slot to 64KB WITHOUT checking that B — still + live at offset 12KB — lay inside the expanded range, so C and B (which + interfere) ended up sharing bytes [12KB, 24KB). The fixed packer rejects that + reuse candidate and places C so it stays disjoint from B. + """ + sizes = [_12KB, _12KB, _64KB] + intervals = [(0, 3), (1, 10), (4, 10)] + offs = _pack_from_intervals(sizes, intervals) + # B and C interfere -> must be disjoint. + assert _intervals_interfere(intervals[1], intervals[2]) + assert not _overlaps(offs[1], sizes[1], offs[2], sizes[2]), ( + f"slot-expansion collision: B@{offs[1]}(+{sizes[1]}) overlaps " + f"C@{offs[2]}(+{sizes[2]}) though intervals {intervals[1]} & " + f"{intervals[2]} interfere" + ) + + +def test_fuzz_overlapping_allocs_never_share_bytes(): + """UNIVERSAL INVARIANT (fuzz): across 200 randomized allocation sets, every + pair of allocations whose live intervals overlap is assigned disjoint byte + ranges. + + This is the strongest guarantee that the packer never lets one allocation's + stores corrupt another's live data. Uses a fixed seed for determinism. + """ + rng = random.Random(1234) + elem_choices = [(I8, 1), (F16, 2), (F32, 4)] + violations = [] + + for trial in range(200): + n = rng.randint(2, 6) + elems = [] + sizes = [] + for _ in range(n): + elem, eb = rng.choice(elem_choices) + elems.append(elem) + sizes.append(rng.randint(1, 20) * 1024) # multiple of 4 -> valid for all + intervals = [] + for _ in range(n): + s = rng.randint(0, 20) + e = s + rng.randint(0, 20) + intervals.append((s, e)) + + offs = _pack_from_intervals(sizes, intervals, elems=elems) + + for i in range(n): + for j in range(i + 1, n): + if _intervals_interfere(intervals[i], intervals[j]) and _overlaps( + offs[i], sizes[i], offs[j], sizes[j] + ): + violations.append( + { + "trial": trial, + "i": (intervals[i], offs[i], sizes[i]), + "j": (intervals[j], offs[j], sizes[j]), + } + ) + + assert not violations, ( + f"{len(violations)} interfering allocation pairs were assigned " + f"OVERLAPPING byte ranges (LDS corruption). First: {violations[0]}" + ) + + +if __name__ == "__main__": # pragma: no cover + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json index 72477cee7685..1e3fdb3bb003 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json @@ -44,10 +44,10 @@ }, "conv/gfx950/n1h8c16k32r3": { "arch": "gfx950", - "bytes": 23082, + "bytes": 23162, "family": "conv", "kernel_name": "irhash_conv_950_a_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_compv4_cshuffle", - "sha256": "65521c817d5bcad307cd1afdeadfa706513097c0ff8c7073c71b2230f77ce6b9", + "sha256": "fca9393b171090b7ea99543bee2559d4bb4526477550d582f8c906e7d34b10fe", "status": "ok" }, "conv/gfx950/n2h16c32k32r1": { @@ -63,7 +63,7 @@ "bytes": 53250, "family": "deep_fused_conv", "kernel_name": "irhash_deep_gfx950_wt16_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "056511051ae3d983c326f353e61cecb8f521c673bcaa0fd6814b5e5205db529a", + "sha256": "6c207bf8faba31f56e5af5fc59b9ae3fc0872f3abba2e143de245efa7fe76a02", "status": "ok" }, "deep/common_gfx950_wt16/h24w24_k32": { @@ -71,7 +71,7 @@ "bytes": 53299, "family": "deep_fused_conv", "kernel_name": "irhash_deep_gfx950_wt16b_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "cf0354729d53507b82aac23b0d97618f1fa4921aa31bf2cc8a87c368f749edf3", + "sha256": "5d89d7d8bf821a5a0f541014d4fc39f5518a651e8d2ff4136db966f9f76d79d1", "status": "ok" }, "deep/gfx1151/native_h16w16": { @@ -103,7 +103,7 @@ "bytes": 72406, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_default_epi_relu", - "sha256": "e04852bbbcfd125ae3060f5c876366a662ceffda5f151e5cdf0f7c4ed5f2eb57", + "sha256": "4815f880ddcdc68d113768180abaed931fe7a74c128d6e43059de612f415817f", "status": "ok" }, "deep/gfx1201/h24w24_k32": { @@ -111,7 +111,7 @@ "bytes": 72407, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_default_epi_relu", - "sha256": "71c6b87694919c000e9d96f1767c86669d9b58da80b70eaa2f4934477bb94f24", + "sha256": "f32710966a99d0ba57f6485f7fc2c466e4d0671c61802fa37b786b7da28985fb", "status": "ok" }, "deep/gfx950/h16w16_k32": { @@ -119,7 +119,7 @@ "bytes": 32107, "family": "deep_fused_conv", "kernel_name": "rocke_gfx950_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a32x32x16_mem_cshuffle_epi_relu", - "sha256": "64cafece518e47ef2ef57a11673aead362059e0bccab16ff37c2c36ddae5af95", + "sha256": "f48e9e4662df0827a023a7b220d1c1c26881cd6666157b2f8671284fe97c0d51", "status": "ok" }, "deep/gfx950/h24w24_k32": { @@ -127,7 +127,7 @@ "bytes": 32111, "family": "deep_fused_conv", "kernel_name": "rocke_gfx950_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a32x32x16_mem_cshuffle_epi_relu", - "sha256": "37a5f88d3876f8f67f70b5340bea6b63c412f36aa6217a6cc677da3e9e271943", + "sha256": "6fa4cfea93adc617eb295e98c190d4d3058093826c6bf3f99a7029fc43d313d1", "status": "ok" }, "fused_moe/gfx950/gather_h128": { @@ -204,10 +204,10 @@ }, "gemm/gfx90a/t128x128x16": { "arch": "gfx90a", - "bytes": 89223, + "bytes": 89303, "family": "gemm", "kernel_name": "irhash_gemm_90a_b_fp16_t128x128x16_w2x2x1_wt32x32x8_compv4_intrawave_cshuffle_pad", - "sha256": "580f9b19159acb58b4f4d6b8be80273cc00568156403f8c45c59a484e35ea987", + "sha256": "89c110546c4828b2b00c7917815f983310959c2f5c139785c7d85794301e56d4", "status": "ok" }, "gemm/gfx90a/t64x64x16": { @@ -220,10 +220,10 @@ }, "gemm/gfx942/t128x128x16": { "arch": "gfx942", - "bytes": 89223, + "bytes": 89303, "family": "gemm", "kernel_name": "irhash_gemm_942_b_fp16_t128x128x16_w2x2x1_wt32x32x8_compv4_intrawave_cshuffle_pad", - "sha256": "46245c54e3f449c1e981655942bc90dc26a4d30fefeb14e0dd5c790c9b006bd7", + "sha256": "12b79158e161c289c632137f9f81f81372f6596ec6323a9c4571e796289daff2", "status": "ok" }, "gemm/gfx942/t64x64x16": { @@ -236,10 +236,10 @@ }, "gemm/gfx950/t128x128x32": { "arch": "gfx950", - "bytes": 94212, + "bytes": 94292, "family": "gemm", "kernel_name": "irhash_gemm_950_a_fp16_t128x128x32_w2x2x1_wt32x32x16_compv4_intrawave_cshuffle_pad", - "sha256": "4b80dfe052894057034bda57a68ee237504d1a79916e86cb52969a882d5b4a88", + "sha256": "91a908b70eb7559bb17c45ed2bad51d55f9319c4235c48eb06d28ff5e36b56e4", "status": "ok" }, "gemm/gfx950/t64x128x32": { @@ -364,10 +364,10 @@ }, "conv/gfx950/n1h8c16k32r3": { "arch": "gfx950", - "bytes": 23219, + "bytes": 23299, "family": "conv", "kernel_name": "irhash_conv_950_a_N1H8W8C16_K32Y3X3_t64x64x32_w2x2_a32x32x16_compv4_cshuffle", - "sha256": "1fe15959b07de36cfb8bd47f8e2d9a5047b50c48368ec221b365d78f4c0b922b", + "sha256": "3d15bb33540576a167142af44d57c0f00ca1a4b8cc40f70ce8c9b9b8dc1491ae", "status": "ok" }, "conv/gfx950/n2h16c32k32r1": { @@ -383,7 +383,7 @@ "bytes": 53436, "family": "deep_fused_conv", "kernel_name": "irhash_deep_gfx950_wt16_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "8903d08364a127d00d15a43d0a55e798d3f74fb37bcb8dd13800fe72582c1beb", + "sha256": "ca7477119474aa8ea0ea70089be478ed0a6f6be04e4d5cda85257a142157d7d4", "status": "ok" }, "deep/common_gfx950_wt16/h24w24_k32": { @@ -391,7 +391,7 @@ "bytes": 53485, "family": "deep_fused_conv", "kernel_name": "irhash_deep_gfx950_wt16b_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_cshuffle_epi_relu", - "sha256": "605a94a47f380c3e7bf4546f5e5f57dc9f7d733feb8f87dd9b525d13d39e670a", + "sha256": "871fb9dce54a3dc8744c792faaadc88635965c89a481f6d0de81d9b1fac3e680", "status": "ok" }, "deep/gfx1151/native_h16w16": { @@ -423,7 +423,7 @@ "bytes": 72597, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_default_epi_relu", - "sha256": "9fc4c18f497636e1c23f6f2a8cbf8ba9fa8b386c987215db965bfba79d59db98", + "sha256": "e880a635284b99c1a8313a7b91f8f94720ab4b17be34fbfe4edada66329c601e", "status": "ok" }, "deep/gfx1201/h24w24_k32": { @@ -431,7 +431,7 @@ "bytes": 72598, "family": "deep_fused_conv", "kernel_name": "rocke_gfx1201_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a16x16x16_mem_default_epi_relu", - "sha256": "ac8d72a94bd835b0dc1accb893bd15da45ca353da0e06b1e181b65cdb8cf680d", + "sha256": "99abe59745195c7ce58aefc20af95879d242dd99a4e849636a4c5ee32aaa779e", "status": "ok" }, "deep/gfx950/h16w16_k32": { @@ -439,7 +439,7 @@ "bytes": 32285, "family": "deep_fused_conv", "kernel_name": "rocke_gfx950_deep_fused_conv_pool_N1H16W16C16_K32Y3X3_t64x32x16_w2x1_a32x32x16_mem_cshuffle_epi_relu", - "sha256": "dd878e5603c4c851e46e2c3d44f71737b4d77aac33ad2ead4137655a1319c4a4", + "sha256": "e4049b4033ca3fa0b6d00e9e42e260d38e3975533c613fff8e96f45711ca11c8", "status": "ok" }, "deep/gfx950/h24w24_k32": { @@ -447,7 +447,7 @@ "bytes": 32289, "family": "deep_fused_conv", "kernel_name": "rocke_gfx950_deep_fused_conv_pool_N1H24W24C16_K32Y3X3_t64x32x16_w2x1_a32x32x16_mem_cshuffle_epi_relu", - "sha256": "17c58019efdde001e5d85ab6c528aca4a22267c7a45e67b1ca9854436dfcde8e", + "sha256": "35bcd9ab9330e6a1b7eb5d7d51bba466e0264e4828711b4516a8da1d40be60fe", "status": "ok" }, "fused_moe/gfx950/gather_h128": { @@ -524,10 +524,10 @@ }, "gemm/gfx90a/t128x128x16": { "arch": "gfx90a", - "bytes": 89234, + "bytes": 89314, "family": "gemm", "kernel_name": "irhash_gemm_90a_b_fp16_t128x128x16_w2x2x1_wt32x32x8_compv4_intrawave_cshuffle_pad", - "sha256": "d2dc79ab69083cae0dac3b088c8b78f2a571c69e08c22bf034a3f232328fc50a", + "sha256": "908a474777c1597c8b8916dfe81f325cfe48775351e46f229de1af224a0dffcb", "status": "ok" }, "gemm/gfx90a/t64x64x16": { @@ -540,10 +540,10 @@ }, "gemm/gfx942/t128x128x16": { "arch": "gfx942", - "bytes": 89234, + "bytes": 89314, "family": "gemm", "kernel_name": "irhash_gemm_942_b_fp16_t128x128x16_w2x2x1_wt32x32x8_compv4_intrawave_cshuffle_pad", - "sha256": "56730e8a4790816a0219a9cbd59b017240641e9ab030aefded6da4d3298d35e1", + "sha256": "76021c3e815aef173e4d17663a3283051355500467549cdc73e7ccc00dfd1847", "status": "ok" }, "gemm/gfx942/t64x64x16": { @@ -556,10 +556,10 @@ }, "gemm/gfx950/t128x128x32": { "arch": "gfx950", - "bytes": 94223, + "bytes": 94303, "family": "gemm", "kernel_name": "irhash_gemm_950_a_fp16_t128x128x32_w2x2x1_wt32x32x16_compv4_intrawave_cshuffle_pad", - "sha256": "3cf0a4dc05cd0de38d85454301636a3b4e494283150925fa8918c87990b3ceae", + "sha256": "e636915286c0c66a1925274dfcecb3775620b817bd247b762fdc4c7584c42baf", "status": "ok" }, "gemm/gfx950/t64x128x32": { From 09e21e7128cd9b569392f0b60058e91b86b7eb29 Mon Sep 17 00:00:00 2001 From: Aleksander Dudek Date: Mon, 27 Jul 2026 13:25:45 +0000 Subject: [PATCH 4/5] feat(rocke): optional cshuffle_no_alias flag for C LDS aliasing control Add an opt-in `cshuffle_no_alias` flag to the cshuffle epilogue that lets a kernel trade LDS/occupancy for latency, addressing yraparti's review request on the common-LDS-pool work. By default (False, unchanged) the smem-pool packer aliases the cshuffle C tile onto the A/B staging bytes (pool = max(ab, c)), which requires the epilogue "step-0 reuse barrier" between the main loop's last A/B read and the first C write. This is best for large tiles (more occupancy, throughput-bound). When True, C gets its own exclusive LDS bytes (pool = ab + c) and the step-0 barrier is elided -- the two are safe together precisely because C no longer overlaps A/B. This lowers latency for small tiles. The kernel name gains a `noalc` tag so aliased and un-aliased kernels are distinct. Mechanism: a new shared `SmemType.exclusive` bit (kept out of the type name so default output is byte-identical) tells the packer to give that allocation its own byte range (skip slot reuse + sentinel-lock it). The flag sets it on the C allocation and elides the barrier, in both the inline GEMM epilogue and the shared CShuffleEpilogue (conv). The bit round-trips through serialization via an `exclusive` op attr so the cpp/both backends match the python backend. Implemented in both engines (byte-identical) with golden coverage for both variants: - Python: core/ir.py, core/lower_llvm.py (packer), core/ir_serialize.py, helpers/{epilogues,tensor_view}.py, instances/common/{gemm_universal, conv_implicit_gemm}.py, examples/common/universal_gemm_verify.py. - C++: ir.h, core/ir/{core_types,ir_tile,serialize}.cpp, core/lower_llvm/core.cpp, helpers/{epilogues,tensor_view}.cpp + headers, instances/common/{gemm_epilogue,gemm_spec,conv_implicit_gemm_conv_epilogue, conv_implicit_gemm_conv_spec_descriptors}.cpp, include/rocke/{instance_gemm_universal,instance_conv_implicit_gemm}.h. - Tests: parity emitters gemm_emit.{py,c} + conv_implicit_gemm_emit.{py,c} gain an alias-off config; rocke_ir_parity_harness.py adds an alias-off representative case; golden re-blessed (additions only, no existing sha changed). Verification: - Byte-identity GREEN at llvm20 and llvm22 (65 families; gemm and conv_implicit_gemm configs=11 bad=0 -- the new no-alias config included). - Golden gate passes; diff is additions-only. - On-GPU numeric (gfx942) bad=0 for both aliased and no-alias cshuffle GEMM. Default behavior is byte-identical; only the opt-in path is new. --- .../rocke/platform/cpp/core/ir/core_types.cpp | 8 +++-- .../rocke/platform/cpp/core/ir/ir_tile.cpp | 36 +++++++++++++++---- .../rocke/platform/cpp/core/ir/serialize.cpp | 18 +++++++++- .../platform/cpp/core/lower_llvm/core.cpp | 11 ++++-- .../rocke/platform/cpp/helpers/epilogues.cpp | 13 +++++-- .../platform/cpp/helpers/tensor_view.cpp | 31 ++++++++++++++++ .../rocke/helper_rocke.helpers.epilogues.h | 4 +++ .../rocke/helper_rocke.helpers.tensor_view.h | 12 +++++++ .../rocke/instance_conv_implicit_gemm.h | 7 ++++ .../include/rocke/instance_gemm_universal.h | 6 ++++ .../rocke/platform/cpp/include/rocke/ir.h | 16 +++++++-- .../conv_implicit_gemm_conv_epilogue.cpp | 1 + ...nv_implicit_gemm_conv_spec_descriptors.cpp | 9 +++-- .../cpp/instances/common/gemm_epilogue.cpp | 17 +++++++-- .../cpp/instances/common/gemm_spec.cpp | 9 +++-- .../rocke/platform/python/rocke/core/ir.py | 31 +++++++++++++--- .../python/rocke/core/ir_serialize.py | 15 ++++++++ .../platform/python/rocke/core/lower_llvm.py | 17 +++++++-- .../examples/common/universal_gemm_verify.py | 7 ++++ .../python/rocke/helpers/epilogues.py | 18 ++++++++-- .../python/rocke/helpers/tensor_view.py | 7 +++- .../instances/common/conv_implicit_gemm.py | 13 +++++-- .../rocke/instances/common/gemm_universal.py | 26 ++++++++++++-- .../rocke_representative_ir_sha256.json | 28 +++++++++++---- .../parity/conv_implicit_gemm_emit.c | 7 ++++ .../parity/conv_implicit_gemm_emit.py | 21 +++++++++++ .../tests/instances/parity/gemm_emit.c | 21 ++++++++++- .../tests/instances/parity/gemm_emit.py | 28 ++++++++++++++- .../instances/rocke_ir_parity_harness.py | 27 ++++++++++++-- 29 files changed, 413 insertions(+), 51 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp index a4f45e363c06..60a160518b8f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp @@ -131,9 +131,10 @@ const rocke_type_t* return t; } -/* SmemType(elem, shape) -> "smem<{elem}, [{d0}x{d1}...]>" */ -const rocke_type_t* - rocke_smem_type(rocke_ir_builder_t* b, const rocke_type_t* elem, const int* shape, int rank) +/* SmemType(elem, shape) -> "smem<{elem}, [{d0}x{d1}...]>" + * `exclusive` is kept OUT of the name (default 0 stays byte-identical). */ +const rocke_type_t* rocke_smem_type( + rocke_ir_builder_t* b, const rocke_type_t* elem, const int* shape, int rank, int exclusive) { rocke_type_t* t; int* shape_copy = NULL; @@ -166,6 +167,7 @@ const rocke_type_t* t->elem = elem; t->shape = shape_copy; t->rank = rank; + t->smem_exclusive = exclusive ? 1 : 0; /* Build the "[d0xd1x...]" body, then the full canonical name. The Python * form is f"smem<{elem.name}, [{'x'.join(...)}]>". */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp index 22040fef4814..b2d08e6f6fd6 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_tile.cpp @@ -152,24 +152,46 @@ static rocke_value_t** rocke_build_mem_operands(rocke_ir_builder_t* b, /* LDS (shared memory) -- alloc */ /* ===================================================================== */ -rocke_value_t* rocke_b_smem_alloc(rocke_ir_builder_t* b, - const rocke_type_t* elem, - const int* shape, - int rank, - const char* name_hint) +rocke_value_t* rocke_b_smem_alloc_ex(rocke_ir_builder_t* b, + const rocke_type_t* elem, + const int* shape, + int rank, + const char* name_hint, + int exclusive) { const rocke_type_t* t; + rocke_attr_map_t attrs; + const rocke_attr_map_t* attrs_p = NULL; if(!rocke_i_live(b)) { return NULL; } - t = rocke_smem_type(b, elem, shape, rank); + t = rocke_smem_type(b, elem, shape, rank, exclusive); if(!t) { return NULL; } + /* Carry the exclusive bit as an op attr so it round-trips through the + * ck.dsl.ir/v1 serializer (the type name deliberately omits it). Only + * emitted when set, so the default stays byte-identical. Ignored by the + * native .ll lowering (which reads the SmemType's smem_exclusive). */ + if(exclusive) + { + attrs = rocke_i_attrs(b); + rocke_attr_set_bool(b, &attrs, "exclusive", true); + attrs_p = &attrs; + } return rocke_i_op1( - b, ROCKE_OP_TILE_SMEM_ALLOC, NULL, 0, t, NULL, name_hint ? name_hint : "smem"); + b, ROCKE_OP_TILE_SMEM_ALLOC, NULL, 0, t, attrs_p, name_hint ? name_hint : "smem"); +} + +rocke_value_t* rocke_b_smem_alloc(rocke_ir_builder_t* b, + const rocke_type_t* elem, + const int* shape, + int rank, + const char* name_hint) +{ + return rocke_b_smem_alloc_ex(b, elem, shape, rank, name_hint, 0); } /* ===================================================================== */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/serialize.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/serialize.cpp index 2e6528fe82af..ba2e21a5eab9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/serialize.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/serialize.cpp @@ -729,7 +729,8 @@ static const rocke_type_t* parse_type(rocke_ir_builder_t* b, const char* s, size } } } - return rocke_smem_type(b, elem, shape, rank); + /* exclusive is reconstructed from the op attr in parse_op (name-only). */ + return rocke_smem_type(b, elem, shape, rank, 0); } /* scalar: reconstruct the singleton by name. */ @@ -1859,6 +1860,21 @@ static rocke_op_t* parse_op(rocke_ir_builder_t* b, line_reader_t* lr, val_table_ op->num_operands = num_operands; } + /* The smem type name deliberately omits the `exclusive` bit; it rides on + * the smem_alloc op as an attr. Rebuild the result SmemType with exclusive + * set so the packer's no-alias behavior round-trips through the serializer. */ + if(opcode == ROCKE_OP_TILE_SMEM_ALLOC && num_results > 0 && result_types[0] + && result_types[0]->kind == ROCKE_TYPE_SMEM + && rocke_attr_get_bool(&attrs, "exclusive", false)) + { + result_types[0] = rocke_smem_type( + b, result_types[0]->elem, result_types[0]->shape, result_types[0]->rank, 1); + if(!result_types[0]) + { + return NULL; + } + } + if(num_results > 0) { op->results = (rocke_value_t**)rocke_arena_alloc( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp index a7c29ae7a387..e64bb67d8e2e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/core.cpp @@ -1221,11 +1221,18 @@ void rocke_ll_compute_smem_layout(rocke_lower_t* L) int aln = ll_smem_align(stype); int first_seq = intervals[gi].first_seq; int last_seq = intervals[gi].last_seq; + /* Exclusive (cshuffle no-alias) allocations must not reuse another + * allocation's slot and must never be reused, so they occupy their own + * byte range. Skipping the reuse search forces a fresh slot; recording + * it with the sentinel last_seq below keeps it permanently "live" so + * nothing else packs onto it. (Mirrors Python's _EXCL_LAST_SEQ.) */ + int excl = stype ? stype->smem_exclusive : 0; + const int ROCKE_LL_EXCL_LAST_SEQ = 1 << 30; /* Find the best (lowest aligned-offset) free slot. */ int best = -1; int best_aligned = -1; - for(int k = 0; k < num_slots; k++) + for(int k = 0; excl ? 0 : (k < num_slots); k++) { if(slots[k].last_seq >= first_seq) continue; /* still live, interference */ @@ -1283,7 +1290,7 @@ void rocke_ll_compute_smem_layout(rocke_lower_t* L) offsets[gi] = aligned_off; slots[num_slots].offset = aligned_off; slots[num_slots].size = seg; - slots[num_slots].last_seq = last_seq; + slots[num_slots].last_seq = excl ? ROCKE_LL_EXCL_LAST_SEQ : last_seq; num_slots++; } } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp index cf4d508ba91b..4ad0522d873d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/epilogues.cpp @@ -414,6 +414,7 @@ rocke_cshuffle_epilogue_t rocke_cshuffle_epilogue_make(const rocke_mfma_atom_t* epi.store_vec = 8; /* Python default */ epi.smem_name_hint = "C_smem"; epi.out_dtype = "f16"; + epi.no_alias = false; /* Python default */ return epi; } @@ -505,7 +506,8 @@ void rocke_cshuffle_epilogue_store(rocke_ir_builder_t* b, /* ---- step 1: publish accs to LDS at the MFMA output layout. ---- */ lds_shape[0] = grid->tile_m; /* storage_shape(tile_m) == (tile_m, tile_n) */ lds_shape[1] = grid->tile_n; - if(rocke_make_lds_view(b, &c_view, _lds_dtype, lds_shape, 2, epi->smem_name_hint, NULL) + if(rocke_make_lds_view_ex( + b, &c_view, _lds_dtype, lds_shape, 2, epi->smem_name_hint, NULL, epi->no_alias ? 1 : 0) != ROCKE_OK) return; c_smem = c_view.base; @@ -525,8 +527,13 @@ void rocke_cshuffle_epilogue_store(rocke_ir_builder_t* b, * from LDS *after* their last drain barrier and emit no trailing barrier, so * without a barrier here a fast wave's first C ds_write would clobber A/B * bytes a slow wave is still reading for its tail MFMA -- a cross-wave WAR on - * the aliased pool. Mirrors the Python CShuffleEpilogue.store. */ - rocke_b_sync(b); + * the aliased pool. Mirrors the Python CShuffleEpilogue.store. + * + * With no_alias the C tile has its own exclusive LDS bytes that never + * overlap A/B, so this WAR cannot occur and the barrier is elided. The + * step-2 C-write->C-read barrier below is a genuine RAW and always stays. */ + if(!epi->no_alias) + rocke_b_sync(b); for(mi = 0; mi < mfmas_m; ++mi) { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/tensor_view.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/tensor_view.cpp index 757b5387903b..fc9362193d1e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/tensor_view.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/tensor_view.cpp @@ -749,6 +749,37 @@ rocke_status_t rocke_make_naive_tensor_view_packed(rocke_tensor_view_t* out, return rocke_make_global_view(out, base, shape, rank, dtype, NULL); } +rocke_status_t rocke_make_lds_view_ex(rocke_ir_builder_t* b, + rocke_tensor_view_t* out, + const rocke_type_t* dtype, + const int* shape, + int rank, + const char* name_hint, + const rocke_stride_t* strides /* NULL => packed */, + int exclusive) +{ + /* Python make_lds_view (with the ``exclusive`` passthrough): + * smem = b.smem_alloc(dtype, list(shape), name_hint=name_hint, + * exclusive=exclusive) + * desc = packed(shape, dtype) if strides is None else with_strides(...) + * return TensorView(base=smem, desc=desc, addr_space="lds") + */ + rocke_status_t st; + rocke_value_t* smem; + if(b == NULL || out == NULL) + return ROCKE_ERR_VALUE; + smem = rocke_b_smem_alloc_ex(b, dtype, shape, rank, name_hint, exclusive); + if(strides == NULL) + st = rocke_tensor_descriptor_packed(&out->desc, shape, rank, dtype); + else + st = rocke_tensor_descriptor_with_strides(&out->desc, shape, strides, rank, dtype); + if(st != ROCKE_OK) + return st; + out->base = smem; + out->addr_space = ROCKE_ADDR_LDS; + return ROCKE_OK; +} + rocke_status_t rocke_make_lds_view(rocke_ir_builder_t* b, rocke_tensor_view_t* out, const rocke_type_t* dtype, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.epilogues.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.epilogues.h index 42311cfd906e..2c00cea4ce30 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.epilogues.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.epilogues.h @@ -172,6 +172,10 @@ typedef struct rocke_cshuffle_epilogue int store_vec; /* elements per wide store; default 8 */ const char* smem_name_hint; /* default "C_smem" */ const char* out_dtype; /* "f16" (default), "bf16", or "fp32" */ + /* cshuffle "no-alias" mode: when true the C staging tile gets its own + * exclusive LDS bytes (not aliased onto A/B) and the step-0 reuse barrier is + * elided. Default false keeps the aliased/low-LDS behavior (byte-identical). */ + bool no_alias; /* default false */ } rocke_cshuffle_epilogue_t; /* Construct with the Python defaults (store_vec=8, smem_name_hint="C_smem", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.tensor_view.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.tensor_view.h index 0398b9a0a002..7536f1955501 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.tensor_view.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.tensor_view.h @@ -285,6 +285,18 @@ rocke_status_t rocke_make_lds_view(rocke_ir_builder_t* b, const char* name_hint, const rocke_stride_t* strides /* NULL => packed */); +/* Like rocke_make_lds_view but forwards `exclusive` to smem_alloc (cshuffle + * no-alias mode): the smem-pool packer gives the buffer its own byte range. + * rocke_make_lds_view is rocke_make_lds_view_ex(..., exclusive=0). */ +rocke_status_t rocke_make_lds_view_ex(rocke_ir_builder_t* b, + rocke_tensor_view_t* out, + const rocke_type_t* dtype, + const int* shape, + int rank, + const char* name_hint, + const rocke_stride_t* strides /* NULL => packed */, + int exclusive); + /* ----------------------------------------- compute-promoting (f32) peers */ /* TensorView.load_vec_as_f32(b, indices, n): vector load + per-lane f32 diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_conv_implicit_gemm.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_conv_implicit_gemm.h index c57e4877bd56..6ebd70fb6848 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_conv_implicit_gemm.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_conv_implicit_gemm.h @@ -176,6 +176,13 @@ typedef struct rocke_implicit_gemm_conv_spec const char* dtype_acc; /* default "fp32" */ rocke_conv_acc_epilogue_t acc_epilogue; /* default identity */ + + /* cshuffle epilogue LDS aliasing (same knob as UniversalGemmSpec.trait). + * False (default): the smem-pool packer aliases the cshuffle C tile onto the + * A/B staging bytes (pool = max(ab,c)) with a step-0 reuse barrier. True: C + * gets its own LDS bytes (pool = ab+c) and the barrier is elided -> lower + * small-tile latency, more LDS. Only affects epilogue == "cshuffle". */ + bool cshuffle_no_alias; /* default false */ } rocke_implicit_gemm_conv_spec_t; /* Default-constructed spec (every field == Python dataclass default). The caller diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gemm_universal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gemm_universal.h index f4e0d2e89adf..0d75b04f788e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gemm_universal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gemm_universal.h @@ -115,6 +115,12 @@ typedef struct rocke_gemm_trait_spec * atomic-adds each warp's f32 accumulator into it. split_k == 1 (default) * keeps the canonical single-K-pass body byte-identical. */ int split_k; /* default 1 */ + /* cshuffle epilogue LDS aliasing. False (default): the smem-pool packer + * aliases the cshuffle C tile onto the A/B staging bytes (pool = max(ab,c)) + * with a step-0 reuse barrier. True: C gets its own LDS bytes (pool = ab+c) + * and the barrier is elided -> lower small-tile latency, more LDS. Only + * affects the cshuffle epilogue; False keeps byte-identical output. */ + bool cshuffle_no_alias; /* default false */ } rocke_gemm_trait_spec_t; /* ------------------------------------------------------------------ DataSpec */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h index d78a4bb78ff1..3c81fe22eb7c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h @@ -131,6 +131,9 @@ typedef struct rocke_type /* ROCKE_TYPE_SMEM */ const int* shape; /* arena-owned array of dim sizes */ int rank; /* number of dims in shape */ + int smem_exclusive; /* 1 => smem-pool packer gives this alloc its own + * byte range (cshuffle no-alias mode). Kept OUT of + * `name` so default (0) stays byte-identical. */ } rocke_type_t; /* -------------------------------------------------------------- attr values */ @@ -534,8 +537,8 @@ const rocke_type_t* rocke_scalar_by_name(const char* name); const rocke_type_t* rocke_vector_type(rocke_ir_builder_t* b, const rocke_type_t* elem, int count); const rocke_type_t* rocke_ptr_type(rocke_ir_builder_t* b, const rocke_type_t* pointee, const char* space); -const rocke_type_t* - rocke_smem_type(rocke_ir_builder_t* b, const rocke_type_t* elem, const int* shape, int rank); +const rocke_type_t* rocke_smem_type( + rocke_ir_builder_t* b, const rocke_type_t* elem, const int* shape, int rank, int exclusive); /* Structural type equality (matches Python frozen-dataclass __eq__: compares by * canonical name, which encodes kind + components). */ @@ -758,6 +761,15 @@ rocke_value_t* rocke_b_smem_alloc(rocke_ir_builder_t* b, const int* shape, int rank, const char* name_hint); +/* Like rocke_b_smem_alloc but marks the allocation `exclusive` (cshuffle + * no-alias mode): the smem-pool packer gives it its own byte range. The + * plain rocke_b_smem_alloc forwards here with exclusive=0. */ +rocke_value_t* rocke_b_smem_alloc_ex(rocke_ir_builder_t* b, + const rocke_type_t* elem, + const int* shape, + int rank, + const char* name_hint, + int exclusive); rocke_value_t* rocke_b_global_load(rocke_ir_builder_t* b, rocke_value_t* ptr, rocke_value_t* idx, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_epilogue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_epilogue.cpp index 530beaa6635e..826373661125 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_epilogue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_epilogue.cpp @@ -330,6 +330,7 @@ void rocke_conv_emit_cshuffle_epilogue(rocke_ir_builder_t* b, int max_store_vec = spec->has_vector_size_c ? spec->vector_size_c : 8; rocke_cshuffle_epilogue_t epi = rocke_cshuffle_epilogue_from_grid(atom, grid, max_store_vec); epi.out_dtype = spec->dtype_d; + epi.no_alias = spec->cshuffle_no_alias; /* .store(b, accs=accs, addr_fn=d_addr, d_rsrc=d_rsrc, * bounds=(b.const_i32(p.M), b.const_i32(p.N_gemm))) */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp index 0a52f696890f..5f0630738820 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp @@ -315,6 +315,7 @@ rocke_implicit_gemm_conv_spec_t rocke_implicit_gemm_conv_spec_default(void) s.dtype_acc = "fp32"; s.acc_epilogue = rocke_conv_acc_epilogue_default(); + s.cshuffle_no_alias = false; /* default: alias cshuffle C onto A/B */ return s; } @@ -376,8 +377,8 @@ rocke_status_t rocke_implicit_gemm_conv_spec_kernel_name(const rocke_implicit_ge char pe_buf[64]; char tag_buf[256]; const char* parts[6]; - const char* flag_names[1]; - int flag_on[1]; + const char* flag_names[2]; + int flag_on[2]; rocke_status_t st; if(s == NULL || out == NULL) @@ -414,8 +415,10 @@ rocke_status_t rocke_implicit_gemm_conv_spec_kernel_name(const rocke_implicit_ge flag_names[0] = "async"; flag_on[0] = s->async_dma ? 1 : 0; + flag_names[1] = "noalc"; + flag_on[1] = s->cshuffle_no_alias ? 1 : 0; - return rocke_kernel_name_join(s->name, parts, 6, flag_names, flag_on, 1, out, out_cap, NULL); + return rocke_kernel_name_join(s->name, parts, 6, flag_names, flag_on, 2, out, out_cap, NULL); } /* ===================================================================== * diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_epilogue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_epilogue.cpp index 7a2fd5eaefc5..860413e709f4 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_epilogue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_epilogue.cpp @@ -729,7 +729,10 @@ void rocke_gemm_emit_epilogue_cshuffle(rocke_ir_builder_t* b, int shape[2]; shape[0] = t->tile_m; shape[1] = t->tile_n; - Cs = rocke_b_smem_alloc(b, storage_dtype, shape, 2, "C_smem"); + /* cshuffle_no_alias marks it exclusive so the smem-pool packer gives it + * its own byte range instead of aliasing the A/B staging bytes. */ + Cs = rocke_b_smem_alloc_ex( + b, storage_dtype, shape, 2, "C_smem", spec->trait.cshuffle_no_alias ? 1 : 0); } /* ---- step 0: reuse barrier. ---- @@ -739,8 +742,16 @@ void rocke_gemm_emit_epilogue_cshuffle(rocke_ir_builder_t* b, * last drain barrier and emit no trailing barrier, so without a barrier here * a fast wave's first C ds_write would clobber A/B bytes a slow wave is still * reading for its tail MFMA -- a cross-wave WAR on the aliased pool. Mirrors - * the Python _emit_epilogue_cshuffle. */ - rocke_b_sync(b); + * the Python _emit_epilogue_cshuffle. + * + * With cshuffle_no_alias the C tile has its own exclusive LDS bytes that + * never overlap A/B, so this cross-wave WAR cannot occur and the barrier is + * elided. The step-3 C-write->C-read barrier below is a genuine RAW and + * always stays. */ + if(!spec->trait.cshuffle_no_alias) + { + rocke_b_sync(b); + } warp_m_off = rocke_b_mul(b, warp_m_idx, rocke_b_const_i32(b, mfmas_m * t->warp_tile_m)); warp_n_off = rocke_b_mul(b, warp_n_idx, rocke_b_const_i32(b, mfmas_n * t->warp_tile_n)); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_spec.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_spec.cpp index 7f3cd19d7775..2ba203aabc7d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_spec.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/gemm_spec.cpp @@ -201,6 +201,7 @@ rocke_gemm_universal_spec_t rocke_gemm_universal_spec_default(void) s.trait.emit_sched_hints_set = false; /* Python None (arch-resolved) */ s.trait.emit_sched_hints = false; s.trait.split_k = 1; /* default 1 (single-K-pass body) */ + s.trait.cshuffle_no_alias = false; /* default: alias cshuffle C onto A/B */ /* DataSpec defaults. */ s.data.dtype_a = "fp16"; /* default "fp16" */ @@ -257,8 +258,8 @@ rocke_status_t rocke_gemm_universal_kernel_name(const rocke_gemm_universal_spec_ char part_pipe[128]; char part_spk[32]; const char* parts[5]; - const char* flag_names[8]; - int flag_on[8]; + const char* flag_names[9]; + int flag_on[9]; const rocke_gemm_tile_spec_t* t; const rocke_gemm_trait_spec_t* tr; @@ -293,6 +294,7 @@ rocke_status_t rocke_gemm_universal_kernel_name(const rocke_gemm_universal_spec_ /* Python flag key f"spk{tr.split_k}" (dynamic name; on when split_k > 1). */ snprintf(part_spk, sizeof(part_spk), "spk%d", tr->split_k); flag_names[7] = part_spk; + flag_names[8] = "noalc"; flag_on[0] = (tr->pad_m || tr->pad_n || tr->pad_k) ? 1 : 0; flag_on[1] = tr->persistent ? 1 : 0; @@ -302,8 +304,9 @@ rocke_status_t rocke_gemm_universal_kernel_name(const rocke_gemm_universal_spec_ flag_on[5] = tr->dtl_prefetch ? 1 : 0; flag_on[6] = tr->active_tile_skip ? 1 : 0; flag_on[7] = (tr->split_k > 1) ? 1 : 0; + flag_on[8] = tr->cshuffle_no_alias ? 1 : 0; - return rocke_kernel_name_join(spec->name, parts, 5, flag_names, flag_on, 8, out, out_cap, NULL); + return rocke_kernel_name_join(spec->name, parts, 5, flag_names, flag_on, 9, out, out_cap, NULL); } /* ===================================================================== * diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py index e7722452a834..a42e53b7a00d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py @@ -143,13 +143,22 @@ def __init__(self, pointee: Type, space: str) -> None: class SmemType(Type): elem: Type shape: Tuple[int, ...] + # When True, the smem-pool packer must give this allocation its own + # byte range (never reuse another allocation's slot, and never be reused). + # Used by the cshuffle "no-alias" mode so the C tile does not overlap the + # A/B staging bytes. Deliberately kept OUT of ``name`` so the LLVM type + # text is unchanged for the default (exclusive=False) case -> byte-identical. + exclusive: bool = False - def __init__(self, elem: Type, shape: Sequence[int]) -> None: + def __init__( + self, elem: Type, shape: Sequence[int], exclusive: bool = False + ) -> None: shape = tuple(int(x) for x in shape) s = "x".join(str(x) for x in shape) object.__setattr__(self, "name", f"smem<{elem.name}, [{s}]>") object.__setattr__(self, "elem", elem) object.__setattr__(self, "shape", shape) + object.__setattr__(self, "exclusive", exclusive) # ----------------------------- Values / Ops ------------------------------ @@ -1119,11 +1128,23 @@ def block_id_z(self) -> Value: # ----- memory ----- def smem_alloc( - self, elem: Type, shape: Sequence[int], name_hint: str = "smem" + self, + elem: Type, + shape: Sequence[int], + name_hint: str = "smem", + exclusive: bool = False, ) -> Value: - t = SmemType(elem, shape) - return self._op( - "tile.smem_alloc", result_types=[t], result_name_hint=name_hint + t = SmemType(elem, shape, exclusive=exclusive) + # The smem type name deliberately omits ``exclusive``; carry it as an op + # attr so it round-trips through the ck.dsl.ir/v1 serializer (whose type + # reconstruction is name-only). Only emitted when set -> the default + # (exclusive=False) serialized form is unchanged / byte-identical. + attrs = {"exclusive": True} if exclusive else None + return self._op( + "tile.smem_alloc", + result_types=[t], + attrs=attrs, + result_name_hint=name_hint, ).result def global_load( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir_serialize.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir_serialize.py index c8b6342ec215..18c9c5cdbc5a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir_serialize.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir_serialize.py @@ -521,6 +521,21 @@ def _parse_op(src: _Lines, values: Dict[str, Value]) -> Op: sc = _Scanner(tail[q + 1 :]) loc = _read_quoted(sc) + # The smem type name deliberately omits ``exclusive``; it rides on the + # smem_alloc op as an attr. Rebuild the result SmemType with exclusive set + # so the packer's no-alias behavior round-trips through the serializer. + if ( + opname == "tile.smem_alloc" + and attrs.get("exclusive") + and results + and isinstance(results[0].type, SmemType) + ): + _st = results[0].type + results[0] = Value( + name=results[0].name, + type=SmemType(_st.elem, _st.shape, exclusive=True), + ) + op = Op( name=opname, operands=operands, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py index eb2c3164be2a..2abee6f50bb0 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_llvm.py @@ -1122,11 +1122,22 @@ def _sort_key(item: Tuple[str, "SmemType"]) -> int: # Each "slot" is (offset, size, last_seq) – the byte range it occupies # and the latest sequence index at which it is still live. slots: List[Tuple[int, int, int]] = [] # (offset, size, last_seq) + # Sentinel "never dies" last_seq for exclusive (no-alias) allocations, + # so the reuse test ``s_last >= first_seq`` is always true for their + # slots and nothing else packs onto them. Larger than any real preorder + # index. (Mirrored as ROCKE_LL_EXCL_LAST_SEQ in the C++ engine.) + _EXCL_LAST_SEQ = 1 << 30 for gname, stype in sorted_allocs: seg = _seg_size(stype) aln = _align(stype) first_seq, last_seq = live.get(gname, (0, 0)) + # Exclusive (cshuffle no-alias) allocations must not reuse another + # allocation's slot, and must never be reused by later allocations, + # so they occupy their own byte range. Skipping the free-slot search + # forces a fresh slot; recording it with a sentinel last_seq below + # keeps it permanently "live" so nothing else packs onto it. + excl = getattr(stype, "exclusive", False) # Try to reuse any slot that is free before this allocation starts. # A "free" slot (s_last < first_seq) provides a candidate base @@ -1144,7 +1155,7 @@ def _sort_key(item: Tuple[str, "SmemType"]) -> int: # (lowest address, cache-friendly, minimises pool fragmentation). best: Optional[int] = None # index into slots[] best_aligned = 0 - for i, (s_off, s_size, s_last) in enumerate(slots): + for i, (s_off, s_size, s_last) in enumerate([] if excl else slots): if s_last >= first_seq: # Still live when we start – interference, skip. continue @@ -1187,7 +1198,9 @@ def _sort_key(item: Tuple[str, "SmemType"]) -> int: ) aligned_off = (current_end + aln - 1) & ~(aln - 1) self._smem_offsets[gname] = aligned_off - slots.append((aligned_off, seg, last_seq)) + # Exclusive allocs stay permanently live (sentinel last_seq) so + # no later allocation reuses their bytes. + slots.append((aligned_off, seg, _EXCL_LAST_SEQ if excl else last_seq)) pool_size = max(s_off + s_size for s_off, s_size, _ in slots) self._smem_pool_size = (pool_size + 15) & ~15 diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/common/universal_gemm_verify.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/common/universal_gemm_verify.py index bef5a185bcd0..6f253eeb2399 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/common/universal_gemm_verify.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/examples/common/universal_gemm_verify.py @@ -69,6 +69,12 @@ def main() -> int: ) p.add_argument("--pipeline", default="mem") p.add_argument("--epilogue", default="default") + p.add_argument( + "--cshuffle-no-alias", + action="store_true", + help="give the cshuffle C tile its own LDS bytes (no A/B aliasing) and " + "elide the step-0 reuse barrier (lower small-tile latency, more LDS).", + ) p.add_argument("--output-dir", default=None) p.add_argument("--no-verify", action="store_true") p.add_argument( @@ -109,6 +115,7 @@ def main() -> int: pad_m=True, pad_n=True, pad_k=True, + cshuffle_no_alias=args.cshuffle_no_alias, ) data = DataSpec( dtype_a=args.dtype, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/epilogues.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/epilogues.py index af662acf247c..cbc76dbc50ab 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/epilogues.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/epilogues.py @@ -365,6 +365,11 @@ class CShuffleEpilogue: # 1-byte stores via ``global_store_vN`` (no buffer-store fp8 # intrinsic exists today on the AMDGPU LLVM target). out_dtype: str = "f16" + # cshuffle "no-alias" mode: when True the C staging tile gets its own + # exclusive LDS bytes (not aliased onto the A/B pool) and the step-0 reuse + # barrier is elided -> lower small-tile latency, more LDS. Default False + # keeps the aliased/low-LDS behavior (byte-identical). + no_alias: bool = False @classmethod def from_grid( @@ -374,6 +379,7 @@ def from_grid( grid: WarpGrid, max_store_vec: int = 8, out_dtype: str = "f16", + no_alias: bool = False, ) -> "CShuffleEpilogue": """Pick the widest `store_vec` that distributes the tile evenly. @@ -394,7 +400,9 @@ def from_grid( if ok: break v //= 2 - return cls(atom=atom, grid=grid, store_vec=v, out_dtype=out_dtype) + return cls( + atom=atom, grid=grid, store_vec=v, out_dtype=out_dtype, no_alias=no_alias + ) def store( self, @@ -444,6 +452,7 @@ def store( dtype=_lds_dtype, shape=lds_layout.storage_shape(grid.tile_m), name_hint=self.smem_name_hint, + exclusive=self.no_alias, ) c_smem = c_view.base # A full-extent LDS window; per-(mi, ni) origins move within it. @@ -468,7 +477,12 @@ def store( # trailing barrier, so without a barrier here a fast wave's first C # ``ds_write`` would clobber A/B bytes a slow wave is still reading for # its tail MFMA -- a cross-wave WAR on the aliased pool region. - b.sync() + # + # With ``no_alias`` the C tile has its own exclusive LDS bytes that never + # overlap A/B, so this WAR cannot occur and the barrier is elided. The + # step-2 C-write->C-read barrier below is a genuine RAW and always stays. + if not self.no_alias: + b.sync() for mi in range(mfmas_m): for ni in range(mfmas_n): diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/tensor_view.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/tensor_view.py index 8645ee977f1b..9e33e2f5436c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/tensor_view.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/tensor_view.py @@ -1043,6 +1043,7 @@ def make_lds_view( shape: Sequence[int], name_hint: str = "lds", strides: Optional[Sequence[StrideElem]] = None, + exclusive: bool = False, ) -> TensorView: """``make_tensor_view(make_lds_alloc(...), desc)``. @@ -1050,8 +1051,12 @@ def make_lds_view( returns a view over it. ``strides`` defaults to packed row-major; pass an explicit stride (e.g. ``shape[1] + lds_pad``) to introduce bank-conflict padding. + + ``exclusive`` (cshuffle no-alias mode) forwards to ``smem_alloc`` so the + smem-pool packer gives this buffer its own byte range instead of aliasing + another allocation's slot. """ - smem = b.smem_alloc(dtype, list(shape), name_hint=name_hint) + smem = b.smem_alloc(dtype, list(shape), name_hint=name_hint, exclusive=exclusive) if strides is None: desc = TensorDescriptor.packed(shape, dtype) else: diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/conv_implicit_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/conv_implicit_gemm.py index 5eef5d762f3b..094a9d4ec13f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/conv_implicit_gemm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/conv_implicit_gemm.py @@ -386,6 +386,12 @@ class ImplicitGemmConvSpec: # It composes simple fp32 VALU transforms directly on MFMA accumulator # fragments before the existing direct/cshuffle store path. acc_epilogue: ConvAccumulatorEpilogue = ConvAccumulatorEpilogue() + # cshuffle epilogue LDS aliasing (same knob as UniversalGemmSpec.trait). + # False (default): the smem-pool packer aliases the cshuffle C tile onto the + # A/B staging bytes (pool = max(ab, c)) with a step-0 reuse barrier. True: + # C gets its own LDS bytes (pool = ab + c) and the barrier is elided -> + # lower small-tile latency, more LDS. Only affects epilogue == "cshuffle". + cshuffle_no_alias: bool = False @property def block_size(self) -> int: @@ -421,7 +427,7 @@ def kernel_name(self) -> str: f"a{self.warp_tile_m}x{self.warp_tile_n}x{self.warp_tile_k}", f"{self.pipeline}_{self.epilogue}", self.acc_epilogue.tag(), - flags={"async": self.async_dma}, + flags={"async": self.async_dma, "noalc": self.cshuffle_no_alias}, ) def validate(self) -> None: @@ -1814,7 +1820,10 @@ def _emit_cshuffle_epilogue( def d_addr(b_: IRBuilder, m_val: Value, n_val: Value): return D_desc.offset(b_, m=m_val, k_out=n_val) - _cshuffle_kwargs: dict = {"out_dtype": spec.data.dtype_d} + _cshuffle_kwargs: dict = { + "out_dtype": spec.data.dtype_d, + "no_alias": spec.cshuffle_no_alias, + } if spec.vector_size_c is not None: _cshuffle_kwargs["max_store_vec"] = spec.vector_size_c CShuffleEpilogue.from_grid(atom=spec.atom, grid=grid, **_cshuffle_kwargs).store( diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/gemm_universal.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/gemm_universal.py index 0f67982d3fc6..dea1472b92d5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/gemm_universal.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/gemm_universal.py @@ -242,6 +242,14 @@ class TraitSpec: # None (default): arch-resolved -> hints OFF on gfx950 (take the uplift), ON # elsewhere (preserve the historical emission). True/False forces the choice. emit_sched_hints: Optional[bool] = None + # cshuffle epilogue LDS aliasing. By default (False) the smem-pool packer + # aliases the cshuffle C tile onto the A/B staging bytes (pool = max(ab, c)), + # which needs a barrier between the main loop's last A/B read and the first + # C write (the epilogue "step-0 reuse barrier"). Setting True gives C its own + # LDS bytes (pool = ab + c) and elides that barrier -> lower latency for + # small tiles, at the cost of more LDS (lower occupancy) for large tiles. + # Only affects the cshuffle epilogue; False keeps byte-identical output. + cshuffle_no_alias: bool = False @dataclass(frozen=True) @@ -314,6 +322,7 @@ def kernel_name(self) -> str: "pref": tr.dtl_prefetch, "actt": tr.active_tile_skip, f"spk{tr.split_k}": tr.split_k > 1, + "noalc": tr.cshuffle_no_alias, }, ) @@ -2455,7 +2464,14 @@ def _emit_epilogue_cshuffle( record_runtime(b, N=N) # LDS staging tile: tile_m x tile_n of output storage dtype. - Cs = b.smem_alloc(storage_dtype, [t.tile_m, t.tile_n], name_hint="C_smem") + # ``cshuffle_no_alias`` marks it exclusive so the smem-pool packer gives it + # its own byte range instead of aliasing the A/B staging bytes. + Cs = b.smem_alloc( + storage_dtype, + [t.tile_m, t.tile_n], + name_hint="C_smem", + exclusive=spec.trait.cshuffle_no_alias, + ) # ---- step 0: reuse barrier. ---- # The common-LDS packer aliases this C staging tile onto the A/B @@ -2468,7 +2484,13 @@ def _emit_epilogue_cshuffle( # still reading for its tail MFMA -- a cross-wave WAR on the aliased pool. # (The single-buffer ``_emit_kloop_simple`` already has a trailing in-loop # barrier, which makes this one redundant-but-harmless there.) - b.sync() + # + # With ``cshuffle_no_alias`` the C tile has its own exclusive LDS bytes that + # never overlap A/B, so this cross-wave WAR cannot occur and the barrier is + # elided -- the small-tile latency win. The step-3 C-write->C-read barrier + # below is a genuine intra-epilogue RAW and always stays. + if not spec.trait.cshuffle_no_alias: + b.sync() warp_m_off = b.mul(warp_m_idx, b.const_i32(mfmas_m * t.warp_tile_m)) warp_n_off = b.mul(warp_n_idx, b.const_i32(mfmas_n * t.warp_tile_n)) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json index 1e3fdb3bb003..edf88ad71c70 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256.json @@ -242,6 +242,14 @@ "sha256": "91a908b70eb7559bb17c45ed2bad51d55f9319c4235c48eb06d28ff5e36b56e4", "status": "ok" }, + "gemm/gfx950/t128x128x32/cshuffle_no_alias": { + "arch": "gfx950", + "bytes": 90008, + "family": "gemm", + "kernel_name": "irhash_gemm_950_a_noalc_fp16_t128x128x32_w2x2x1_wt32x32x16_compv4_intrawave_cshuffle_pad_noalc", + "sha256": "f44a4d73475a7b371519e965930412329d1861295e712cdec80a78b5b75a9d18", + "status": "ok" + }, "gemm/gfx950/t64x128x32": { "arch": "gfx950", "bytes": 44732, @@ -308,16 +316,16 @@ "gfx1250": 2, "gfx90a": 3, "gfx942": 4, - "gfx950": 14 + "gfx950": 15 }, "expected_failures": 0, "families": { "conv": 7, "deep_fused_conv": 9, - "gemm": 12, + "gemm": 13, "moe": 9 }, - "ok_cases": 37 + "ok_cases": 38 } }, "llvm22": { @@ -562,6 +570,14 @@ "sha256": "e636915286c0c66a1925274dfcecb3775620b817bd247b762fdc4c7584c42baf", "status": "ok" }, + "gemm/gfx950/t128x128x32/cshuffle_no_alias": { + "arch": "gfx950", + "bytes": 90019, + "family": "gemm", + "kernel_name": "irhash_gemm_950_a_noalc_fp16_t128x128x32_w2x2x1_wt32x32x16_compv4_intrawave_cshuffle_pad_noalc", + "sha256": "5ef0e672c62be29dfc55c156c111eade24002744a9ae6ff38cec0f4f079f6abb", + "status": "ok" + }, "gemm/gfx950/t64x128x32": { "arch": "gfx950", "bytes": 44743, @@ -628,16 +644,16 @@ "gfx1250": 2, "gfx90a": 3, "gfx942": 4, - "gfx950": 14 + "gfx950": 15 }, "expected_failures": 0, "families": { "conv": 7, "deep_fused_conv": 9, - "gemm": 12, + "gemm": 13, "moe": 9 }, - "ok_cases": 37 + "ok_cases": 38 } } }, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/conv_implicit_gemm_emit.c b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/conv_implicit_gemm_emit.c index c77d329e5de2..85dafc23fecf 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/conv_implicit_gemm_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/conv_implicit_gemm_emit.c @@ -94,6 +94,13 @@ static int make_cfg(int idx, rocke_implicit_gemm_conv_spec_t* spec, const char** spec->chiplet_chunk_size = 64; *arch = "gfx950"; return 0; + case 10: + /* cshuffle with cshuffle_no_alias=True (idx 2 shape). */ + spec->problem = rocke_conv_problem_default(16, 112, 112, 128, 128, 3, 3); + spec->epilogue = "cshuffle"; + spec->cshuffle_no_alias = true; + *arch = "gfx950"; + return 0; default: return -1; } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/conv_implicit_gemm_emit.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/conv_implicit_gemm_emit.py index 65ff7f039bb4..c899ccc740cf 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/conv_implicit_gemm_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/conv_implicit_gemm_emit.py @@ -186,6 +186,27 @@ def _spec(idx: int): ), "gfx950", ) + if idx == 10: + # cshuffle epilogue with cshuffle_no_alias=True (idx 2 shape): the C tile + # gets its own exclusive LDS bytes and the step-0 reuse barrier is elided. + p = _cp(N=16, Hi=112, Wi=112, C=128, K=128, fy=3, fx=3) + return ( + ImplicitGemmConvSpec( + problem=p, + tile_m=64, + tile_n=64, + tile_k=64, + warp_m=2, + warp_n=2, + warp_tile_m=32, + warp_tile_n=32, + warp_tile_k=16, + pipeline="mem", + epilogue="cshuffle", + cshuffle_no_alias=True, + ), + "gfx950", + ) raise SystemExit(f"unknown config index {idx}") diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gemm_emit.c b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gemm_emit.c index a9ea0692c0b8..0383c5799e11 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gemm_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gemm_emit.c @@ -223,6 +223,25 @@ static int make_spec(int idx, rocke_gemm_universal_spec_t* spec) spec->block_size = 256; spec->batched = false; break; + case 10: /* test_noalc: cshuffle with cshuffle_no_alias=True (idx 1 shape) */ + spec->name = "test_noalc"; + spec->tile = (rocke_gemm_tile_spec_t){.tile_m = 256, + .tile_n = 256, + .tile_k = 64, + .warp_m = 4, + .warp_n = 4, + .warp_k = 1, + .warp_tile_m = 32, + .warp_tile_n = 32, + .warp_tile_k = 16}; + spec->trait.pipeline = "compv4"; + spec->trait.epilogue = "cshuffle"; + spec->trait.cshuffle_no_alias = true; + spec->data.dtype_a = "fp16"; + spec->wave_size = 64; + spec->block_size = 1024; + spec->batched = false; + break; default: return -1; } @@ -240,7 +259,7 @@ int main(int argc, char** argv) { if(argc < 2) { - fprintf(stderr, "usage: %s \n", argv[0]); + fprintf(stderr, "usage: %s \n", argv[0]); return 2; } int idx = atoi(argv[1]); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gemm_emit.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gemm_emit.py index c2f67c024042..fcdedf693e54 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gemm_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/gemm_emit.py @@ -238,6 +238,32 @@ def _spec(idx: int) -> UniversalGemmSpec: ), "gfx942", ) + if idx == 10: + # cshuffle epilogue with cshuffle_no_alias=True: the C staging tile gets + # its own exclusive LDS bytes (not aliased onto A/B) and the step-0 reuse + # barrier is elided. Same shape as idx 1 (the aliased cshuffle config) so + # the two configs isolate the no-alias knob. Both engines must agree. + return UniversalGemmSpec( + name="test_noalc", + tile=TileSpec( + tile_m=256, + tile_n=256, + tile_k=64, + warp_m=4, + warp_n=4, + warp_k=1, + warp_tile_m=32, + warp_tile_n=32, + warp_tile_k=16, + ), + trait=TraitSpec( + pipeline="compv4", epilogue="cshuffle", cshuffle_no_alias=True + ), + data=DataSpec(dtype_a="fp16"), + wave_size=64, + block_size=1024, + batched=False, + ) raise SystemExit(f"unknown config index {idx}") @@ -245,7 +271,7 @@ def main() -> int: return run_emit( _spec, build_universal_gemm, - usage="usage: gemm_emit.py \n", + usage="usage: gemm_emit.py \n", ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/rocke_ir_parity_harness.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/rocke_ir_parity_harness.py index 93a4be210743..a5ca7e5c69d3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/rocke_ir_parity_harness.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/rocke_ir_parity_harness.py @@ -64,6 +64,7 @@ def gemm_spec( pipeline, epilogue, wave_size, + cshuffle_no_alias=False, ): from rocke.instances.common.gemm_universal import ( DataSpec, @@ -92,17 +93,18 @@ def gemm_spec( pad_m=True, pad_n=True, pad_k=True, + cshuffle_no_alias=cshuffle_no_alias, ), data=DataSpec(dtype_a="fp16", dtype_b="fp16", dtype_c="fp16"), wave_size=wave_size, ) -def build_gemm(name, arch, *args): +def build_gemm(name, arch, *args, **kwargs): def _build(): from rocke.instances.common.gemm_universal import build_universal_gemm - return build_universal_gemm(gemm_spec(name, arch, *args), arch=arch) + return build_universal_gemm(gemm_spec(name, arch, *args, **kwargs), arch=arch) return _build @@ -302,6 +304,27 @@ def add(family, case_id, arch, build): 64, ), ) + add( + "gemm", + "gemm/gfx950/t128x128x32/cshuffle_no_alias", + "gfx950", + build_gemm( + "irhash_gemm_950_a_noalc", + "gfx950", + 128, + 128, + 32, + 2, + 2, + 32, + 32, + 16, + "compv4", + "cshuffle", + 64, + cshuffle_no_alias=True, + ), + ) add( "gemm", "gemm/gfx950/t64x128x32", From af6e17720696a507f1e651abdf1c12ea4a007d14 Mon Sep 17 00:00:00 2001 From: Aleksander Dudek Date: Mon, 27 Jul 2026 19:30:09 +0000 Subject: [PATCH 5/5] fix(rocke): per-case cshuffle LDS budget + validate smem exclusive arg - conv is_valid_spec sized LDS as max(ab, c) unconditionally, which under-counts when cshuffle_no_alias=True (C gets its own bytes, so the real footprint is ab + c) and could accept an over-budget kernel. Compute per case: max(ab, c) when aliased, ab + c when no-alias. Mirrored in the C++ engine (conv_implicit_gemm_conv_spec_descriptors.cpp). - rocke_smem_type now validates `exclusive` is 0 or 1 (error otherwise) instead of silently coercing; the Python SmemType already defaults it. - Add regression tests: no-alias elides the reuse barrier + marks C exclusive, and the conv LDS budget flips accept/reject with the flag. --- .../rocke/platform/cpp/core/ir/core_types.cpp | 5 +- ...nv_implicit_gemm_conv_spec_descriptors.cpp | 6 +- .../instances/common/conv_implicit_gemm.py | 6 +- .../tests/core/test_cshuffle_reuse_barrier.py | 95 +++++++++++++++++++ 4 files changed, 108 insertions(+), 4 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp index 60a160518b8f..8d730e9b3ae3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/core_types.cpp @@ -147,6 +147,9 @@ const rocke_type_t* rocke_smem_type( if(!elem) return (const rocke_type_t*)rocke_i_set_err( b, ROCKE_ERR_VALUE, "smem_type: NULL element type"); + if(exclusive != 0 && exclusive != 1) + return (const rocke_type_t*)rocke_i_set_err( + b, ROCKE_ERR_VALUE, "smem_type: exclusive must be 0 or 1, got %d", exclusive); if(rank < 0) rank = 0; @@ -167,7 +170,7 @@ const rocke_type_t* rocke_smem_type( t->elem = elem; t->shape = shape_copy; t->rank = rank; - t->smem_exclusive = exclusive ? 1 : 0; + t->smem_exclusive = exclusive; /* guarded to 0/1 above */ /* Build the "[d0xd1x...]" body, then the full canonical name. The Python * form is f"smem<{elem.name}, [{'x'.join(...)}]>". */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp index 5f0630738820..03d586dc8d01 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/conv_implicit_gemm_conv_spec_descriptors.cpp @@ -742,7 +742,9 @@ bool rocke_implicit_gemm_conv_is_valid_spec(const rocke_implicit_gemm_conv_spec_ * _ab_bytes = (a_rows*a_cols + b_rows*b_cols) * _ab_dtype_bytes * _ab_lds = _ab_bytes * (2 if double_buffer else 1) * _c_lds = tile_m * tile_n * _c_dtype_bytes (if epilogue == "cshuffle") - * _total_lds = max(_ab_lds, _c_lds) (not sum -- they can overlap in pool) */ + * _total_lds = (ab_lds + c_lds) if cshuffle_no_alias else max(ab_lds, c_lds) + * (aliased default overlaps C onto A/B -> max; no-alias gives C + * its own bytes -> sum). Mirrors Python is_valid_spec. */ { rocke_conv_lds_layout_t lds_layout; char lds_reason[256]; @@ -776,7 +778,7 @@ bool rocke_implicit_gemm_conv_is_valid_spec(const rocke_implicit_gemm_conv_spec_ int c_dtype_bytes = (s->dtype_d && strcmp(s->dtype_d, "fp32") == 0) ? 4 : 2; c_lds = is_cshuffle ? (s->tile_m * s->tile_n * c_dtype_bytes) : 0; } - total_lds = (ab_lds > c_lds) ? ab_lds : c_lds; + total_lds = s->cshuffle_no_alias ? (ab_lds + c_lds) : ((ab_lds > c_lds) ? ab_lds : c_lds); if(!rocke_archtarget_fits_lds(target, (long)total_lds)) { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/conv_implicit_gemm.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/conv_implicit_gemm.py index 094a9d4ec13f..69120f729767 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/conv_implicit_gemm.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/instances/common/conv_implicit_gemm.py @@ -579,7 +579,11 @@ def is_valid_spec(spec: ImplicitGemmConvSpec, arch: str = "gfx950") -> Tuple[boo _c_lds = ( spec.tile_m * spec.tile_n * _c_dtype_bytes if spec.epilogue == "cshuffle" else 0 ) - _total_lds = max(_ab_lds, _c_lds) + # cshuffle_no_alias decides whether the cshuffle C tile shares the A/B pool + # bytes, so the LDS budget must be computed per case: + # OFF (default, aliased): C reuses the A/B bytes -> pool = max(ab, c) + # ON (exclusive): C gets its own LDS bytes -> pool = ab + c + _total_lds = (_ab_lds + _c_lds) if spec.cshuffle_no_alias else max(_ab_lds, _c_lds) if not target.fits_lds(_total_lds): return False, ( f"LDS budget {_total_lds} bytes " diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_cshuffle_reuse_barrier.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_cshuffle_reuse_barrier.py index adaae7b7d742..3eb8925f2b77 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_cshuffle_reuse_barrier.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_cshuffle_reuse_barrier.py @@ -125,6 +125,101 @@ def test_conv_cshuffle_has_reuse_barrier(): _assert_barrier_before_first_c_store(build_implicit_gemm_conv(spec, arch="gfx950")) +def test_gemm_cshuffle_no_alias_elides_barrier_and_marks_exclusive(): + """With ``cshuffle_no_alias=True`` the C tile gets its own exclusive LDS + bytes, so the step-0 reuse barrier is elided: there is NO barrier between the + C_smem allocation and the first C store (the small-tile latency win).""" + from rocke.instances.common.gemm_universal import ( + TileSpec, + TraitSpec, + UniversalGemmSpec, + build_universal_gemm, + ) + + spec = UniversalGemmSpec( + name="vg", + tile=TileSpec( + tile_m=128, + tile_n=128, + tile_k=32, + warp_m=2, + warp_n=2, + warp_tile_m=32, + warp_tile_n=32, + warp_tile_k=16, + ), + trait=TraitSpec( + pipeline="compv4", + scheduler="intrawave", + epilogue="cshuffle", + cshuffle_no_alias=True, + ), + ) + ops = _flatten(build_universal_gemm(spec).body, []) + c_allocs = [ + i + for i, o in enumerate(ops) + if o.name == "tile.smem_alloc" and "C_smem" in o.results[0].name + ] + assert c_allocs, "expected a C_smem allocation" + c_idx = c_allocs[0] + assert ( + ops[c_idx].results[0].type.exclusive + ), "no_alias C tile must be an exclusive smem allocation" + later_stores = [ + i + for i, o in enumerate(ops) + if o.name.startswith("tile.smem_store") and i > c_idx + ] + assert later_stores, "expected at least one store into the C staging tile" + barriers = [ + i for i in range(c_idx + 1, later_stores[0]) if ops[i].name in _BARRIERS + ] + assert not barriers, ( + "cshuffle_no_alias must elide the step-0 reuse barrier (C has its own " + "LDS bytes, so there is no A/B WAR hazard to guard)" + ) + + +def test_conv_cshuffle_no_alias_lds_budget_is_per_case(): + """is_valid_spec must size the LDS budget per ``cshuffle_no_alias``: aliased + uses max(ab, c); no-alias uses ab + c. A spec whose max(ab, c) fits gfx950's + LDS but whose ab + c overflows it must be accepted aliased and rejected + no-alias (regression for yraparti's PR #8844 review comment).""" + from rocke.instances.common.conv_implicit_gemm import ( + ConvProblem, + ImplicitGemmConvSpec, + is_valid_spec, + ) + + def _mk(no_alias): + return ImplicitGemmConvSpec( + problem=ConvProblem(N=4, Hi=28, Wi=28, C=64, K=64, Y=3, X=3), + tile_m=256, + tile_n=256, + tile_k=32, + warp_m=2, + warp_n=2, + warp_tile_m=32, + warp_tile_n=32, + warp_tile_k=16, + pipeline="mem", + epilogue="cshuffle", + wave_size=64, + cshuffle_no_alias=no_alias, + ) + + ok_alias, _ = is_valid_spec(_mk(False), "gfx950") + ok_noalias, why = is_valid_spec(_mk(True), "gfx950") + assert ok_alias, "aliased cshuffle (pool = max(ab, c)) should fit gfx950 LDS" + assert not ok_noalias, ( + "no-alias cshuffle (pool = ab + c) should exceed gfx950 LDS and be " + "rejected; sizing it as max(ab, c) would silently accept an over-budget " + "kernel" + ) + assert "LDS budget" in why + + if __name__ == "__main__": # pragma: no cover import sys