From 1c835dc0a56f4556894a1fe17bdecfc96254b44e Mon Sep 17 00:00:00 2001 From: yihonglie Date: Sat, 26 Sep 2026 09:36:26 -0500 Subject: [PATCH 01/13] [MLA v4 nm] Cost-model num_kv_splits auto pick and kv_len_hint The gfx950 v4 nm auto split pick (ignore_total_kv=1) maximizes CU occupancy only, so it prefers whole multiples of cu_num: for gqa=128 single-token decode it picks 13-16 splits (several workgroup waves) for batches whose KV already fits one wave, paying extra partial writes and a wider stage-2 merge for no added parallelism. - On gfx950, pick num_kv_splits by minimizing a fitted latency model: workgroup waves x per-wave KV time, plus the partial-write/stage-2 cost of splitting. Other archs keep the existing pick. - Add kv_len_hint: callers that pass capacity-sized kv_page_indices (e.g. CUDA-graph decode) cannot be planned from total_kv / num_seqs, so they pass the per-seq KV length to plan for. The pick depends only on shapes and this int, so it stays fixed across graph capture and replay. - Fix the stage-2 merge for a caller-supplied split_indptr with one split per seq on a grid > 1: it took the in-place BF16 copy (keyed on split_indptr[-1] == num_seqs) although the kernel wrote FP32 partials, returning NaN. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie --- aiter/mla.py | 53 ++++++++++++++++++++++++++++++++++++-- op_tests/test_mla_v4_nm.py | 48 +++++++++++++++++++++++++++++++--- 2 files changed, 96 insertions(+), 5 deletions(-) diff --git a/aiter/mla.py b/aiter/mla.py index 61db158267d..2ddf2d8b428 100644 --- a/aiter/mla.py +++ b/aiter/mla.py @@ -359,6 +359,38 @@ def get_meta_param( return num_kv_splits, num_kv_splits_indptr +# gfx950 v4 nm decode latency model used to pick `num_kv_splits`: +# +# t(i) ~= L + W(i) * (C + A * kv_len / i) + [i > 1] * (S + D * num_seqs * i) +# W(i) = ceil(num_seqs * tg_factor * i / cu_num) (workgroup waves) +# +# A is the per-token KV cost of one workgroup, C the per-wave fixed cost, and +# S + D * num_seqs * i the FP32 partial write + stage-2 merge that every split +# launch pays. Least-squares fit (relative error) on MI355X, gqa=128, qSeqLen=1, +# CUDA-graph replay, 1..16 splits over uniform-KV shapes (7..448 seqs, +# kv_len 192..8320). The previous occupancy-only pick (ignore_total_kv=1) +# rewards filling whole multiples of cu_num, e.g. 13-16 splits for 49-119 seqs +# whose KV fits one wave. +_V4_NM_SPLIT_COST_GFX950 = (12.74, 3.21, 0.0300, 3.38, 0.076) # L, C, A, S, D (us) +_V4_NM_MAX_SPLITS = 16 + + +@functools.lru_cache(maxsize=1024) +def _v4_nm_pick_num_kv_splits(num_seqs, tg_factor, kv_len, cu_num): + """Cost-model split count for gfx950 v4 nm decode (see model above).""" + L, C, A, S, D = _V4_NM_SPLIT_COST_GFX950 + kv_len = max(1, kv_len) + best_cost, best_splits = None, 1 + for i in range(1, _V4_NM_MAX_SPLITS + 1): + waves = -(-num_seqs * tg_factor * i // cu_num) + cost = L + waves * (C + A * kv_len / i) + if i > 1: + cost += S + D * num_seqs * i + if best_cost is None or cost < best_cost: + best_cost, best_splits = cost, i + return best_splits + + # Persistent MLA-decode kernel gate: the persistent kernel # ("mla_a16w16_qh16..._ps") is slower than the non-persistent split-KV kernel # ("mla_dec_stage1...") above a concurrency threshold (~batch 16-64 on gfx950 bf16 @@ -1677,6 +1709,7 @@ def mla_decode_fwd_v4_nm( # the nm path (page_size=1 -> kv_seq_len comes from the token-level kv_indptr). # None flows through to a nullptr kernarg; the host guards the deref and the # kernel never loads through it, so no buffer is allocated. + kv_len_hint=None, # int; per-seq KV length the auto split pick optimizes for ): """v4 MLA decode forward. @@ -1703,6 +1736,11 @@ def mla_decode_fwd_v4_nm( for the kernel tile) -- identical to `mla_decode_fwd`'s non-persistent path. Pass an explicit int to override. Note V4 nm is always non-persistent, so only that branch of `get_meta_param` applies. + On gfx950 the auto pick instead minimizes a fitted latency model + (`_v4_nm_pick_num_kv_splits`) for sequences of `kv_len_hint` KV tokens. + Pass the longest KV length the call can see (e.g. window + top-k for a + sparse layer) when `kv_page_indices` is a capacity-sized buffer; without + it the model uses `ceil(total_kv / num_seqs)`. Multi-pass mode (`num_kv_splits > 1`): 1. If `split_indptr` is None, build a uniform one: @@ -1771,8 +1809,14 @@ def mla_decode_fwd_v4_nm( # could shrink it and desync the buffer shapes). Only synthesize a # uniform split_indptr if the caller didn't pass one. total_kv = kv_page_indices.shape[0] + tg_factor = max(1, -(-num_heads // 64)) # ceil(num_heads / 64) + if num_kv_splits is None and get_gfx() == "gfx950": + if kv_len_hint is None: + kv_len_hint = -(-total_kv // max(1, num_seqs)) + num_kv_splits = _v4_nm_pick_num_kv_splits( + num_seqs, tg_factor, int(kv_len_hint), get_cu_num() + ) if num_kv_splits is None or split_indptr is None: - tg_factor = max(1, -(-num_heads // 64)) # ceil(num_heads / 64) # v4 nm forces occupancy-only split selection: ignore total_kv so the # split count is driven purely by CU occupancy (drops the avg_kv HBM # term + fp8 min-block cap in get_meta_param). @@ -1921,7 +1965,12 @@ def mla_decode_fwd_v4_nm( 0, # stride_lse_bs (unused, HAS_FINAL_LSE=False) page_size=1, # v4 nm KV cache is page_size=1 KV_INDPTR_IS_PAGE_LEVEL=False, # page_size=1 -> token-level indptr - MAYBE_FINAL_OUT=True, + # The kernel only writes the packed-BF16 result in place when + # num_kv_splits == 1, which never reaches this merge. A ragged + # split_indptr whose entries are all 1 still gets FP32 partials, + # so the in-place copy (keyed on split_indptr[-1] == num_seqs) must + # stay off. + MAYBE_FINAL_OUT=False, HAS_FINAL_LSE=False, USE_VALID_SPLIT_COUNT_REDUCE=int(num_kv_splits > 1), BATCH_NUM=num_seqs, diff --git a/op_tests/test_mla_v4_nm.py b/op_tests/test_mla_v4_nm.py index 3254de5e74a..03a44c64242 100644 --- a/op_tests/test_mla_v4_nm.py +++ b/op_tests/test_mla_v4_nm.py @@ -1023,7 +1023,9 @@ def test_v4_nm_accuracy_and_perf(): _run_one_point(batch=2, kv_seq_lens=64, q_seq_logical=1, seed=0) -def _run_varlen_point(kv_lens, gqa_ratio=128, seed=0, attn_sink=True): +def _run_varlen_point( + kv_lens, gqa_ratio=128, seed=0, attn_sink=True, split_kwargs=None +): """Accuracy at a RAGGED (per-seq variable kv_len) decode shape with auto-split (num_kv_splits=None). @@ -1094,12 +1096,11 @@ def _run_varlen_point(kv_lens, gqa_ratio=128, seed=0, attn_sink=True): kv_indptr=kv_indptr, kv_page_indices=kv_page_indices, kv_last_page_lens=kv_last_page_lens, - split_indptr=None, max_seqlen_q=1, sink=sink, sm_scale=sm_scale, out_16_nosplit=0, - num_kv_splits=None, + **(split_kwargs or {"split_indptr": None, "num_kv_splits": None}), ) # Single/multi-split output contract (see ATOM paged_decode.py:986): the # merged result lands in `output` for resolved splits > 1; the single-pass @@ -1141,6 +1142,47 @@ def test_v4_nm_varlen_ragged_kv_tail_split_guard(): _run_varlen_point([66, 50, 40, 33]) # all-short +@pytest.mark.parametrize( + "num_seqs,kv_len,expected", + [ + # One wave of workgroups: split only when KV is long enough to pay + # for the partial write + stage-2 merge. + (7, 1152, 8), + (7, 384, 5), + (35, 1152, 3), + (56, 1152, 2), + # 49..119 seqs with short KV: the occupancy-only pick chose 13-16 + # splits (5-14 waves) here and ran 3-7x slower than one split. + (49, 384, 1), + (119, 384, 1), + (126, 1152, 1), + # Past one wave, long KV still splits to even out the tail wave. + (140, 8320, 4), + ], +) +def test_v4_nm_split_planner_picks(num_seqs, kv_len, expected): + assert aiter.mla._v4_nm_pick_num_kv_splits(num_seqs, 2, kv_len, 256) == expected + + +@needs_gfx950 +def test_v4_nm_kv_len_hint_accuracy(): + """Auto split driven by `kv_len_hint` (the caller's worst-case KV length).""" + _run_varlen_point([1152, 900, 64, 1152], split_kwargs={"kv_len_hint": 1152}) + _run_varlen_point([8320, 200], split_kwargs={"kv_len_hint": 8320}) + + +@needs_gfx950 +def test_v4_nm_one_split_per_seq_on_wide_grid(): + """Every seq on one split while the grid has 4: the kernel still writes + FP32 partials, so the stage-2 merge must not take its in-place BF16 copy + (it used to, keyed on split_indptr[-1] == num_seqs, and returned NaN).""" + kv_lens = [384, 384, 384, 384] + split_indptr = torch.arange(len(kv_lens) + 1, dtype=torch.int32, device="cuda") + _run_varlen_point( + kv_lens, split_kwargs={"num_kv_splits": 4, "split_indptr": split_indptr} + ) + + @needs_gfx950 def test_v4_nm_ragged_short_seq_no_corrupt(): """A short seq must not be corrupted by over-allocated splits (host ragged From 31020169ebc79e6b2f00888758235073fbd1bfb9 Mon Sep 17 00:00:00 2001 From: yihonglie Date: Mon, 28 Sep 2026 00:24:56 -0500 Subject: [PATCH 02/13] [MLA v4 nm] Expose the split plan to callers instead of kv_len_hint Planning inside mla_decode_fwd_v4_nm needs a per-seq KV length, and the only one the wrapper can see, kv_page_indices.shape[0] / num_seqs, is meaningless for CUDA-graph callers whose index buffers are capacity-sized. - Add get_mla_v4_nm_split_plan(num_seqs, num_heads, kv_len, split_indptr=): returns the split count (cost model on gfx950, occupancy pick elsewhere) and fills a caller-owned persistent split_indptr in place. Callers build it in their metadata step and pass both to the wrapper. - Drop kv_len_hint; the wrapper's own auto pick is unchanged from main. - Tests: plan fill, accuracy through the wrapper, and CUDA-graph replay with different KV lengths under a captured plan. _run_varlen_point now returns its checkAllclose error so the new tests gate on it. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie --- aiter/mla.py | 75 ++++++++++++++++----- op_tests/test_mla_v4_nm.py | 130 +++++++++++++++++++++++++++++++++++-- 2 files changed, 181 insertions(+), 24 deletions(-) diff --git a/aiter/mla.py b/aiter/mla.py index 2ddf2d8b428..a7d245d8997 100644 --- a/aiter/mla.py +++ b/aiter/mla.py @@ -391,6 +391,55 @@ def _v4_nm_pick_num_kv_splits(num_seqs, tg_factor, kv_len, cu_num): return best_splits +def get_mla_v4_nm_split_plan( + num_seqs, num_heads, kv_len, *, split_indptr=None, device="cuda" +): + """KV split plan for `mla_decode_fwd_v4_nm`: `(num_kv_splits, split_indptr)`. + + `num_seqs` / `num_heads` are the decode call's `qo_indptr.shape[0] - 1` + and `q.size(1)`; `kv_len` is the per-seq KV length to plan for (the + longest one the call can see). On gfx950 the split count minimizes the + latency model above; other archs keep the occupancy-only pick the + wrapper uses when no plan is passed. + + The split count is a host int that depends only on these arguments, and + `split_indptr` is the uniform `[0, s, 2s, ..., num_seqs * s]`. Pass a + persistent int32 `split_indptr` of at least `num_seqs + 1` entries to have + it filled in place (one device launch, no host sync): CUDA-graph callers + keep the buffer alive across replays and rebuild the plan with the same + arguments before each replay. Pass both results to the wrapper. + """ + tg_factor = max(1, -(-num_heads // 64)) # ceil(num_heads / 64) + if get_gfx() == "gfx950": + num_kv_splits = _v4_nm_pick_num_kv_splits( + num_seqs, tg_factor, int(kv_len), get_cu_num() + ) + else: + num_kv_splits, _ = get_meta_param( + None, + num_seqs, + num_seqs * max(1, int(kv_len)), + num_heads, + 1, + dtypes.fp8, + tg_factor, + 1, # ignore_total_kv, as in mla_decode_fwd_v4_nm + ) + if split_indptr is None: + split_indptr = torch.empty(num_seqs + 1, dtype=torch.int32, device=device) + else: + split_indptr = split_indptr[: num_seqs + 1] + torch.arange( + 0, + (num_seqs + 1) * num_kv_splits, + num_kv_splits, + dtype=torch.int32, + device=split_indptr.device, + out=split_indptr, + ) + return num_kv_splits, split_indptr + + # Persistent MLA-decode kernel gate: the persistent kernel # ("mla_a16w16_qh16..._ps") is slower than the non-persistent split-KV kernel # ("mla_dec_stage1...") above a concurrency threshold (~batch 16-64 on gfx950 bf16 @@ -1709,7 +1758,6 @@ def mla_decode_fwd_v4_nm( # the nm path (page_size=1 -> kv_seq_len comes from the token-level kv_indptr). # None flows through to a nullptr kernarg; the host guards the deref and the # kernel never loads through it, so no buffer is allocated. - kv_len_hint=None, # int; per-seq KV length the auto split pick optimizes for ): """v4 MLA decode forward. @@ -1736,11 +1784,10 @@ def mla_decode_fwd_v4_nm( for the kernel tile) -- identical to `mla_decode_fwd`'s non-persistent path. Pass an explicit int to override. Note V4 nm is always non-persistent, so only that branch of `get_meta_param` applies. - On gfx950 the auto pick instead minimizes a fitted latency model - (`_v4_nm_pick_num_kv_splits`) for sequences of `kv_len_hint` KV tokens. - Pass the longest KV length the call can see (e.g. window + top-k for a - sparse layer) when `kv_page_indices` is a capacity-sized buffer; without - it the model uses `ceil(total_kv / num_seqs)`. + Callers that know their per-seq KV length (and CUDA-graph callers, + whose `kv_page_indices` is typically capacity-sized) should build the + plan with `get_mla_v4_nm_split_plan` and pass both `num_kv_splits` and + `split_indptr`. Multi-pass mode (`num_kv_splits > 1`): 1. If `split_indptr` is None, build a uniform one: @@ -1809,14 +1856,8 @@ def mla_decode_fwd_v4_nm( # could shrink it and desync the buffer shapes). Only synthesize a # uniform split_indptr if the caller didn't pass one. total_kv = kv_page_indices.shape[0] - tg_factor = max(1, -(-num_heads // 64)) # ceil(num_heads / 64) - if num_kv_splits is None and get_gfx() == "gfx950": - if kv_len_hint is None: - kv_len_hint = -(-total_kv // max(1, num_seqs)) - num_kv_splits = _v4_nm_pick_num_kv_splits( - num_seqs, tg_factor, int(kv_len_hint), get_cu_num() - ) if num_kv_splits is None or split_indptr is None: + tg_factor = max(1, -(-num_heads // 64)) # ceil(num_heads / 64) # v4 nm forces occupancy-only split selection: ignore total_kv so the # split count is driven purely by CU occupancy (drops the avg_kv HBM # term + fp8 min-block cap in get_meta_param). @@ -1966,10 +2007,10 @@ def mla_decode_fwd_v4_nm( page_size=1, # v4 nm KV cache is page_size=1 KV_INDPTR_IS_PAGE_LEVEL=False, # page_size=1 -> token-level indptr # The kernel only writes the packed-BF16 result in place when - # num_kv_splits == 1, which never reaches this merge. A ragged - # split_indptr whose entries are all 1 still gets FP32 partials, - # so the in-place copy (keyed on split_indptr[-1] == num_seqs) must - # stay off. + # num_kv_splits == 1, which never reaches this merge. A + # caller-supplied split_indptr with one split per seq on a wider + # grid still gets FP32 partials, so the in-place copy (keyed on + # split_indptr[-1] == num_seqs) must stay off. MAYBE_FINAL_OUT=False, HAS_FINAL_LSE=False, USE_VALID_SPLIT_COUNT_REDUCE=int(num_kv_splits > 1), diff --git a/op_tests/test_mla_v4_nm.py b/op_tests/test_mla_v4_nm.py index 03a44c64242..49455cc742b 100644 --- a/op_tests/test_mla_v4_nm.py +++ b/op_tests/test_mla_v4_nm.py @@ -17,7 +17,7 @@ import aiter import aiter.mla # main no longer auto-imports submodules; need explicit from aiter import dtypes -from aiter.jit.utils.chip_info import get_gfx +from aiter.jit.utils.chip_info import get_cu_num, get_gfx from aiter.test_common import checkAllclose, run_perftest # --------------------------------------------------------------------------- @@ -1112,7 +1112,7 @@ def _run_varlen_point( f"\n[v4 nm varlen] gqa={gqa} kv_lens={kv_lens} total_kv={total_kv} " f"resolved_splits={resolved}" ) - checkAllclose( + err = checkAllclose( out_ref.float(), out_asm, rtol=3e-2, @@ -1120,6 +1120,7 @@ def _run_varlen_point( tol_err_ratio=0.02, msg=f"mla_v4_nm varlen [fp8_dequant_ref vs asm] kv_lens={kv_lens}", ) + return float(err or 0), out_asm @needs_gfx950 @@ -1165,10 +1166,123 @@ def test_v4_nm_split_planner_picks(num_seqs, kv_len, expected): @needs_gfx950 -def test_v4_nm_kv_len_hint_accuracy(): - """Auto split driven by `kv_len_hint` (the caller's worst-case KV length).""" - _run_varlen_point([1152, 900, 64, 1152], split_kwargs={"kv_len_hint": 1152}) - _run_varlen_point([8320, 200], split_kwargs={"kv_len_hint": 8320}) +def test_v4_nm_split_plan_fills_buffer_in_place(): + buf = torch.full((10,), -1, dtype=torch.int32, device="cuda") + splits, indptr = aiter.mla.get_mla_v4_nm_split_plan(7, 128, 1152, split_indptr=buf) + assert splits == aiter.mla._v4_nm_pick_num_kv_splits(7, 2, 1152, get_cu_num()) + assert indptr.data_ptr() == buf.data_ptr() + assert buf.tolist() == [i * splits for i in range(8)] + [-1, -1] + + +@needs_gfx950 +@pytest.mark.parametrize( + "kv_lens,kv_len", + [ + ([1152, 900, 64, 1152], 1152), # planned for the longest seq + ([8320, 200], 8320), + ([100, 60, 128], 128), # short, window-sized KV + ], +) +def test_v4_nm_split_plan_accuracy(kv_lens, kv_len): + splits, split_indptr = aiter.mla.get_mla_v4_nm_split_plan(len(kv_lens), 128, kv_len) + err, out = _run_varlen_point( + kv_lens, + split_kwargs={"num_kv_splits": splits, "split_indptr": split_indptr}, + ) + assert torch.isfinite(out).all() + assert err < 0.02 + + +@needs_gfx950 +def test_v4_nm_split_plan_cudagraph_replay(): + """Plan built once into a persistent buffer, captured, then replayed with + different per-seq KV lengths written into the same kv_indptr.""" + device, gqa, n, kv_cap = "cuda", 128, 14, 1152 + torch.manual_seed(0) + sm_scale = 1.0 / (_QUANT_D**0.5) + qp, qr = _native_to_2buff_for_asm( + torch.randn(n, gqa, _QUANT_D, dtype=dtypes.bf16, device=device) + ) + kp, kr = _native_to_2buff_for_asm( + torch.randn( + n * kv_cap, + PAGE_SIZE, + NUM_KV_HEADS, + _QUANT_D, + dtype=dtypes.bf16, + device=device, + ) + ) + qr, kr = qr.contiguous(), kr.contiguous() + sink = torch.randn(gqa, dtype=torch.float32, device=device) + qo_indptr = torch.arange(n + 1, dtype=torch.int32, device=device) + kv_indptr = torch.zeros(n + 1, dtype=torch.int32, device=device) + kv_page_indices = torch.randperm(n * kv_cap, device=device).to(torch.int32) + kv_last_page_lens = torch.ones(n, dtype=torch.int32, device=device) + output = torch.empty((n, gqa, V_HEAD_DIM), dtype=dtypes.bf16, device=device) + split_buf = torch.empty(n + 1, dtype=torch.int32, device=device) + + def set_lens(lens): + kv_indptr.copy_(torch.tensor([0] + np.cumsum(lens).tolist(), dtype=torch.int32)) + + splits, split_indptr = aiter.mla.get_mla_v4_nm_split_plan( + n, gqa, kv_cap, split_indptr=split_buf + ) + assert splits > 1 # exercise the stage-2 path under the graph + + def run(): + aiter.mla.mla_decode_fwd_v4_nm( + qp, + qr, + kp, + kr, + output, + qo_indptr, + kv_indptr, + kv_page_indices, + 1, + sink=sink, + sm_scale=sm_scale, + num_kv_splits=splits, + split_indptr=split_indptr, + ) + + set_lens([kv_cap] * n) + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + run() + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run() + + for lens in ([kv_cap] * n, [1 + 83 * i for i in range(n)], [64] * n): + set_lens(lens) + graph.replay() + torch.cuda.synchronize() + ref, _ = _torch_attn_decode_fp8_dequant_ref( + qp, + qr, + kp, + kr, + qo_indptr, + kv_indptr, + kv_page_indices, + kv_last_page_lens, + sm_scale, + attn_sink=sink, + ) + assert torch.isfinite(output).all() + err = checkAllclose( + ref.float(), + output.float(), + rtol=3e-2, + atol=3e-2, + tol_err_ratio=0.02, + msg=f"mla_v4_nm split plan graph replay lens={lens[:3]}...", + ) + assert float(err or 0) < 0.02 @needs_gfx950 @@ -1178,9 +1292,11 @@ def test_v4_nm_one_split_per_seq_on_wide_grid(): (it used to, keyed on split_indptr[-1] == num_seqs, and returned NaN).""" kv_lens = [384, 384, 384, 384] split_indptr = torch.arange(len(kv_lens) + 1, dtype=torch.int32, device="cuda") - _run_varlen_point( + err, out = _run_varlen_point( kv_lens, split_kwargs={"num_kv_splits": 4, "split_indptr": split_indptr} ) + assert torch.isfinite(out).all() + assert err < 0.02 @needs_gfx950 From 1f0176a396981646689911db1ae4b9c7c7715c04 Mon Sep 17 00:00:00 2001 From: yihonglie Date: Mon, 28 Sep 2026 02:20:07 -0500 Subject: [PATCH 03/13] [MLA v4 nm] Return the split plan as a NamedTuple, per-arch cost table - get_mla_v4_nm_split_plan returns MlaV4NmSplitPlan(num_kv_splits, split_indptr). The field names are the wrapper's kwargs, so callers pass a plan on as mla_decode_fwd_v4_nm(..., **plan._asdict()). - Latency-model coefficients live in a per-arch table; archs without an entry keep the occupancy-only pick. Only gfx950 is fitted. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie --- aiter/mla.py | 66 +++++++++++++++++++++++--------------- op_tests/test_mla_v4_nm.py | 32 +++++++++--------- 2 files changed, 56 insertions(+), 42 deletions(-) diff --git a/aiter/mla.py b/aiter/mla.py index a7d245d8997..e1e62c11c57 100644 --- a/aiter/mla.py +++ b/aiter/mla.py @@ -5,6 +5,7 @@ import functools import os +from typing import NamedTuple import torch import triton @@ -359,60 +360,73 @@ def get_meta_param( return num_kv_splits, num_kv_splits_indptr -# gfx950 v4 nm decode latency model used to pick `num_kv_splits`: +# Per-arch v4 nm decode latency model used to pick `num_kv_splits`: # # t(i) ~= L + W(i) * (C + A * kv_len / i) + [i > 1] * (S + D * num_seqs * i) # W(i) = ceil(num_seqs * tg_factor * i / cu_num) (workgroup waves) # # A is the per-token KV cost of one workgroup, C the per-wave fixed cost, and # S + D * num_seqs * i the FP32 partial write + stage-2 merge that every split -# launch pays. Least-squares fit (relative error) on MI355X, gqa=128, qSeqLen=1, -# CUDA-graph replay, 1..16 splits over uniform-KV shapes (7..448 seqs, -# kv_len 192..8320). The previous occupancy-only pick (ignore_total_kv=1) -# rewards filling whole multiples of cu_num, e.g. 13-16 splits for 49-119 seqs -# whose KV fits one wave. -_V4_NM_SPLIT_COST_GFX950 = (12.74, 3.21, 0.0300, 3.38, 0.076) # L, C, A, S, D (us) +# launch pays. The occupancy-only pick (ignore_total_kv=1) instead rewards +# filling whole multiples of cu_num, e.g. 13-16 splits for 49-119 seqs whose KV +# fits one wave. +# +# gfx950: least-squares fit (relative error) on MI355X, gqa=128, qSeqLen=1, +# CUDA-graph replay, 1..16 splits over uniform-KV shapes (7..448 seqs, kv_len +# 192..8320). Archs without an entry keep the occupancy-only pick. +_V4_NM_SPLIT_COST = { + "gfx950": (12.74, 3.21, 0.0300, 3.38, 0.076), # L, C, A, S, D (us) +} _V4_NM_MAX_SPLITS = 16 +class MlaV4NmSplitPlan(NamedTuple): + """KV split plan for `mla_decode_fwd_v4_nm`; the fields are its kwargs.""" + + num_kv_splits: int + split_indptr: torch.Tensor + + @functools.lru_cache(maxsize=1024) -def _v4_nm_pick_num_kv_splits(num_seqs, tg_factor, kv_len, cu_num): - """Cost-model split count for gfx950 v4 nm decode (see model above).""" - L, C, A, S, D = _V4_NM_SPLIT_COST_GFX950 +def _v4_nm_pick_num_kv_splits(cost, num_seqs, tg_factor, kv_len, cu_num): + """Split count minimizing the latency model above with coefficients `cost`.""" + L, C, A, S, D = cost kv_len = max(1, kv_len) best_cost, best_splits = None, 1 for i in range(1, _V4_NM_MAX_SPLITS + 1): waves = -(-num_seqs * tg_factor * i // cu_num) - cost = L + waves * (C + A * kv_len / i) + t = L + waves * (C + A * kv_len / i) if i > 1: - cost += S + D * num_seqs * i - if best_cost is None or cost < best_cost: - best_cost, best_splits = cost, i + t += S + D * num_seqs * i + if best_cost is None or t < best_cost: + best_cost, best_splits = t, i return best_splits def get_mla_v4_nm_split_plan( num_seqs, num_heads, kv_len, *, split_indptr=None, device="cuda" -): - """KV split plan for `mla_decode_fwd_v4_nm`: `(num_kv_splits, split_indptr)`. +) -> MlaV4NmSplitPlan: + """KV split plan for `mla_decode_fwd_v4_nm`. `num_seqs` / `num_heads` are the decode call's `qo_indptr.shape[0] - 1` and `q.size(1)`; `kv_len` is the per-seq KV length to plan for (the - longest one the call can see). On gfx950 the split count minimizes the - latency model above; other archs keep the occupancy-only pick the - wrapper uses when no plan is passed. + longest one the call can see). The split count minimizes the latency + model above where the arch has coefficients, and is the wrapper's own + occupancy-only pick elsewhere. The split count is a host int that depends only on these arguments, and `split_indptr` is the uniform `[0, s, 2s, ..., num_seqs * s]`. Pass a persistent int32 `split_indptr` of at least `num_seqs + 1` entries to have it filled in place (one device launch, no host sync): CUDA-graph callers keep the buffer alive across replays and rebuild the plan with the same - arguments before each replay. Pass both results to the wrapper. + arguments before each replay. Pass the plan on as + `mla_decode_fwd_v4_nm(..., **plan._asdict())`. """ tg_factor = max(1, -(-num_heads // 64)) # ceil(num_heads / 64) - if get_gfx() == "gfx950": + cost = _V4_NM_SPLIT_COST.get(get_gfx()) + if cost is not None: num_kv_splits = _v4_nm_pick_num_kv_splits( - num_seqs, tg_factor, int(kv_len), get_cu_num() + cost, num_seqs, tg_factor, int(kv_len), get_cu_num() ) else: num_kv_splits, _ = get_meta_param( @@ -437,7 +451,7 @@ def get_mla_v4_nm_split_plan( device=split_indptr.device, out=split_indptr, ) - return num_kv_splits, split_indptr + return MlaV4NmSplitPlan(num_kv_splits, split_indptr) # Persistent MLA-decode kernel gate: the persistent kernel @@ -1785,9 +1799,9 @@ def mla_decode_fwd_v4_nm( path. Pass an explicit int to override. Note V4 nm is always non-persistent, so only that branch of `get_meta_param` applies. Callers that know their per-seq KV length (and CUDA-graph callers, - whose `kv_page_indices` is typically capacity-sized) should build the - plan with `get_mla_v4_nm_split_plan` and pass both `num_kv_splits` and - `split_indptr`. + whose `kv_page_indices` is typically capacity-sized) should build a + plan with `get_mla_v4_nm_split_plan` and pass it as + `**plan._asdict()`. Multi-pass mode (`num_kv_splits > 1`): 1. If `split_indptr` is None, build a uniform one: diff --git a/op_tests/test_mla_v4_nm.py b/op_tests/test_mla_v4_nm.py index 49455cc742b..c1dcd2bbc4b 100644 --- a/op_tests/test_mla_v4_nm.py +++ b/op_tests/test_mla_v4_nm.py @@ -1162,16 +1162,22 @@ def test_v4_nm_varlen_ragged_kv_tail_split_guard(): ], ) def test_v4_nm_split_planner_picks(num_seqs, kv_len, expected): - assert aiter.mla._v4_nm_pick_num_kv_splits(num_seqs, 2, kv_len, 256) == expected + cost = aiter.mla._V4_NM_SPLIT_COST["gfx950"] + assert ( + aiter.mla._v4_nm_pick_num_kv_splits(cost, num_seqs, 2, kv_len, 256) == expected + ) @needs_gfx950 def test_v4_nm_split_plan_fills_buffer_in_place(): buf = torch.full((10,), -1, dtype=torch.int32, device="cuda") - splits, indptr = aiter.mla.get_mla_v4_nm_split_plan(7, 128, 1152, split_indptr=buf) - assert splits == aiter.mla._v4_nm_pick_num_kv_splits(7, 2, 1152, get_cu_num()) - assert indptr.data_ptr() == buf.data_ptr() - assert buf.tolist() == [i * splits for i in range(8)] + [-1, -1] + plan = aiter.mla.get_mla_v4_nm_split_plan(7, 128, 1152, split_indptr=buf) + cost = aiter.mla._V4_NM_SPLIT_COST["gfx950"] + assert plan.num_kv_splits == aiter.mla._v4_nm_pick_num_kv_splits( + cost, 7, 2, 1152, get_cu_num() + ) + assert plan.split_indptr.data_ptr() == buf.data_ptr() + assert buf.tolist() == [i * plan.num_kv_splits for i in range(8)] + [-1, -1] @needs_gfx950 @@ -1184,11 +1190,8 @@ def test_v4_nm_split_plan_fills_buffer_in_place(): ], ) def test_v4_nm_split_plan_accuracy(kv_lens, kv_len): - splits, split_indptr = aiter.mla.get_mla_v4_nm_split_plan(len(kv_lens), 128, kv_len) - err, out = _run_varlen_point( - kv_lens, - split_kwargs={"num_kv_splits": splits, "split_indptr": split_indptr}, - ) + plan = aiter.mla.get_mla_v4_nm_split_plan(len(kv_lens), 128, kv_len) + err, out = _run_varlen_point(kv_lens, split_kwargs=plan._asdict()) assert torch.isfinite(out).all() assert err < 0.02 @@ -1225,10 +1228,8 @@ def test_v4_nm_split_plan_cudagraph_replay(): def set_lens(lens): kv_indptr.copy_(torch.tensor([0] + np.cumsum(lens).tolist(), dtype=torch.int32)) - splits, split_indptr = aiter.mla.get_mla_v4_nm_split_plan( - n, gqa, kv_cap, split_indptr=split_buf - ) - assert splits > 1 # exercise the stage-2 path under the graph + plan = aiter.mla.get_mla_v4_nm_split_plan(n, gqa, kv_cap, split_indptr=split_buf) + assert plan.num_kv_splits > 1 # exercise the stage-2 path under the graph def run(): aiter.mla.mla_decode_fwd_v4_nm( @@ -1243,8 +1244,7 @@ def run(): 1, sink=sink, sm_scale=sm_scale, - num_kv_splits=splits, - split_indptr=split_indptr, + **plan._asdict(), ) set_lens([kv_cap] * n) From 67d7422df246493ac2cf65758b75d87e9542f395 Mon Sep 17 00:00:00 2001 From: yihonglie Date: Tue, 29 Sep 2026 09:34:09 -0500 Subject: [PATCH 04/13] [MLA v4 nm] Add a persistent decode kernel with in-kernel split and merge A persistent variant of the v4 nm fp8 decode kernel (gqa=128, one query token per row). Each call is one launch of grid (2P, 1, 1): the kernel plans the KV split over P partitions from kv_indptr, runs the attention and merges the split partials itself, so callers need no host split plan and no stage-2 merge, and the launch is CUDA-graph safe with varying KV lengths. - hsa/gfx950/mla_v4: mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co, symbol _ZN5aiter39mla_a8w8_qh64_qseqlen1_gqaratio64_nm_psE, registered as the ps=1 row of mla_v4_asm.csv. - csrc: mla_decode_v4_ps_asm in module_mla_v4_asm, with its own documented kernarg struct (same 21 x 16 B layout, repurposed slots marked), shape / dtype / contiguity / workspace checks and its own kernel cache. - aiter.mla: MlaV4NmPsWorkspace, get_mla_v4_nm_ps_workspace(device, num_partitions=128) and mla_decode_fwd_v4_nm_ps(...). One workspace serves calls that are ordered on the GPU; allocate it outside capture. - Tests: accuracy and LSE against the fp8-dequant reference (uniform, ragged, K=0 rows, N up to 3000, K up to 8320), kv_indptr with offset and extra entries, workspace reuse across batch sizes, partition counts, CUDA-graph replay with changing KV lengths, agreement with mla_decode_fwd_v4_nm, argument checks. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie --- aiter/mla.py | 104 ++++++ aiter/ops/attention.py | 37 +++ csrc/py_itfs_cu/asm_mla_v4.cu | 258 +++++++++++++++ ...mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co | Bin 0 -> 153984 bytes hsa/gfx950/mla_v4/mla_v4_asm.csv | 14 +- op_tests/test_mla_v4_nm.py | 307 ++++++++++++++++++ 6 files changed, 716 insertions(+), 4 deletions(-) create mode 100755 hsa/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co diff --git a/aiter/mla.py b/aiter/mla.py index e1e62c11c57..63277fcec17 100644 --- a/aiter/mla.py +++ b/aiter/mla.py @@ -454,6 +454,55 @@ def get_mla_v4_nm_split_plan( return MlaV4NmSplitPlan(num_kv_splits, split_indptr) +# Workspace of the persistent v4 nm decode kernel (mla_decode_v4_ps_asm). +_V4_NM_PS_CNT_INTS = 2 * 65536 + 16 * 512 + 4 * 1024 +_V4_NM_PS_ARANGE = 65537 +_V4_NM_PS_MAX_PARTITIONS = 1024 + + +class MlaV4NmPsWorkspace(NamedTuple): + """Scratch of `mla_decode_fwd_v4_nm_ps`; build it with + `get_mla_v4_nm_ps_workspace`. P = `desc.size(0)` partitions.""" + + o_acc: torch.Tensor # [2P, 128, 512] fp32 split partials + lse_acc: torch.Tensor # [2P, 128] fp32 split partials + desc: torch.Tensor # [P, 8] int32 in-kernel plan + span: torch.Tensor # [P, 2] int32 in-kernel plan + cnt: torch.Tensor # int32 merge counters, zero at rest + arange: torch.Tensor # int32 arange, read-only + + +def get_mla_v4_nm_ps_workspace(device="cuda", num_partitions=128) -> MlaV4NmPsWorkspace: + """Allocate a workspace for `mla_decode_fwd_v4_nm_ps`. + + One workspace may serve any number of calls that are ordered on the GPU + (same stream, or one CUDA graph replayed at a time); calls that can run + concurrently need one workspace each. Nothing carries over between calls, + so one workspace can be reused across batch sizes and graphs. Allocate it + outside CUDA-graph capture. + """ + if not 1 <= num_partitions <= _V4_NM_PS_MAX_PARTITIONS: + raise ValueError( + f"num_partitions must be in [1, {_V4_NM_PS_MAX_PARTITIONS}], " + f"got {num_partitions}" + ) + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + "get_mla_v4_nm_ps_workspace: allocate the workspace before " + "CUDA-graph capture" + ) + P = num_partitions + i32 = dict(dtype=dtypes.i32, device=device) + return MlaV4NmPsWorkspace( + o_acc=torch.empty(2 * P, 128, 512, dtype=dtypes.fp32, device=device), + lse_acc=torch.empty(2 * P, 128, dtype=dtypes.fp32, device=device), + desc=torch.empty(P, 8, **i32), + span=torch.zeros(P, 2, **i32), + cnt=torch.zeros(_V4_NM_PS_CNT_INTS, **i32), + arange=torch.arange(_V4_NM_PS_ARANGE, **i32), + ) + + # Persistent MLA-decode kernel gate: the persistent kernel # ("mla_a16w16_qh16..._ps") is slower than the non-persistent split-KV kernel # ("mla_dec_stage1...") above a concurrency threshold (~batch 16-64 on gfx950 bf16 @@ -2041,3 +2090,58 @@ def mla_decode_fwd_v4_nm( ) return logits, attn_lse + + +def mla_decode_fwd_v4_nm_ps( + q_packed, # [N, 128, 512] FP8 packed Q+e8m0 + q_rope, # [N, 128, 64] BF16 + kv_packed, # [rows, ..., 512] FP8 packed KV pool, page_size 1 + kv_rope, # [rows, ..., 64] BF16 + kv_indptr, # [>= N+1] int32 + kv_page_indices, # [*] int32 + sink, # [128] FP32 attention sink logit + workspace: MlaV4NmPsWorkspace, + out=None, # [N, 128, 512] BF16 + return_lse=False, + lse=None, # [N, 128] FP32, used when return_lse +): + """v4 nm decode (128 heads, one query token per row) with the persistent + kernel: one launch that plans the KV split over the workspace's + partitions from `kv_indptr`, runs the attention and merges the split + partials, so no split plan or stage-2 merge is needed. + + Same math and packed layouts as `mla_decode_fwd_v4_nm` with gqa=128 and + max_seqlen_q=1. N = `q_packed.size(0)` (N <= 32768); row j attends to pool + rows `kv_page_indices[kv_indptr[j] : kv_indptr[j + 1]]`, so `kv_indptr` + may hold more than N+1 entries and need not start at 0. Rows with an + empty KV range are left unwritten in `out` (and in the LSE). + + `workspace` comes from `get_mla_v4_nm_ps_workspace`; see its sharing + rule. Returns `out`, or `(out, lse)` with the natural-log LSE [N, 128] + FP32 (sink included) when `return_lse`; `out` / `lse` are allocated + when not given. + """ + require_gfx1250_asm("mla_decode_v4_ps_asm") + if out is None: + out = torch.empty( + (q_packed.size(0), 128, 512), dtype=dtypes.bf16, device=q_packed.device + ) + if not return_lse: + lse = None + elif lse is None: + lse = torch.empty( + (q_packed.size(0), 128), dtype=dtypes.fp32, device=q_packed.device + ) + aiter.mla_decode_v4_ps_asm( + q_packed, + q_rope, + kv_packed, + kv_rope, + kv_indptr, + kv_page_indices, + sink, + *workspace, + out, + lse, + ) + return (out, lse) if return_lse else out diff --git a/aiter/ops/attention.py b/aiter/ops/attention.py index aa593484fec..880168df0c9 100644 --- a/aiter/ops/attention.py +++ b/aiter/ops/attention.py @@ -884,6 +884,43 @@ def mla_decode_v4_asm( ) -> None: ... +@compile_ops(MD_NAME_V4, ffi_type="ctypes") +def mla_decode_v4_ps_asm( + # [N, 128, 512] FP8 packed Q + e8m0 scale region + Q: torch.Tensor, + # [N, 128, 64] BF16 + qrope: torch.Tensor, + # [rows, ..., 512] FP8 packed KV pool (row-dense, page_size 1) + KV: torch.Tensor, + # [rows, ..., 64] BF16 + kvrope: torch.Tensor, + # [>= N+1] int32 + kv_indptr: torch.Tensor, + # [*] int32 + kv_page_indices: torch.Tensor, + # [128] FP32 attention sink logit + sink: torch.Tensor, + # workspace (aiter.mla.get_mla_v4_nm_ps_workspace); P = desc.size(0) + # [2P, 128, 512] FP32 + o_acc: torch.Tensor, + # [2P, 128] FP32 + lse_acc: torch.Tensor, + # [P, 8] int32 + desc: torch.Tensor, + # [P, 2] int32 + span: torch.Tensor, + # int32 counters, zero at rest + cnt: torch.Tensor, + # int32 arange + arange: torch.Tensor, + # outputs + # [N, 128, 512] BF16 + output: torch.Tensor, + # [N, 128] FP32 natural-log LSE (sink included); None = not written + lse: torch.Tensor | None = None, +) -> None: ... + + @compile_ops(MD_NAME, ffi_type="ctypes") def mla_prefill_asm_fwd( # [num_seqs, num_heads, head_size] diff --git a/csrc/py_itfs_cu/asm_mla_v4.cu b/csrc/py_itfs_cu/asm_mla_v4.cu index cfd73118569..76ed4eb0703 100644 --- a/csrc/py_itfs_cu/asm_mla_v4.cu +++ b/csrc/py_itfs_cu/asm_mla_v4.cu @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -542,3 +543,260 @@ AITER_CTYPES_DEFINE_ENTRYPOINT_VOID( impl_ptr->launch_kernel({arg_buf, &arg_size, gdx, gdy, gdz, block_dim, 1, 1, stream}); } + +// ---------------------------------------------------------------------------- +// Persistent-scheduling v4 nm decode (CSV row ps=1) +// +// One launch per call: grid (2 * P, 1, 1), block 256. The kernel plans the +// KV split over P partitions in-kernel from kv_indptr, runs the attention +// and merges the split partials itself (the last arriving partition of a row +// combines it), so there is no host split plan and no stage-2 merge. +// Same math and packed Q / KV / rope layouts as mla_decode_v4_asm with +// gqa=128, max_seqlen_q=1 and page_size=1; rows with K=0 are left unwritten. +// +// The kernarg block keeps the 21 x 16 B layout of MlaV4KernelArgsLegacy; +// slots marked "repurposed" carry a different value than the non-persistent +// kernel, and several upper halves of the pointer slots carry an extra field. +// ---------------------------------------------------------------------------- +struct __attribute__((packed)) MlaV4PsKernelArgs +{ + void* ptr_O_acc; // 0x00: workspace o_acc [2P, 128, 512] FP32 split partials + // (rw) + void* ptr_out; // 0x08: out [N, 128, 512] BF16 (upper half of slot 0) + void* ptr_L_acc; // 0x10: workspace lse_acc [2P, 128] FP32 split partials (rw) + void* ptr_lse; // 0x18: final LSE [N, 128] FP32, nullptr = not written + void* ptr_Q; + p2 _p_q; // 0x20: Q packed FP8 + e8m0 scale + void* ptr_KV; + p2 _p_kv; // 0x30: KV packed FP8 + void* ptr_LTP; + p2 _p_ltp; // 0x40: kv_indptr + void* ptr_LTD; + p2 _p_ltd; // 0x50: kv_page_indices + void* ptr_desc; // 0x60: repurposed: workspace desc [P, 8] int32 (rw) + void* ptr_dbg; // 0x68: debug probe, nullptr + float scalar_f; + p3 _p_sc; // 0x70: 1.0f/sqrtf(kV4DimNope+kV4DimRope) + unsigned int s_gqa_ratio; + p3 _p_gr; // 0x80: 128 (heads) + unsigned int s_kv_split; // 0x90: 1 + unsigned int _p_ks; // 0x94: pad + void* ptr_cnt; // 0x98: workspace cnt int32 counters (rw, zero at rest) + unsigned int s_num_p; // 0xa0: repurposed: number of partitions P + unsigned int s_num_q; // 0xa4: number of query rows N + p2 _p_nq; // 0xa8: pad + unsigned int s_plan; // 0xb0: repurposed: planner config F | (MT << 8) + unsigned int _p_pl; // 0xb4: pad + void* ptr_span; // 0xb8: workspace span [P, 2] int32 (rw) + unsigned int s_zero; + p3 _p_z; // 0xc0: 0 + void* ptr_QTP; + p2 _p_qtp; // 0xd0: repurposed: workspace arange int32 (read-only) + void* ptr_STP; + p2 _p_stp; // 0xe0: repurposed: the same arange + unsigned int out_16_nosplit; + p3 _p_o16; // 0xf0: 1 + void* ptr_QROPE; + p2 _p_qrope; // 0x100: Q rope BF16 + void* ptr_KVROPE; + p2 _p_kvrope; // 0x110: KV rope BF16 + void* ptr_sink; + p2 _p_sink; // 0x120: [128] FP32 attention sink logit + void* ptr_zero; + p2 _p_pz; // 0x130: nullptr + unsigned int s_zero2; + p3 _p_z2; // 0x140: 0 +}; +static_assert(sizeof(MlaV4PsKernelArgs) == 21 * 16, "persistent v4 nm kernarg is 0x150 bytes"); +static_assert(offsetof(MlaV4PsKernelArgs, ptr_cnt) == 0x98 && + offsetof(MlaV4PsKernelArgs, s_num_q) == 0xa4 && + offsetof(MlaV4PsKernelArgs, ptr_span) == 0xb8 && + offsetof(MlaV4PsKernelArgs, ptr_sink) == 0x120, + "persistent v4 nm kernarg offsets"); + +// Workspace sizes the kernel indexes into (aiter/mla.py +// get_mla_v4_nm_ps_workspace). +static constexpr int kV4PsHeads = 128; +static constexpr int kV4PsDim = kV4DimNope + kV4DimRope; +static constexpr int kV4PsMaxParts = 1024; +static constexpr int kV4PsMaxRows = 32768; // byte offsets of out / q wrap above this +static constexpr int64_t kV4PsArange = 65537; +static constexpr int64_t kV4PsCntInts = 2 * 65536 + 16 * 512 + 4 * 1024; +static constexpr unsigned int kV4PsPlanCfg = 6u | (1u << 8); + +AITER_CTYPES_DEFINE_ENTRYPOINT_VOID( + mla_decode_v4_ps_asm, + (aiter_tensor_t * Q, // [N, 128, 512] FP8 packed Q + e8m0 + aiter_tensor_t* qrope, // [N, 128, 64] BF16 + aiter_tensor_t* KV, // [rows, ..., 512] FP8 packed KV, row-dense + aiter_tensor_t* kvrope, // [rows, ..., 64] BF16, row-dense + aiter_tensor_t* kv_indptr, // [>= N+1] int32 + aiter_tensor_t* kv_page_indices, // [*] int32 + aiter_tensor_t* sink, // [128] FP32 + aiter_tensor_t* o_acc, // workspace [2P, 128, 512] FP32 + aiter_tensor_t* lse_acc, // workspace [2P, 128] FP32 + aiter_tensor_t* desc, // workspace [P, 8] int32 + aiter_tensor_t* span, // workspace [P, 2] int32 + aiter_tensor_t* cnt, // workspace [kV4PsCntInts] int32, zero at rest + aiter_tensor_t* arange, // workspace arange(kV4PsArange) int32 + aiter_tensor_t* output, // [N, 128, 512] BF16 + aiter_tensor_t* lse, // [N, 128] FP32, nullable + hipStream_t stream), + (Q, + qrope, + KV, + kvrope, + kv_indptr, + kv_page_indices, + sink, + o_acc, + lse_acc, + desc, + span, + cnt, + arange, + output, + lse, + stream)) +{ + auto check_buf = [&](aiter_tensor_t* t, AiterDtype dt, const char* name) { + AITER_CHECK(t != nullptr && t->data_ptr() != nullptr, + "mla_decode_v4_ps_asm", + ": `", + name, + "` is NULL"); + AITER_CHECK(t->dtype() == dt, + "mla_decode_v4_ps_asm", + ": `", + name, + "` has dtype ", + AiterDtype_to_str(t->dtype()), + ", expected ", + AiterDtype_to_str(dt)); + AITER_CHECK( + t->is_contiguous(), "mla_decode_v4_ps_asm", ": `", name, "` must be contiguous"); + AITER_CHECK(t->device_id == Q->device_id, + "mla_decode_v4_ps_asm", + ": `", + name, + "` is on another device"); + }; + AITER_CHECK(Q != nullptr, __func__, ": `Q` is NULL"); + check_buf(Q, AITER_DTYPE_fp8, "Q"); + check_buf(qrope, AITER_DTYPE_bf16, "qrope"); + check_buf(KV, AITER_DTYPE_fp8, "KV"); + check_buf(kvrope, AITER_DTYPE_bf16, "kvrope"); + check_buf(kv_indptr, AITER_DTYPE_i32, "kv_indptr"); + check_buf(kv_page_indices, AITER_DTYPE_i32, "kv_page_indices"); + check_buf(sink, AITER_DTYPE_fp32, "sink"); + check_buf(o_acc, AITER_DTYPE_fp32, "o_acc"); + check_buf(lse_acc, AITER_DTYPE_fp32, "lse_acc"); + check_buf(desc, AITER_DTYPE_i32, "desc"); + check_buf(span, AITER_DTYPE_i32, "span"); + check_buf(cnt, AITER_DTYPE_i32, "cnt"); + check_buf(arange, AITER_DTYPE_i32, "arange"); + check_buf(output, AITER_DTYPE_bf16, "output"); + if(lse != nullptr && lse->data_ptr() != nullptr) + check_buf(lse, AITER_DTYPE_fp32, "lse"); + + AITER_CHECK(Q->dim() == 3 && Q->size(1) == kV4PsHeads && Q->size(2) == kV4PsDim, + __func__, + ": Q must be [N, 128, 512]"); + const int64_t num_q = Q->size(0); + AITER_CHECK(num_q <= kV4PsMaxRows, __func__, ": N=", num_q, " exceeds ", kV4PsMaxRows); + AITER_CHECK(qrope->numel() == static_cast(num_q * kV4PsHeads * kV4DimRope), + __func__, + ": qrope must be [N, 128, 64]"); + AITER_CHECK(output->dim() == 3 && output->size(0) == num_q && output->size(1) == kV4PsHeads && + output->size(2) == kV4PsDim, + __func__, + ": output must be [N, 128, 512]"); + if(lse != nullptr && lse->data_ptr() != nullptr) + AITER_CHECK(lse->numel() == static_cast(num_q * kV4PsHeads), + __func__, + ": lse must be [N, 128]"); + AITER_CHECK(KV->size(-1) == kV4PsDim && kvrope->size(-1) == kV4DimRope && + KV->numel() / kV4PsDim == kvrope->numel() / kV4DimRope, + __func__, + ": KV / kvrope must be row-dense [rows, 512] / [rows, 64] with " + "equal rows"); + AITER_CHECK(static_cast(kv_indptr->numel()) >= num_q + 1, + __func__, + ": kv_indptr needs at least N+1 entries"); + AITER_CHECK(sink->numel() == kV4PsHeads, __func__, ": sink must have 128 entries"); + + AITER_CHECK(desc->dim() == 2 && desc->size(1) == 8, __func__, ": desc must be [P, 8]"); + const int64_t num_p = desc->size(0); + AITER_CHECK(num_p >= 1 && num_p <= kV4PsMaxParts, + __func__, + ": num_partitions=", + num_p, + " must be in [1, ", + kV4PsMaxParts, + "]"); + AITER_CHECK(o_acc->numel() == static_cast(2 * num_p * kV4PsHeads * kV4PsDim), + __func__, + ": o_acc must be [2P, 128, 512]"); + AITER_CHECK(lse_acc->numel() == static_cast(2 * num_p * kV4PsHeads), + __func__, + ": lse_acc must be [2P, 128]"); + AITER_CHECK(span->numel() == static_cast(2 * num_p), __func__, ": span must be [P, 2]"); + AITER_CHECK(cnt->numel() >= static_cast(kV4PsCntInts), + __func__, + ": cnt needs ", + kV4PsCntInts, + " entries"); + AITER_CHECK(arange->numel() >= static_cast(kV4PsArange), + __func__, + ": arange needs ", + kV4PsArange, + " entries"); + if(num_q == 0) + return; + + const HipDeviceGuard device_guard(Q->device_id); + + MlaV4PsKernelArgs args = {}; + args.ptr_O_acc = o_acc->data_ptr(); + args.ptr_out = output->data_ptr(); + args.ptr_L_acc = lse_acc->data_ptr(); + args.ptr_lse = (lse != nullptr) ? lse->data_ptr() : nullptr; + args.ptr_Q = Q->data_ptr(); + args.ptr_KV = KV->data_ptr(); + args.ptr_LTP = kv_indptr->data_ptr(); + args.ptr_LTD = kv_page_indices->data_ptr(); + args.ptr_desc = desc->data_ptr(); + args.ptr_dbg = nullptr; + args.scalar_f = 1.0f / std::sqrt(static_cast(kV4PsDim)); + args.s_gqa_ratio = kV4PsHeads; + args.s_kv_split = 1; + args.ptr_cnt = cnt->data_ptr(); + args.s_num_p = static_cast(num_p); + args.s_num_q = static_cast(num_q); + args.s_plan = kV4PsPlanCfg; + args.ptr_span = span->data_ptr(); + args.ptr_QTP = arange->data_ptr(); + args.ptr_STP = arange->data_ptr(); + args.out_16_nosplit = 1; + args.ptr_QROPE = qrope->data_ptr(); + args.ptr_KVROPE = kvrope->data_ptr(); + args.ptr_sink = sink->data_ptr(); + + CFG* config_map = &cfg_mla_v4_asm; + // Separate cache from mla_decode_v4_asm's; keys are the distinct symbol + // names. + static SynchronizedCache impl_ptr_map; + const std::string arch_id = get_gpu_arch(); + const std::string kernelName = get_heuristic_kernel_mla_v4( + "fp8", "fp8", /*gqa=*/64, /*ps=*/1, 0, 0, /*qseqlen=*/1, /*lse=*/1, arch_id, config_map); + auto it = config_map->find(kernelName); + AITER_CHECK(it != config_map->end(), __func__, " not find kernel ", kernelName); + const char* name = it->second.knl_name.c_str(); + const char* co_name = it->second.co_name.c_str(); + AiterAsmKernel* impl_ptr = + &impl_ptr_map.get_or_create(name, [&]() { return AiterAsmKernel(name, co_name); }); + + size_t arg_size = sizeof(args); + impl_ptr->launch_kernel( + {&args, &arg_size, static_cast(2 * num_p), 1, 1, 256, 1, 1, stream}); +} diff --git a/hsa/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co b/hsa/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co new file mode 100755 index 0000000000000000000000000000000000000000..dd75375eb3e6879b78465021a9de52b7b8084630 GIT binary patch literal 153984 zcmeEv3t$xGx%Rg+v$MO&ZrFremXJUQ0m4NL5HUrLZbAZqM2r+HwNx&GE-`9?=s{0d zW;VN^a>Z5vzb&)?b}OKp#}_Oz!L_J7`QX2K?J zAOY$*r@yqXPrmuC^WC2J{oa|K9TnA!1(V4wGVPzpPBN4JH-X#u`#ORDb&Z*>l`t#* zo5Ji&##IU8=CL2g9=$LFt$I4TO!rhI4uW0_i*O&yFq$yz%NjL%1H z?U|&ofZNTI8epM*ENI;{B=J7Gp!O zAWbXLK%?(1xSWExJYeb72ZK}It&(23@uitX>9dEM* z!DnW#zj5oG>+9C8zwyRRcii#hmQCw7uKmX?o40O4%*KsdaBb}!_3Llk^w`aHw{KWq z_vq}~*5A76;cLD+`}Uh|x?|H;_UP<8Hs7h3;kCsr9P%u7(&Ia^qmAk)Qe;dMp}ixO0J&K8y=$aIlgc(81x(L{HG+RoX; z$^@A%b`uv>8%?Z9P}@11xH>_mi`~SkfyJ^eL2c)3q9;M7VVkJ$2anuwW8KY0$JA>D zNXFWyUF2K^#BV5DFEIbM3(Vhm9{C@+bwk}PMh6^I&ZC6*9ysQ|&;nX6v;eRQQNgGN z@zD+Ix8CR(=!MXE6fvk5Ixe(;y%$=*zVj&H;I(%Qbisl1$UmqH9=Xs09>356o;r^L z4wjwig6{LkKd1{1UuXf(UuXd@o<{)(7oF*Xp7Y2*s0&`c&;nk$&;pK)rhuVJ-pDkl zgVzqQ|FPp*b)!>6kM$?WH7xO*XCps*rjaKTRCi8xJ@#ILTqADe2sQeHD{2O|pbrw% zckWjHF+r{ow{nEzT|S_vyk=qv>N{sEUo%+~WV`6i9MJ4vGdUB~H-cum5@Z{;nYv2# z=Bz^f zQx{$+Okmd$sq-@0j_9cLQK z4$t1QsR1V`gX+5vyOzoHwb^&vT)$=Qjkn)<+tzhooqfmMw{Ez-?ydRf(Sq5xY=p%$ zP}pk%`}*vETz}W5o3`AJZlk+*8%IBPZQ63j=G$)zn~ZZL?fB?p|D|)QG!dsrQwK*~ zhF~gA&fNMro?DNm^oC^fYY64BBDxmZ7x?hjcdz z_pC$O9wzj318s^B?N!>`)cze<%4`Z($S7`L53H>;mW>&j~ zEYBZN1j$?N%1anOLAtMh2T8 z<;arYu?a#B<)!-)15&#bOADTyhy9JnRPVrjg`_!?SV^y(54$0`BiRc?SAh$bYS(WWhuOi>CjxFkljNNYu zS(iAyC^Q;#I+ISvc3fi}qlK;J+1`w-jBr*)MrT$=mYSZG(U~DUy4Z>L&sL3L?$d;gLmE|T`P4}l4-5<$7y@D9tezy8*^MaAh=%=yi zW7V;}E>#_SpX%B_F60_J)~{sjRb5%GaF)x}ndL%V`?Eu1v$8tJWxHCkGRB3+j=K*n z7^jZS=z`dy*<;5Bv}+U|8yM>v+cMULzEQLMS!hpMXq;-pxC-+yZfR*AYMX6L zN^Lsorv8^qslxG;6pzBXB1+bmRK*mIBr{*sn|``ETgCq{PO`XjyODMr($=Phn?=#? z^`@O}osWCTR^+IF{@2G%tj8FMjd9CPgB&{{XHAggvK36chJdV7TwFQ4Nr zEW|LE=7|32XGQdZe7aeh9FD9LXkG^}ul*6p=WkDvB!%vg%1JhxeSg`%5_4PP^A&SE z*%p;lDd^fw>j|3!*3;hYqL}0rTP9*Zurk#uP4HO-56yQz|FPceG!~fqDf+d)?B{bI zx4iOgJKG^p-=?q~W-HpTm9K^QURo2$<`VDtoQKp2nG@8B*e_&rvA0+02u#%1#6+xH zhlKXn?OsJ{Rwon%BNO(UL-G>4wkGU$z9!@`T39yE@>()2;Y^FAGt-i(%9)lfAy@+zOCVF00~Qr&5YDtn0m&@2NM=87PijJqeyJ!BF;l(f2UMxoqFN>( zE$Ypb#@?pt^tx`#Q>h2IY5SzdjY*jdB#z-Uwe0rp|~N?Xw_GMmkrf zKZrD~K^BZs8)+`j!d$*Y2yVw(@0*RkCHB(sB1vGb+F&#I@Px@Uj%7;z=9z-sLt(*` z$}SaLp8fWee45uX=J$SAcD@sT+xb4flIFCP=D3sXkW`nH9Y6mO2em5DXQ%lfrPN8i z&$f77r-E|t-WKA49H&%AV`;Iac`u`vmUsLmlm}^Z4hQGv>b!UAP0F#XYLM+zl6#_-lFd2xkcSUZO`syv(aYS=Xy!s;kp3y z0QRMb!uCaGsjha{OzMZH0^{~ufqMh4S$l&nv@c(A`DAY}=(5Ry8LFk-!uxt}U^-9t zWMGC%OE*Qa_`K;IK^yo{2H#jQH*8$@Z$X_8pq^I9{SgK|kT{_dZIMkK5gGR|xgaB8 zL4Qq~3SPCz(Ja{(u=HlBl=e!}t%W3S4s?!8!S2n5&Qb{7Wju5kCyTsg#wM46II}20 ztXN;T1~N-W+AO5CA)Q%tdW-b56QKW0LVDr>Y`~E`J-yr6)lX$<>zB1V$aJw|{jvab z2S3)Y=`8CPb=b^do7voHGs9}JnLEwQ7M5&&@Nhcx%~pslnr=1+Y?2f(t4M>e&CCKY zc3YT9QDN^jp+-N0jZ+Ex#f*U_Jq)^LGub_c%|UyQ+xeHb5H0kiCwb_)V4mR>&4pfO zBYjhe+>JgH*zZoTrc^1$q;5$HZ|^k)G49xJM48({YmlZzlJk0H!J`UN9@!E# zXZrP8o|!jDv7{sqgYs*`oFM(2`h(}6cbk@fPOU`x`WvyI!%2tv(wnJC*kU>6B{#YSfNR_gw+>heQRplg4EXh9g zbS!2vJG!=uC|6E3N$AV$T5qt8Y)Tn6qmz%%Ur&f;lc-=^7^~~Y)HBCq*+YjPqHX8) zm;^f)i?>z|YWI{ZWTWj`eKapnmY1Dw{*xeNn@pydx?1w?P)$J|7ECb(bDKf0<6+4Z zGZ}wdjlU(YsmS;n_v1FjC_UzCpQ+S@XQ+U%t<93;XG&TW2t!AhDq$Z}gpTdKiV~=0 z!Jv%!mW2B;v7LQhG~UE^_3Z#3QW|gV=^)KH3G=GjY44}AB5HH|e6DuZ_2ZUJ=?(@P z%+$^2xd5A7XyJAa;;`R6VG0OB0b{3W-pfu61!GOf<}}zCnhi4<`;Hhewd2mX&5jk; zjz+17Z8p|(2G(=NS=Y1OYnIJn8S7a#!^)OvJ)Gm8tR36$2H%L(ju_M06cU?lsNY6o{u~QbIBb#NfBsQr@W~*0} zL{%)>6>-REuy2~xG&VMD*)ORg^b^O}U8=NSYPFaxEosm^9PB=bEt-**78vVr1kzw# zQ+RBeBY?K34vW|#9q_B7r3v{gAqo8|$#N(9rA20zuw?P8CQ-yWfdl>1%=-xE6afqR zOy>P)i&|uikQsZRSy6GnO-xi&789cT!F0XKseodolQ2`CAebbDb@2U-=9_)LBV-Tw z+MR-+q>7?XfeL2Bp3juVx0x(3Cn4!&e7-TSH|12TfN`KcMz>yu3A6$K02F|K1d71_0%8+#{RxQq<+6a!h*yWU_50W#-XWcl?Zn!` zdUW@{@d48R8Teb^Uw|J1d7PPf|7;e#e=&oblZ2iFxbq$$fJm-~{%`|PA9IfrdfY%- z12dXu24_O=fQ^iEwv|DlY1Qr-O*40|oVhZ*N+sN+tWpKo13sM1ZQ3p1Z)(Ty2I^ht zrS%e{9Uw_Ni(_Nu9yjV!(q!Has>jGfZU0Db`|p8&Mfg8~r-6SMC{w|`_;4?N?_bS9 z?`d<3x2kuQGGW!Krd7%+|0>_A_Q%FbJ$Rgu-@}AbA$sttKy>*vbf)5uPEaQ(KSIB~ zkN#bbv<ouK|{;6~Xe@x5Bj3OmNH5GG%I@LRk_})XKnwQaQRPHZ8#LQ0HQ0X<*6H zCC!`ErR_^AE0rbapWe!efeEY$dWb)=oV^*jhJ7D@_d~uF`J>3ZSD=4z$N%5Rz?kFy zW9&QNeAyCXuSS?jGAr!$2ovNU3=arN){ddg|$V zy3nHl+`te}(dRC+$vwvr#unFe63~PA4=9{0^Eu7zs0VYJIZzLw74-mnT-sc3V?tgV zbQN&p)ELMWd!L5zE>Jkg z%aK*=-y#$@JvLp7D=bu$>C=5vVpFtmL4l%7nc@@KPa>kaMiJRt!0Qz%bIlijs%(&w zqhFpJh;Cqgk%_%>RQ0V0T>tqWD__8TdjQwJP*WL>RaS*be~499hnd9K3Y?o%SFH{3 zFgP(*c{TQR#{5XPmbFEySvYbvY^W;M9ytyhpJe%@c|vb zq2rf0+F~|-FR8Iqj+ewP;pm7tIA+8$IL?X9;W#cfjwA9l(^JF9#}WBBA|FTOf6e+sc$MP11sB?to(Kpt>>%IztWYm8!*oMgAMYH>|iJ zOt=K|x(f5AoY3V`lyT#HepXy?pm_e&Cg6DRYPNbSjvY;X%yz5PVyO!IH&%zMtE!Z$ z`PH6T?3EKtbO^rHs|yYwu4?sa503{jXU)l}(`1Ek)`zlkLv^+12IG36e*Jf6uGfU* zXsD+mP;)?z9;m1Yd;#NCfpCSta`$44MWw$ouv)Ql{Co}Sieb%QZB#j(ijtb@i?Q!U z7#2yDP!*2h-YL#fT^+8fu2S|xFZDw|%%=7xn}vIeiwm+v!BZs&9t+CJqqSF6qA(C| ztmHF_l85kT7HGKO3XUc0FJ#-`zP@-8r(;TEtWj}dt#UowiN!h0_0J$*jPjw(;nECu z9qM4@iYK4HO2+)V2J=wHJQ`my-y0LI_rgUGQ-m_FBCPQ)FzRrRzK$!5yz54vH`|=# z9S>d77!L(SRLz>DD3@HKjYpm~9=zW%PnwHe1(;uXcKAB7ljle(E1t^#w#|_+|2#2e zp~4C=1`Gc|>7g?GlF}0Y5*n$Oi8jsw|@;tG; zJP)fkF&bN5PF15~glV6xav9^8Ja`;`zJ|9%n|K z#l?1-XW5s}#~MaEY&Ic@-`lN{pOxetC|O`nx}v-`R$lJu0Z*wV3kuYdlDwQNN(!4w zN~Y#iEc8sMm|NR4cW!}?T?ff8h+!Quq^-)&_w2|0HWkm;=B>>vjHj|wZsBdNFm(p?yASIX?U_n{^D@w`bfjUHOHApP70$Q^I;_@i8^%H4<6z=#HS=-cf7{GB z4|qFr4F6)_GYwyj&|IUwbYq_Lb&?0fI$@pUu{R#7W+6q$){d*$&@+~Hz_PrC?_mxOIex)9N-I;hP zK7O?xfBl(wIX-@k9)H7`c&i>yYndq@{X}3pn6Zv2EDZ_cjCxQ!e6DB0w7*T4wf*g* zGyB*!^fUQ>W9jK0b{umVV^3(Qv_)HBds-A0%>%Enywg91O~;%XN{WWokcM_PiN`yg z8m226xtF9P7QOAeo#B8ergA`=UnY5r*`=CcC6Ri zp`oN`Xbov-XB&9D)2U&)qM<9KVGdhAxE-VEk9=xJlhzJJ)!Gr#5RV`5u+^!dUD41P z(lDJh@pzY0!yM!v)E}d12l>J7(&rD&mmGb*kRF1wjUQwA<>73DupmPpLyGIMYH`Ok zEJ)G&mEx3CEiR;C0okE+uN3Fb*5Yb3ETH;nKGFQ8`ZG|!pSkmB{^a4nvDS1t;)0!} zz|I<{W44atb$>OY`%97Q{drZwZ+83v`^z6_e|h)k<7p51@acZb!nid~oYR7RRO645f*fPNCf+D6 z;d?q=i_HgbfPaF#eb5m|?-{!9BhLZs`|*3eL*JK`nwo+Atx(_J3WwR>3ibW1aG3qA zP~YDQhuPl>_5H0d!Tv@%$A#{1#9uM+7XueLt`KTrOL=<0DIeo75gaCJ9A@dM3R`^D z?KEd-j?wwd0aiBEUshJ&SoG6d96wvcs>;$mzrnK&)q+o|>RH}m@g+s8l~}UV&px@C_9nlwJg_{r9A|QNd(&K3-g1|#Q2E8b^YQ&1#`hoM?r}H9i?}!h8_&X$<-*@qbluribS8c+6$)@dFnrwWk96S;ld`yxxQ~;Ov52jP-a6%7(r{_2pc#!1J+s7R>M*m@y0UNz+Lv4du&Dz7zRo^Enoq zUBu6Fxt=^j=aJu^;Sr_2$E(!840#1|400UuCZr$oHsmDaUC4Wo_aPrZK7#xa@)vKV znjD>r^{dpg9ZJ0fu|n(+CnO!>g5*H*AO(;@$Sg<^q!>~HS-7Lpmkhq5a@+`)Ln$pM38+F{M<0c(%((z^;X{|K5;X@E^@W!AITndi96nh!z z2mb+gGwBDwS!h3DCE8C|P4OQvRz&f@Vu}aWP&{y@g6r^6m5@FGTzCN2fvdYUT+@T= zzh%sQT*LB{xQ_6;4>a6}^64!7U6!}C|2K@ikMb$}US1&&{~E*y9ma^$uC0ch<`$vj zpl`h!fLPo6P5uE+$z#7h@$Pr$N$&!G{+)SJp7`nx#-4iP-CmIuS)YER&?ICJwHuWc zE3$uqHTVnY6A$z+DyYS?eedHSb`tt&67>`E|NJE_e^Khu9MT=9>wE9?1bgqL-}~M) z%JYkV)AE!|I+{cF)6o5rm!t2|(-Z8`RK~x5TPwpo=V;DQ{gLO__weZn_V9eRxvyrC zz$RNYEI{~Lgr^`(Sj3LJ;VDJA@INL)(c|Umd;atUdp?!7LNAYy%KNNd9wC)ixAbTZ z`9!Ha8jJVdm?yml{N)K6i@qIH#!9^mLMmgGUIroE^LrSpVvJQK#)^ESR2JQPysmr- z<8=z-Mde+Gd~}b5RNm)+bdQ8o-Y+qBR9;PLUk>?Tjq>7kZR70V(jWeC+5|>RxjndUe8BJ`M#j%BP>QA z)|d9>IOEc<(bE%B`Y-C~2`T;7rF}Vhap`Z+(-TtqwR(EOB0Zh*-Y+@NeFUDnGH!0( zsq2QM>tf=+%hqe_ZvuPcMA3~$CBnHk-@~!^77gd$evg#5Szo)GwRJi{EcQ+i=X*`y zzwzHF_GXILQd4T;s;NTz0*{ zCVb(_%!yx^FPZ1hPm0a=HKMaiFS)8r($=fVq?DDhORl<}nXh*znaj$OP@lHXX*g?A zX&HR+-#t;3toY{u=K#@uZST{tdM>Wd#r1i>dB9>|G4L|rWk4#E?~NK3&&T!oxPCeC za$r&EI`5p)4PGjj@0l8|oLjolJFoO6uY!Dh@6>Q%ajD09S!u2J0MhY2RKqp%OY6Ls zm)_=`^u;wvJxI&)t_Djuma~|DXv^Opo7lP z_rVzn_CeZDX6XCdj0F1|kJtCF8430;9sgzS!%J=mI=YDklG<5GB zx|a^!Unm^EKPabu#n;R4v0kodnK~% zat&9*kF_f^5LMwTtn8uZ;RDf1_`3FA0fet>>NR91v1pqGNau260#SO7jn3fCW-qx( zXEq6G49`c+WGtVha-7W4I2N$69H%h$tsmnRx@~m!ZrD-8ePlCH8AYi=t~-UNxrfr? zXR)BOz2A?YbSff_KifHdye&m#l}ysgLK$ASRLpbA(()tKUa?kk?Q3 zk7aBwum3E505`Xhgj}v$;4GfXLH?kk)!jnzbxXDQSvaHb(dxlt0{s4p6WKu2H)P|K zqiw6vzf=~=+(~(X?X-W#`w4g(^*K8&fu3aW6qQT14wYMznwHz6wK2rT@V4c;1=ioQl-8)T%cv@I|v>JCGP> z%IlDB+j^fPksZGgagFQ4j%?UOWEYRu?cyASNk7Te?P3qoDeDhH4TsGJ#}TC`dzaD^ zQhKs?DLr8w()-sx>)>{;o}Tm#N>51X$+o5Rg!M?@vi_)p+qQaovRx@XA*CnVmC_S# zMf#oV`yAYMMfybcDYcR8P%49v${;(G${^f^GD7Qr>fm;$UMJa~l%9~%ll@8Q2^*2V zWBrE?ZinjW$^NADgp{7_PfAayApN$@LJGG-_4H(aQhGv4PxdFJC-ftI_Oe$)No}7;&*Odk#aU0buZF{Hg8DDds08s({PPpt1^^D_VGTX>)5;{WvJ~MZ>t_a znr#if6y(utMh%PELCv;)1o4dx;gpfsw2vdL(r}R4rP-Alt~6|2vTdo&Xxn>FA+5jR z*%UWyQa5bUQeY{t3|IzS1Y86x2bKdXfEB>Sz{S8Nz$L(Ll+)61G(|zXxy`EKLU*a+ zEiLtV%S!#IQ?vUIBkj(HzLZ6!L2r3!i?^b*&AYgCr*}zdJGWtZrG-nC#)1$)-13xVEMfQTLfIZTVOrN zx9x6WOb^<~ZDkG14O{kQ#5dlZHs%$ea(Diilc=BD%o=XwwyvA8V~F?PJ$KA;V9VWQ zW9Y2Y$(V=EI)QcE7A|A#O~miKd&QW3VCe1*W9SSN@%3~D3f%fOu6NwMWsHe^?w>!m zT8chq_Mh3aM!>;zW;rSjhJR=%2U8hzZt2#z=^m!>jlscg+OyB*ON!@UDvQoDsVwaO z100;t9|X>bKfhG8`iF9G8L$5=zN#YhAwQ=ifB1}#b1=ozxg&A(EciEwN1=}x99(`j z4(8{1Bjn(-`OAiKuyNiv3!F9HXvqvT+9lpNeT zN)B!v%)w_n)e!he`yGFrKL5FgMX%Tu!b53w~l~=TfxD<(m7be zGaQVv1~|BN1RT5v9Q-q#gEcH>Ka1nwJ)_{@J>cM9=^U)#U=BuG2RL}ovnfN*o6mK= zJoG%doBDGPI2i5b9IW9&+-j~ZcINlB(^1Mn1*7S(&;33bp z4A{Z?GaRF`gJDB`5cgb(@r=iL*}<^0P}aaRDTC}_{dtm$ZU>|NoP#wSYzOPlt{kSm z9f=)`_Hz!_@Z#ISXd~xf4F_{@yd8}CIR|SvtQ|b`nW*9H;Gxe-4Yh;c#WOpj;owo* z!SGR^j~#qAfA&c1;Do;BQP{y}^OuvIJ75RH&pnibDSp5X#(OVAIQXL3!DsWI4CUbe zZaa9!S)L!bn0D|>#lvv$r*?2+4*q+wgX5kHG3?;@=R-IL$J@a}{Lq|(M`{Oa&p{15 z4>iESD09?yFrFde9IRon@thau;L+N_c;1L}u!e)}VEwr;&cPSc4o15<2WvRk4%VMR z;~YE^I~eWe9IWBRw}a6}&cPZE=HPfc81-`w)^J!mc<8fS#&cP3lFkul2M>KdigR$B z9W3Jg415q8U7SHOD{3!*`H0gHnaL+W64XQm$`#;AwTr)|?a1S|`HvGKjvV#-( zkhw4OZ|EBy`kWHA(eMk?v#-ifzp(DV z9;rXL9_h({d?Ehettf+h$|Lm$Z$of6{#*Km zhx&gx7sD?MKdttBvhFXAe|C8<^#lCE2KNs2`xOF-HrB>UpU?m9Pjt# z{0keH+RSwg4U6Ob!G_dnR7v|gxKk!ihu;CZZJAHKLMX`&7`C0!*DOW3_q~Oz14}h_iXQSp34q4xVIQ{nf$+J<6hI8VYqh~ zJ9tFg3xDybxEFroQE@N)#-rk1_>C`^d*MqS75Bnk9u@b(Z#*jQh2MA(_rfns-1fKf z2PfoS*un76YTT>)i9g1@@CTojd*KHj>MtI|y}Hksb1(eCBj#S%#N<2X?@MYpn0s|U zGUs0ShDYn~HSA;ZDf4$QHLM=$2c~-);9mHH*u6vj!Up%2{$=FcJ8HZ46YvFJh~1mWXG|RTxAX-M$Gv28 zo`rko^LAYf|Lza_-e|F!$=e5pcnFFMOYzdo}#H z-K+bPM{M_^t(l%6kNCXoUid9J_i8wp zd-dN3xnR2&K2OfQ8h+gF)&0nvdq-mTqOF{JH5_dB>i*#o+r4NL=UxrZ-|j_QIQMEe zn0w>xUev?6SHofL-r@YfL+#$7eqe)poBnu)dz;|@J=eMOsBPc>ruTsp`Fv~U2pROe z6y6IOc>m~P`g|FDwONdP?0sPJ;o`dSE>iJu{A<|0RE`Vp0#iON$Ps zR3_3H-230^^Yu?3hI>D?eFt;z-<$0_%6EW=wS7PJ`D(T={G7zSe~&(21^#Ewy&4YY z-ZCi)$$k=nf%)#sb1aj%kA8;@1uCJ*VBPpd8{}$`{AI=k4=V@-^;dlR2K1dv%|0F6D{i-h|IOlif>Y#&Pe$ zM0W2d<@0@X+A!Svsogu6d;i|--cNnL{5Ru2_4y9}8(7A(&h%_D=Uxp5b8o!e3tuni zUJZw}doR@I`|}S+&b_0ydp`l6?}gaC7t`n4Q>1b4AiEdu{krE2;@)%Gy;KfyFXfBl zUT*iEm(O>u#=Y)&=jL9+?(IRIIPM+R?xiy0xHqBQ`$_qHzcpnT?)}v69n8IdZ+7pe zKHoFkdm+y{!@v3W?(-e`orghZ&hd6H`hs(>hQr#u7wYqse|v^|$>+Pew@R&QstVG# zPU+icRkLS%s)~v{^gGhi_1}?p;yq#G_jSp}q~C2Gur>AHhBo3qEbnTJeYK z)7pstSbqI?k_|uS(BGOi^3gjg^n1baWj}_p3r176@!Q6RKhtW5m4!7{BIv{v#GM*4$!z zx6_1qBjov_^nLD$Sa*}y*b{7`iTUt-JBz=ny+%EzR7a0B;hV!jAz0l`-yW`_Z(Em@ z@$Vr|n&43C@w>;dg$t8@CmkuhP5?~x{o*L!E<8$Pp#`G$}24i0@AnBD;y56OY#LKHm1NY5T+e-YoF zMm?A8XKePO`QF+1p4RN2oA6a7B|5ue4gVI8G@FG;TBVH56ucIld= zO8l<9k1ag&-P(lvQ?M zbRT*nyH{{-ApAGMxsmWYg7YTA*951B@Py#3CH%hNtRs9&aNb7v6Tw+e_D%+xDyw2sY`%%DRqggF zyW&tBK83v%iPfwP)3=_lUAr#)`E~eSxYepCHkTEkfUs^cPD7 z^u192ZD-^cE|Wac+Nvy}UZ; zuUQ|ycKy}i&tF{?mewy1S1qqnV(j$@eNS__poOdKc13YGd|yDCD%Q^H_u>22)%b>H zHp9>tHZ|MEL49Ib4fAQRTJ9lE6&rt4{f#yPWr~AtyD7FU(WFR_>=PZH_jE@ zOBKG0?aRw6hym$a=akN0oNlLb=ze+njC2q61KPp+z=`%|;~L#JF9Y8-_NAv6#OV9R zD8nR5-gOPd-VF^U*sCIMIq`e(8xg(h9m2OE++)|m^$2gF{Ha=a zE5dhC{tPX=4dHtz|2Qq&h_ILP=W5{{FVpuj_`8Yg+!}6FH1zanSjW?(@-+2s4Y%Uk zxb!~bwjK=|dB27lPYUY3kv_fdmcKHu5Z+@Zyj>oqhH@P^mj?#-F&Ap9!zf!>G55PqHdK<~rj2=`Hby$|0+_y?3< z@56qCe?m;xp{;8lNG5LO#PDAJ1n7zi2#?XRpM4=$v35;1iTJluxK0UmTx=kdL^- zhwEx_@zh7?yV(YpQ2vBmvJhMXJs97Nm~fTACIa*DO%i&}%Q=C#Wg)ob}g}CZ5K|uX)@W*5xJfaP2w`Z{uU=)VNF`1V;^U8HK@R{5m+w zxK7+O5xSLvHu@5@(HH|i;)4m|jWK9Jd|QHeV+?j8zCA&_F$N*ThZDpbW6**4&IIws z80BK`*n;*GxVNBoZx#2bD8Hsaq&5dS}+ z?^obl>_*sl63!Rsti<4=YFw|uck@C$6@i)qa`Zq&Mc@laSAj5nZ!ad4dEejm^hrAN zP=xEeN_`LHWymX#W02#JHzECyw;?AX??T>#ybt*R@)6{ZkiS3}zB?~LtPnfI2}y^z zAUTjcNCBh}G7C}!DTb6l7DC*Ra!4hl8d3vU30VzU16c>z2=PGbAoY-~kZq7ghyw9L zS|B?iAxH;gFJvF&0OS$K5d8(dIS&eT z{X1|X@IBxr;4gt-Sl21wRN$|G_z0crk0SF*I1e(p```G5iMa&ezXPG&L`~+^Ub_As zT{ko7J!^5Wr!f8DGtATJj55GSUp2fqzMwsgPBTxkX7f=HJiPG6pdY}j} zlUec#=5nu)8C3I&d=bGec$qBpbW@}x_v|IK%RPC}Yb5h*Z)TRNnwALsJ9Mr9-rI?d z3!X#=j0>^mm>?vbj_tU{GzMkNGKY~xW#$r95dx~ooZ>ZG%wf?ido5;jr`S7LEh%b> zSj-kxw3s{1LUOodKMUpWH-*IHCRN@qtFZ4|%xbbK3OgaTXtLQHu!~|qyGCI#ATqH< zWQsbOB{#`27Cae)c36UX4=5_zFNRo=5)nv@Wfi$lYl$7y-Q>?AejzY)#a@)PasUU?Nf(b_|!RRa%eTaKzKBe8s zBqh~k_T?r|PzBZ!O~E>)zbE+O(=enl`7BM6Dm5oXTNEh(HCQnzCLfDQ3d6yINiqAV zZk&7ax|1c0L=#h~-Z8R?jbRg;0z0ri1zXS-yVlw>`R*7ld?u|QQQxDm`j9wPK^=3lo1;P2G0EQ{7DUv)P0g z`&xvog`?IdBhzEkxn1)u#C=aR3z#RZy(VAe7nWw6)nP2mCbn9T(6@&pUi7UiCIn+> zm8i!F0ij(HT2*nc#gmmUsAd-E%a;WvBw0PUfzLYgi+F z$C^xx-zQG2g}r0yOrOL3eU5oC5`C{`*d&FuL^6>tKbX{>iM+~QN3EEW;!$O10W&3c zD_9d_Fju|O>E>F8FJc$$sGMZP@7hlBl8}C;ENi;wczUL1H*BatdV0Q+Ii#%gbS9^#^RjFi zzU{T%V9+a^Znoi^UlGPJC9P|_jCMt1Ezw$*KJ^wMZE6y`w=W6(X#?6a1ZGD&1@`js ztdpdX+X$8!8t6e~vzn-v}ZJ2RN6t>CST3NyTGh{%-oCd#2BJ}f7iW*EnRyGcXr)Z zIH`A@nvXfI&D)x{F}`K5@G&+^`CT}dXG%eS8*}A{Bv*b|c1i5SiFsH|CbZ9Fo`mh~ zuO}E#017}6C;}y*#N+uoKr8F|F|7Dk*N-~z|GIvWrYaCODN+5r%&{gjFB8bi1oATJ zdZ^~4X-Seo$QElTlxrw1N>U`^E$r_?*w?<_-#ocM!1*BN^1XhMO%X*OLq7=2B>DH= zH^l_bVMtH>%ak?me1{1OnV(}xEc~_U*oe{fYu@=j6N>3NuqYOOVmfnb*WYBqEV>RX zv||j5yF?ZXe|0*sH|2YS2?Z1n%!!2`ozC*K>wQefqw7Fd66I?XknbzgS)mqxf(hvq z4|K-D4^Jm=GnIFo33j>;v|1@&i1K}TI(Uu8zs|5bAs)y=T}_2DrNOB>&97n~zPn0( zF~2|QiYK4HYRT@7%(DACG8gaa%v^@KRd)Yew(Nn<%<|T~nH2~xdte5uc(5aLQF9j$ zSKOb^D!vrb{@#;~JMc%cvEJG`_7wQK?kk9~j))ucs*(wPmG`Xx@pME=#W42Fy8cSW zmifbBPsP1$nZ@DY_E0F~p|M_u`LRqN(`Cz&l-l%;W@_Uyw5#I&woGsO>DK> zv|cOO&c0qu_N8oBUoY;xC)_bL6b{Yk>FDrO-tXIfAiQ@*jQMGgh%Ma%>I{Vnurv2p zR4RZb@jzRqFC6A+mR7K3yiI!DQk%a~VpkuRFgDn?z4pA0h=XZu zNy^)ad&l2)>>+t=k!kdI(&=Uy+)+$_gC7L&R4M+h!~VDA-gb$;{~7GMzrgLbcUQ5O zPP{s<(7Xiwyu|c`Nons8s<7U^YrX{YKD)mP5p>T~&o8haEWUTAM86pu?And|N!?2* ze@2dZG4d}){>Arq2*`r`xn^d{?q7lkqit^?fBC&F5`Bv_*tHA!t=)?$f2zw|j{N1w zUyl69g8bvmOvvtEj0od?jv@b|dxH}Fu5GZZ75U}va?0;;ninDeBII9${K$g*sb(f- z_m?BW=!039Bf+j_Gxw8v%$H(VREEx1KiLr9bq4Hfxm_se- zbMOcDRKL%MeexwfkExFqp~1^`bqG8Ub{4_f(j01O@u1$;6x68*JG~M(;sdmw<{;X) zn8tT8=GzJNW#7)=F_p-K?w> z_g{+pr|a&3B9tOdQGAWWO=)ODsdVy$TYXh5t?+vn)t)@@q4KIo-*soK^}q}fEM?hat~F zo`-xB@**Sx>46-DybSpkIt1bG|s4&)@{r;v9czks|4 zIR$wi@*BtpkPjgrLH-l+N64Qbe}SBaupMQhutT0FGzpWThfNj(N{b~Rh(6WgR||!F zp~*-5gK6W&+!G|?%esDanVZj>IKCpTB98KLj}sKENCNL1NTJe8t|LSWNUz~fp4CpYrtoe7F)n# zZC2?z_~tRX4m@bXJz4U#a=|n0bR8IcC+;3CxX1cyG=JgCjQzj~{~+*3@%OE`yWhC- zX)$FP(E7qmhkHmr{6utmsV`~}pYp;Y$8wJc~l&`XZnfU^$$i=z%W8^ zm~(BLgUX558z>yFC#*xd{3A_{8-RI7{>iZhn0@39$Mrzhkz08Fc>Q5D+M0Rf#@y?G z=|{ehyAqgoXefX*XT_`O6vo&);C4^WPrn(ycbN~zh79|48bG+r*ocR;_OeDV4O zjVJkiF^*+58ZMT7u-}SBsJE?~6{sLIpJ`kV1cVeCkB0((;^VP5@F5?M&cG=L_4&TQ zPkEnr1%AZ)yd%)Z`#crvjn&$fP(_wQ4!&pK#Ko@{;6LH*RT z%g6gEvg;n+PcQD;!u#o)yJ{WOFVFAV!29L7UDxw|>E5-1_scW8mgUmCdVEhApI49V zS-|JjBYWmLXg(g?GlP%gPW+BCjbodtjiVpG!%XAoQ`;Psz);}&F^hrifemBIfh~dB zF=fDDV9S_4_;UoAa-iN#kc^mR3!X zld|r&g{3x$CXQG6e>9;7hvF$$kx)+Z?pe1^iunU(K-z~?CZILvqb?`@d;IpcSUu&DUr$O?**kJcQ-;nM-+>qn#ZYc0R)iBFT&jUQt zQ0^t)%f5zc@7{)$-j0Sf-cZ9v@6LugZ%e~gufL&D(_!|swOgUPd>cB<)TAVZ=3g7^ zxM|7BJ_XNADXGj#&ysn8^lVupke)TW2gvh5=OHdl!hD?7+}6GgaoZ3_&!^o5r03PP z0_pj+Ek+)kr`!#H=(gVpY#Z>d{}9++DC4MUI-}>^>XD9~f4dDx&%@OL>G`-?BR|eE zpapvGqPD&-FnWg0gE)Go?k3YB>lGOt7YI>g@q zyaC9Zwr*-ik117-nc5;`gRVi`8pK@>yxu4)*~B)X-5b9vu#LbC-xk;gs}JrI^YdBm9V}4;@mId?=}A-ur6BT7BB2IjP-k89)0~`M=utwlLe-Lm`*&MkOe{cPj zz_tRn{JX#!fb~CvT@1YK{|M|>VBL=eRtK#8p};l+Jvg=Z0B`!9z&;OL{SASw2444d zfn5u%`HH}5fXnv_>}p{3mjqS~T=sy#mg)Di6yeJI1lY!&zQwx*Rsk%(SAd=4=__l- zxsP6ewmmJ*wzv5qDu>GB7B&rE+wY}?_cJs(``hFG+mi21r*{?SaC{=IbfCiDy6|;sciPQ zPM7knO{zJ`t56vOXHwFO#9`lI((}Y=-)GWc3In?-{3es0qA>7r3ctam777Cu3imPT z5efqjQ1}Fs_E8wPm%_)H)Inh&&hCjTrJWQ8`cG0@;YE2F_i2sO>3_&rKk4+qw@F8U zPWUq2J8|QchyKiUBkR9TcD!u#McCh7AW6#qA)Ow!C+YGj|ElTo_J5tnb*`j;o#}Xo zbC1AvdWoqP(&ZN((scO+hkoVQhH%NDpFyWj!nxg`KGM(aq!QBQ=O5B^`Qk&GEGm9heGN{^uK=gC*|CLbono6x_sUXnl7LH!Yz)~xbAx4MqQ_u z^fN)pMY??E3z{yU{(`2TZ8M)7Z&L{y`<~(l9P0KS=Z$ya5&jU0@+5|`06^n zWYx!4KO>Zwq0?g=-K5h4%Pl_mYKo<3?3u3*==3zM5mnRWU&MDc$hP{X3SA!fJig08 zeg2%P>GFqFO_zUGeUp5p;U4FYn5CED`f|P(MEt&~^HNuG0se`N`g=eZjEvu`l>ug5EFrvHtb7FAG6_mPB^FaaL{E{;XQ zD}!x+h2|J+e?N^S>`}hffjcP-yE3HBL7{`fuq*dc7`TtZup19h82AW_Ii4jCbLJ)dG)a+${e}+B*Vv<9alC;wM&oWh zG8ya5>7{ve<-?kdzVKmvT|ca^>xX~F=h6I!^|kx3zIGqh*Y3kFg6w?q1RJEX6-LyP!)N;#tMS4Z^y>c~t!UufNt z?o8{BbZ1(3G+$V%;CK-_^z+c64?~CUh7SD{bm+&SLq7r?`T(8{*#{kZFLdY*=+GhP z&^w_+w?K#XLx)zNLr0)Pe-k?NbI_rmg%14;bm%9cLq7%``XF@ZhoD1uL5J>y4jqOL z-3}eP4LWoXIAC(4&ZqrF;7~?uN%HaAmkQIsDw;(O)fVDtP2aL-De+<{Tk(TSyM*ZVW>_(*5bZ$d; zj>}if7}vRxp6lE|u5$yq-fWbUi~bk>T=E5!rRms)-WgXGd_G*qMjozX1G$b3D-168&~#19F~*LjWW2-4dgmE(2eVE;6Eb2W*ccb z_3a3g4t*Ptbm*Ibq%+?HBwIug{-xw|_?z_R>wu*9Ujrn)c{z~u<|?43Hyir0)}|%6 zPI~iVAnDDEfTTB<0o_dkBi*+Vo+hq4<1ItdeH-u6!rQh0q5C%8riJTkfzW*$Z_>h^ z4M5~ybAuLMcRdjK*IcKCSFZpf|C;4mxMmp;`PVGg!j)w>DaYBtY<4hmDf>zUk6T3O zS&qxog7C@TADJ*wh|>8l`O^2opPlCQ1r2}pM8O&mSX0FC*^{q+!rn0+MQBsWSC99$ z`tnoYzq86`eD&~?5BTI|xsLqyxqBTR_@?-?08Q}G`v&>wTO*lBmmf@SCttm?H@Q}{ z+w0Vnlmf@zR1Y)RyVEEyY*6lN(S7xf9*1fZY-+AUP?KfZlZ<+M;Ny?Mzpv1(TZL#3 z{PgQ_Pr1%Cj}<<7^3Rj+V?6v2xk+(;dF#i0@|fY=7oVp4;vM#M5BcEJ4F9|Qao;=7 z1D|`^2z>6Z!S6Gi&)uf`-0f+a&;2OUlFvOiX&9gTm@_{2e}<1p5&r0HgsspBTcHuQ zLL<)P8gVAqh%>oHoXIufOs)}Ua*a5XYs8scBhKU+aVFP@Gr2~b$u;6kt`TQ)jSULU z_0Io1kaUrs{52R2oc2X+u?+qk0Q~>cGvKEo-?xuFJjLJj#PpV~uTJ0D_2_i}-Y-uH zb$w;}1bw_oXTL`dL^FER)fjD=IA{4jd~>g6n56VBlPmpk(Uty~*_Hl?1(rSe_*da{ zgM9w0ESj%>mG0|b#m@>cE%5J*uOBND|JQu{e$st#c0+aYvc{HbWdSuGf2md$!tuU+ zlkVFW%96OxUn~WpOth25_C3vg^s#+k=RSEn`hs0RiIFcp1HO1(R{|Xi3mQ5W&QbOq z;=cN!J_6`F+@1iIkRJ2{ruqMYMHK!a)BOHGuIu;fetuw1Y~MEq{QPe*f$Q`@?&r^V z9M1>rJ2K$s|B>$J2RdW>{&~R9|6~0e9?1Rt8IMuE7Y6+NZ|i=3ApHC)_w#2wBC-sn zWjpt0?rO`7wYA|@XeCzO+McN+VsD( z89h%!>$t7sUifHt5BO*+;G3=3O}^Q1gw6qIEGlqR<8N>GENTvBl8-i{9qnps<39dM z%$G`iOe-ss@Tg9h`}iyOpdEXHnclS1t-m23f12jwuh4z`OUTFH1|NTWXlkgvZAK5C z84y_S@rpeHJJ8-S;O9r(Czjls#C&aS&^EmN6^qzXO1BK@s68|Xmfjc6TteyYPr^M0 zGkxvtJnfQ2tdio5wt!KNOW2*c-yf(5A^82nn%^Jo8;Re)R`>hgg0ug{&4%B8PXYP; zmy+H*E>HK{+nBO5BrN5=@F`~UX)gt*P(K)c|MKQ|zkQPCx6d&A_BK|&CnO+?=C_C6 ze{nhT>(3;R-@mNcFO`-0b-%q;^V_F7&7iCkWhNUdLw;n@{Pytsmm?zH?_b)i`M!f) z0kmK4E~EB4tmabWFGc=R1{o&h3r-RRhzOvSQ?)SH&|EMi~Me$@PK|Gt4-Cv5vR728T|L2MyxP1xG7t--by+q&EBX_&iD!Q6d{ z4Sv1^IgU@at}4f~w%Vm`Lt83O2;ViH<{<89m%63A2i@Y8e0k6( zvv@pR%X(!7?L)8BuwH4Pedv`yLR!Kq!k|;C2?O;eppJw=pR6PdbVVq|A?EI(LzWW; zT2Ju@36ADQ;)M=bKsab!BI=)zF3W}Pm`XS(@?*(+dJ*@Vfcur>{Lfy0OVbXMpAYm1 zwWk*~?M2Gd{Q`~Scb_x#_T6iDt;6r6pI_vpHus|GdA!SW_#NPJ{`@NEdeD@gU+!!N zP5ya>^BquaSf*3z(^$lLQ%=kM6x7p=>8;Ml}H2c3_B=I{BLb35pSJhhq-^>1U+B5AT$o;q)Q$8P*JWBYQP` z`SgCj4PQRJ``JUz8A|VJ))4uwt{x)aRqc?!JL!GQ93tPxj3FOn(=%Q+Og`g=Ve%O- z8TOo$-b-NEPo4Cvg2UyrS~*-kt6PW5XSHJZx198BzBybzo8`mhvspS^KAZaC*JM-M zS~OC&tpy`x+nP7B!b$D2ZsbMcT~667-(~e?`7X;h%XjJBe4leI_@;fYY3Lfzsoz^O zv<!ta3} z^t{kvgN_Fi+a2p$){SvELSqvXn@B!5V0#kv!cTIRHR*OA{Mp_pe>QyYUh12metL(m zX?7t4K#}i_*zS>0^0wzW)ILlXg%$}q1lC;0bNe} zZ))}xbR2Z0*K61Pd|}etgHjzdS02{1w@4O%zAgNGXeXj??f$9rx9D4Rzx2$L_Fm-t zJ^DShlZ&5`?PT&Z+^3)6KK+brC&kZjzkP=L?K9kOpLs*Hn~A%*Z|>&4xx4SaDf-?X z?t6Q<@9p8fw@0?85qr3=?cu(*hx^(d*`AzxSyt_3S+%!9v=i!c)J~|+Q9GeNN9}}U z*h={NR>0S{9KOB=`1kGiww*bDrI{5l#!PhqfzP?)c`fA|on+jiFHGF-3`1-u? z^#$SUyA{5^Z^GBN48Fdl@bxW$uWu22ee>Y!n+;#zO!)e)hOcire0|g4>ze{!Uln|P zGhRzQ|=hN}+eEdhM zoJ3h4$u|ppvjyKa=6jgRL4D-$S{%Px9N)^vzpu)H4}|JNeBv48Xj&My30?Cc=G{VN z>2>m>tSWJQ6CZy-m4$hDdPee^lfief;M>T2_fa`|ojf>RERJvB4~AhTRy3-57?w7>2zVhMgFOofw9V7>11)hK(48 zZ5W1a7=}$4hD{iTEf|I^7=|4fh8-A&9T8hTRv2-4}+v7lyqThMgCNofn3k z7lxe|hMgCNofn3k7lxe|hMgCNofn3k7lxgeG752OX;_6MY&4 zt4NMJqU5+iez&%)lX4q#xhP{$D3nKbm#_sx7$?HsN^-P77l^ssgv9W-$2-!KR{0W~ zkn1T?_Mb!8ZNJbQD@g9XhRJ=ByPyrU|0~T=Py3(&+CQW@*!F|E!2au+L)dblb+rF$ z%^~bb&>6Jzx0_u!^U%BPS^0PCeyv(S(TD)SIj!X!#bjXbOdnMGY7NDUd!A4W1)xWIpBGE z__?~swwuWMBOm&Mbk+puEVAv!v28boZM$sPcBDVDu3XPHo`Y>X2m5&(5O?CeWRY!` z%lajU^-Ct{7vLFtZe|e^bPRTJy)@f=SU}hywB9PP>ut7 zhHO1M^bFa0Rg-zw$#=D4(4U?3KE63f>g?r%q|RP8 zNWPDTK?k$xnJ*n8b$0y_`OKFL`Khpn7Y&m-dBHHLljjYS&#Z3P%}#novxiBYJZqTL z$uoz^XEbBjRoT?Wt{yIRa_w;0#-C z>gFb4+f{8|Gc=6y{F~Pg{a>EfeFXEmLug(XGJ*Oz#nxM~ri=PI^>0THa)RO_&$6x+ z{sLw1!2{nCYD;Q%U=31IxG|6Bc9pE4m}Zg_nB#@dhhikVjS;={E-_YQ+t13jAI58V z_ZV-m?Pq1%&&swR#+EpaaYY@S56ZTmm2E#O+kRHI{V-kxKgI`a`&nW8Mdo;4Alok? zBa!BKAroG6Nh;$cArnNOd-TUnij7w7C+W#TC&fxq`?>fL*?uNJBHPcTN4T#)BHK^# zBeMN?9+B;*;1TZ6kI42j@ln}+@*m|s`)J>NR`kioW&6r`T(+;w$GJ~FF5B0L$7TCU ze_Xb&w8yy*J}%pr^GVshQl4ZP_T)vP{ZL;dn~(Y;*?iO&sr`^_`)AvJG@sOnd0t_& zKo=)#Ey5mqp6BbjKnElKP1-G35sY?6_5+MTF`xG|s3-&TzS2I3@I1vlJ3*zr5;^t+ zAKRhM$NaCf4=mDtqIfgD)?>`OoyrmRQ>2_9@v&{H9Lxus>+vx2Y^C#s-4r?h`+V#n zbw1{S4fUvt)ME?tZlZEb^>~1fZB*r8KG$52`TI^;plm z?NpAb9_#qnT2&6_aLx5t!#r(tzNsE9d@M}IguP}FK2I9AVaSg))J>gZ;b-h=jky!+ z&Fy$;2^LG!{SR2ioD*{FI;v^oc8A_?4x##46yMZ2Ptf3X_sI6KK7J*~!DXpWn2DJt|&c!-v+!FaJ)tYJY?*bC*jl^7(X&p5VX(5a= zNY*rJ)^S*qciUUZt-ENQ`j})SF2EZ9W8lfxnmP#^n_|(3j&n+SK9cOweqNfS**>z7c(g29Bj45uTwK?Z zjkQ@Lwe@Es_kcDcBzbaVc}cmF4jLn=Yn(;P3}-fGHVwa%%5vvP{JaU0&d8^-!=l{b z6A@dnVf9$zv%4%>dfT~yZ0$*LETh1pIce=U)`zQkyj9sbIQ~0vyv&36;;&_5UN|F2 zV?Qy!r`?D3V*iS5BeqAd{Rg(q*xu~S)qaBQ32eW?wgcPa*bZZRyR%gL5w?f0eGl9B zvHd%?ZP>Qr8T|m;!`Oa^?JaDNVf!(*|HAXxiS4(YjoGdkan-A z-g7njuyUjnzy_X&v7l|w00Bo({0F|kYf#GTdf&dwmpEA#+lfb zG+Oflt$FKj+l{$wn#*4WTYO2*y3X@xU8j@Qc!oP~u;Mq`E?tM#!J_5t4`o54OmAh-X5e34SB=wSWUY3BpHI zQ-63A&*f039exfg{1qB}9&@=Jj+%mNB4#0JG1b2Lz@Y0j+*9*`!Fy11P&IEIt}jAP#&d8y;YJ+xOsV%+t$|#RC-C6aGXf9R&I~*_eOBOnyS;v5 zado}s@!%Y5ee;YNxy>_Y=7wsUZfl-Zo2y~YFErh%J!qd5NTJ_X<9GFcB^(a>XEz~qM*7Oy%juG2sxGs`4emE=jG=I z(hw_n300a8B>ogJM%L2cgEg}QdHMPE zv#YBypAJ5pHxkDkH)z^e*R=XN*P0Mu<|5Wh1<^>RjwLesorZv~)%W@`KwK;aX zybfl5ADprvFs7iO{(Rz_hBrQMqCDRv+V=iXej2Ha{njt@wa&9>h1QFM%?qaHCed}K zpp1eAfwSrSiJ4kqe!eU_-Ku@b>J63@7YCXbP04-GvnVi*c&6rR%?n)e+yd}SoS1<= zw131URQ^l*%J7aR78lo#CEh6$wB|(xxuQLIth7Xbf5?@K_pR+e+sE}OLLL|7IC`)+ z5Ex0v3npk2TrOFM6y(XX`h(2@Pp)WR#n>+>$e?mED&+OD@!r2wQ=Cyay(S|S*B^~~ z fBF>6G}Z3Q!GGtz-wi1(36WqS&=Jjla*$jwa1PrBa7O0GwlMH_Fe3eLXz>WuMI zresW*HVxN#Y2P`h!}*Y-;~;a#LiUb<44w#CJP$It4zjrjGTH-KeJx~mF=TfMWcVz| z^3jm#7ecn50~w!*a`F-9GT!6Kpgu6ca-~M?XFS$1)BDY{_{6i*z;o`@sTtH~zGV5j zb`Iz{mdkK`ZU6a}xf(swv6jW+UNSGGIrH87@-DE#_T0U1JY*R?qX{+`1>hTJxd~B34&ih6XcJ8%VVVrwR%ieCq{)jKzv$FB5+!L%?uC^Vz?~#2W$cNnQ z2U^{2xsAc>2U@ac9BQo!KCn6~xUnTW>xI_p;KtRN!TVaWGGA;h58k&rBeJtBBdOWKIeRxf_12RmBQ(z{wG;dg4Vy(KO6sn%4IH&djM_#7CoBD3FW4S9ZQf*q@A zELt19XLVArtz{bO;S8>8nU1=p1lP7qL4EDPaLZKOLvnCU%gi96tb@O9X&W&j(1x5R zZHSX52jvh3uSTg`S@?#y4ey0k5w!5N>euKR5DE)mOyXLlAqyE62#`ScsI zxRfaRRvA(Bb)3^mIzD6xch1ZSA%9M&?ZrI%xby_%f=SQIB8%oGIv#m=h+bgPh7-NeqKzVYIq3kS=w%kPUC>J{n#f&LN%ju* zeUyI*^vjh02^6w%5K+iuOTMp@!vXwd7QDl8VVq+hoHScvJ_Hk8VGkrC)-hyh>lE=PsfbB(u4{23&Llml<2Ljy^qX}p4rfa%`V0EX zx|U?;npVm=LE|aSmK01+qPRyI|E%lGaA<*ju&eHbUDvE>xuN8y+i2_^!hM~K+;3F3 zZ1A6JO_u!ja#;z9@K2-rR{b$4sASZod%y)C}FpD*6v}A&Zt~M zxK*zcv4l><5;_q}=tL|b(Yf8>Arg{HBe8{!H3 zufPH({~f67*hb_aO2ushN4%r(MT6#1&Y(2PHAL}`FhJ<|Ynmo})1Zh!L_B1U#IdGF z_@F^EC}&Wbh(Sc$Ba}UzcSu8(aXnDPAR>0rCGpo0wU*mbx zaZvogal{`A`Gz?rA>S~5r~EEyTULvh#ABh5$U9W^?UoT<#N}1qA>$Igt%yUsb6q4Z z@lM1c-nA|gmw0C;V)i(O@y;}@>At(h&^RiK-rp#evFFnJZQFGW<8;bFbSdSJLHu6` z?Rr1wAi9)u5K&AZBA)i8tw)R^Viu{b$UH zmqzCyT4&`vMC(KzBC>NXy&Ex#A{PIf=${P9i(v z@}R4Bk({JhvoGWKf~Yt}$}Kd7{Jz8U6R<=Sv3lbKbco3pU%O#BNHldKb3eNmB(v}QAFlAK}-UIRv`LuE` z?vvU(wGCf$lQj~b7?Sac))R|Q+^gafHTGxVT8JMIxrFR`{-IFw5)Eq_U_YhOvqj8e z)`O$kvsO2)vt!fxAP6z zhiR*{p@>-=Y7sGunkCejfVn0ZrK_Uz4QY`)9$tEm-YA_#x{7oZt)YiM*FiChtDv8n zjQUCB8-kt^IXR%0L{1J+k#7k4NaWlAeHiJ#M9d;|kjU!++LVAglAeM75&4Edp9&3i zh?qs_9FcDb6#0fCF^kYWBHs|`7RsTN;FRS;*NA*Wpe-+uj)kE`^Y#f*Ym@hA+P+lG zRe`=qb^-c`xbG0nNp0qQLs)BUeh#!YnVv-*_;R1*{5!m6H2& z-85dKn~F8MwDwlzB|`RFnU{#yJWk>@kEK}i*qa@&0@oy zH1XQ*WUTFe893z`*#}B_Mh<|gd_&PT29<~Rs@oJQRuS*9oaV7Xt3<3~8e$dK+#%<& z(VpqO{@=+rR4}v(_q(pSobwG$8Cs70(=p#rnuwPSH9K4ESS0d(3*{Rc3i)u>Q0uOt zAxCptNcMN?+o}(9e{G^ZZ_GE8!1;zE@s#NfM`*YnPpRh8yT#jxd_#yKX`%M@isle< z33MIp|59^^d_$l@wjpjr$TrXi=s4nVM7|-=@6$fwaYVi$&>zu0;yQ%v0~Ps(5Vs-n z4S{yrtx@@g219M44MP?Q*#!Egh@nLLjqb}+@Gg@en@AQZc@+dUxd-{tmy9@GIdG!# zFP0pr#(kX5`G$nea#x$knYb5r4%SMBC}xB54V{g=Rw2xnhYFzK#oB3gPC~4FL!)KBA@GSc)SzMwHKKw#5Hnue)-%!)-&qMc*Q2B;%T&#r#)lw5hzM(5o@18{di`6P>Pi(2W`}v zJIPjR!ZYbB-_SoN*U)I>##L*lW8@mbno(Ns+!WFD2;Lc7H3ym&=Bn-HtiwW2i-*bShKW|_CYt$K4Q{3Xdkql_OYgAE$xGbU#7Zp ze5xbLp5M(n6I9sqFHn3c?w#}^REUx?wP%ll^aPxQ}gJ471txcTsDB?3${s)pv@R2*M zD{>wg{No|&Gw?rzWgo&fSVSMv*2=zsZ>(S+W7iJahtFXH?SqP(NANdnqJ7YZ=s3o# zTWKG38|`BZx}EkxchEk@pgUn*7cyz{c|-QYju&$3Zq9jhH|IRMn{yuB%{hx3*zTYr?~ytW^Gf2pF`AyoNZ9N+E^KyCVY7pZyhrLh%tIOT9SxG@U~Wd(@8A>m zJE*YVLF-_P)~WL`e`U^hWQm#Yh|5;_j!=fM=cz20B}Tp@J_mC`!sZ9Bu;tbD5y#bY zz9T-*!}*TDFKm8LVe^BEd`GGr%;Ut)cf@6ta=s&+BW!$7VdH}eyIqxqxgc}CBTLMD zM_l$~&Ub_|gv}2sY<^HLp0^kD^UHTM2=iB`f4(F1kX;n}OW!^t+vsojei%Mgo$+_j zKY@M<`UlW|fW8Cz8R&bUf*11r?~wDK!mj#9!sOs*2`?cgc+D<+(JFLt7ryUGwwBl$ z4d18B(LUUN$CKhW#4QsN0j)kVLqa% zQGBzQk0|k7%X~z0qWESrA5r3)!+bhIDqd3;yZ(-!8^6dKV}SbMX%7I8kczm$#eB;A^XUQgv2!%E7mr z>0B36Dg$3p*tG)R(V+2+jLm=Hjp6uKy4NY_tr+UOF+$T;(mrU=i+vmqh2-%j+6Nw{ zebAPWJl@tNkFTM9;A?3gbX}J`-mVQ5<*cWD;2pFNdJokDX3@bTyKSiFBM0*=R@8Xg z!QVe8_g%xDg_mU8!FTq^{nF8L|K0zT`(F36SuXAIg8`qU^_R)x@1Z;o%B$w`{($mm zzh>A0QQrF~kM?Jbmir%|Jld~wAIL&Hq{}ZmkaZ04qId+y^mmC^Rr-c<7~ef^ z^b`l0Jf(qEo=E|(ws&79;!N8SM@+H=Ds2C`bFCJ_&mkPKq=dT(pD*z&!b?j9UO@Ol ziH{d-)}V-)OaDYX|qc=09sET5#R1 zf_ko(wD?9)nhXxBME_5?F+lY-~iO6C6Jp5Um*OM`T7W{Er}>!MrjUdWy) zGp$l+5oj@JF=z>B38>3g7j*mP1q*x&f`z_CL60vG zEb=W07W?XhCBCJ>(bvvOcF*-#3yM6}LTnyvMc9h5m0-)fb{3Lv1rE68W3&!ha2;Zt zK|R-7wL;M18ntitOKh z=QMOlqGxF+F17y9mk_6=j(_aGZ>0_M7FyQ(olDx5G%m5?TTak>{VvxkuH(@_ah+my z9c`Gn-g)YJxtDs%6D;bdIXF6vOKfuatRG>6!qOceVUo9m>diMox^&JuKN zbR9D<^Q329-o38RTAFqt#`EBcxlKbBfUcaoen0@UX=!#pcL!Sp-c}v>R7ePa}h>^?ou zlOgjp$ndkV#+PJ0=o;FGTYN3;gRax80&fpVx}NrdchElQfXp9|`LII=WWJJbW|{vX zWbfxK^S9%gXNt_PL|q4DzES4ayxFv?tNPJ{V;XH4&k}96Wjs%GwJqZXqTjY(Z#mZCy3r)OWjHICR^%LM89E6eVXVkw$x6dx7kuds7J2F zmeE8s!IlvwI>?sMLUf2NqmAe=TgDoq!)+OBiH@{otRp(gmeEdhv@K&j(XqCS4x;02 z8TSyqz?QLrsN0rtAJHOP#zvxLwu}dee%Y2mezM%%)>QJFO1uQiqYYPmnR zrjj2ncfU22{BpU!w5F1uF86h7D*5ele`8H0|6cBIt*PYi%l*AImHdCXf3T*KKQQ-0 zYbyB-bN^&brTPEdzgSbrx0w4^YbyB~bN_*IM}<0Pv^jP)hANw@u!g!4G_f&x7uFZ< zqWx5|aT@9{HeG-%fUO=I^1e4L$F>67N^Bu)`0iIj8@9FB+Oc(D!*{(JHU<+L9lPE` z{@%nU$F8#mH}AS+D85IJ+%4ow>B(23#&%vBds(b$y3Ge$fySMoY11;5KShoEh}SjD zI?5}@l=_%d+0K8ndV=TISdRV8S{yuoy5$%Kj2Lq)vsjnBw?FecrFQbms4*jzRXSRh zh4E(4E3X?W)jsXiLPvjNg*S8Mv6Z1zJK5~p4$`=i&hfgfqiBqaIyTWcI5(_pc5(e$ zzW&vb>)ZMIUf5v)pLLX$T61{Zc!wA}<9vOrOvmdllgIOqIznpf+>O^B%Kk76C z&yQ>@l%;OqL0(bcSSrjP8Qc5 zEqQQl%y}IBjU(!s#7E<2;;VC~jZ$NKQI}ag88hkhhWhOenAXa)_dic1@1zx3s@f<(Js<+3U#& z%}JHCmX7u2ub|^O4~uhVxFnr*4*M`Plwm>sQBg)O{tP+}-ZwtNcpld!>e03=KlI-a z6m%>ee+Sot&aFW`h}K5?K&XzW3*6Wt%w_*)jr=HH_)F-!2|iHu9R>AW1^Uhc zozsOn{RwmR`y}-%p9dYI?J%x~<8>cNx{SfAzfq0gu+ z@_|xWkj1(_D~swEW_?EIke@V$J}Vd3XML8}gFbUXev*Gx=(7pk^%)%(epl$b{J+Gf z&xG$aPJNcg`YbOFeWrZ8z3DUM_a&a*^qKMpN9(gZ)@OM!^qB|Gf_%rZ^cfu|pK`Q5 zy9w(Yd(>z6K2%J7hHpucyzOlsN9Z$rPqttBtnslS{nBTR&kpIAK5INM!P zeTMIceIEL(H{UtQ8llhLc~|JOKeImja#X)1o^L6A_B+;RlE$acK45(&X+)o)jJWmL z?^vHn>d}689Qy17QAUkRQlmaY-IP8%QU5+YuMp}f^qHiSqkaB4A4K$7EdM{1u?h7Q z`b^SU<~af4>F>M%ozsOn34JDMcYTKI34JE1)Mu`UzUp#GeYQ_hsn0N%KxNZ*xK!%1 zk0dqdv&Ap=PoKpcpG}|MO`pXxJ{!;aY$7;qXX9C)jqgF9sqtAQ z#j|AFX}2*P)~ty!9M;qJIEKqfk~+^N zsX^y$d!c_iFXkAowziwji)Rcsfpy-5K6GBJG2BH_eZ;Ku;u*tDV4XLi2c4(Ja7s=K zotM{L=c%z=G++Lo%6(F z^Cb0XbB{ykEoPl3sZr;lZc68!lyM&FDRiEs#&O=_h|cSEoQHY}ohRwZ8|R@;Lgz`^ zUFYF?Lgz`^ZJg&#l{&9R(r)9tx(uoF8YDI7y#M!nMCXw_+LGP8>m0}&H80ZBd|M1Z zq)W`#^fIqz_Br-6hZe)fSRl&oWzHz_ z8SUr$mrr8of`Rc*&$<9Hdj0bAA(rn{`}s(YpXj_4l|l15n6J|3o(Q8Ydj3dZl0PEj zqVe8;%ejqro{Pwcm~sL07BS>PpYs!DxzN*GL=3sm=iGyk3q8y^2)Ph7*AS0f=xrY1 zc;rG~a{=+ng`562hFlo15qp*kr=g8V{qRKn`=XzleE6z=NBiLwCi#zFE*yR~4!HpT zZw$F0?1kR^yk@!3lTSB>ToCp`Z$4Z#_7d`;2j6Y<@o1lIJaVEpf9&zdiN1WQW;vni zuEv#SJ7VGg#E=&Qc0|we;&iqnPLiKg$O4luRLO~Gf2P7D|MAO?{k!9k9q^;YkR8IN z=*?edmK{C$#$w10VN>+x3se0|$c`R-VA02;eP8j&j^6xQ$0IxX@=f(uc3l0J7_wu) z*63MwoX)n!N%A)dSzz)hDLE1C7gCtyKYrQqY*$2fP+!{$pT$AU4aOXsAVxTb9O<{u zLdEc!Wk*lGj8o;a5asp~qidEE)W@S^eCfD|TVfvt9gmL3jmKx9+ZLv{TeGb(H`3q+)X8jY(25x$(v%H6R{~)&`zPOww|cLF)?R^;M{t+<1KzYHs&< z;%rr)?T^ob&Ouyjw|H97?@zhkLi#M8+!c`#G3CNZ^jWZ6C^yT6o_rb9zw&yCmpopd zg($a|SjUqg7y9y9#48v4e~uv+V%QOJ$pyqT_De1xwy|Gw0kMt!k_(7!JYl(jc*lOp z1=t(?k_(7!JYl&IXH24y3q8ao3b}B~{T5O#{J1j?xo{GF7P0Jwo_rb9e?JeOg}6>H zF=Hn~PW0uoP-CQcTGK7E*RR^h_MG z<0SelV%Zcu`7)?ae;z&yah+abZ%&5n=*wr(U)eGKPcdZ2fURMa9cO~AaU4DiAv=1A zlM%Axl>05D?6~{si0q*DyC>_5;j_@-OQ2jXVm=jXkwbF7TTT~}8|U}pv!EO=C?}Hh z#pJUv=YTntK8sNw#gG>RwuVt&4EQYcwYnrr)E2!CR$;wUDI?bYG$JF;bf1McHA*f} zoQ2$9LpbGnqJ7HsL~^0GxC(EYlnY&w;`>cTUnU)=+)#9!azphb7fzPX;_UwyLoN*1 z5k|RiCfE^Ykk6v!sW{}qndY;|k@7*=^1^<=+Bj{rz}JLuoa6-UD>>0yTtz5X$_ec} zLA%KbI!nP{C);N+_`hSwivc^rC@;Opp3jgi+a zWd|K6*+KhCcJvxoQ6Oc9x3Fi~(S_qAJLq^U*>SRc7N7nxhU^%yHH@<3Ot3Z1P@hF$ zXGC_8o_;%1`YgKTh-*sTHT-1eh(m6;+{ETxYX%=@ZaB_SOumVM9C60=FB<1{KAm#K zQJpBq^yiT?PUWkPm4`shN6~R}-nf7HAcjniVb7^pmA>Y#Rg6l!bJr?HCEmGf6{8aG z+_joVI+3|+6{8aG+_j2PiFfWgg!e`zXD-u#&6ys5Er60k0;iq?gq z4*1Rp&8?oyd~-?{dLUE!n{#f*A7bdj7~>P63kPz}#h;6(A35iQE?nJDUAWq)3wz5w zCv@Rz)`d}iQ=to27dWGH&9`Ku$W=g=bn$y61QFPZxH}NoUrD1G(u&nf>owGH3aklRZxC&3A9sjj?jno$PU9 zZ~lIz8)+QaQ+{#v+(Ur zM^hZ6t}i*4UEE_wu+KH>EF$&z#5n+;?K=%YiXv&-(Hdt}Xst z$COd?V`@wpo%>FWDV4tLE&pA=#+RnpUtM2v9=y26m&n^D^rfT&x$sV8eECnzg_r+m zL|>A;`e=CbuKz;LwQjMP=5gd1zP?8|*p?F@duh?z!OJbf?|MrtPMpO4& z9_M;SN3oW znNrnP-1o_lDZRyg_gAK@_+1Q{62mr)OQsC0S3lT(3fHSE8`dCGFz<09^BKy%)J*Hx zg-pSmg|ZpVa~HZyiTWOn!iYwHYbF|x{aebEV?XMLOgYnH#HC%)rpAscM%?V9jv-@; zNyhZ%cRm?1CLSNMs(-xp%e8OEkTnDLi!t`|G>RLK%YHclzp!r0nAX5|vt>*^U?o3J zUJgGgcYgawKjh9C7GF;C1mhW_-08vB7enp{+om`F-N}$U@%ZBUD|asWZ49|HVB;9& z&S?~L9+!=C0{%7K-Z8D8?`H3qd}+GeiTW;t!ieTXeRClikNsQ9o#(bir z)c3a4+;?xW(z;z^ika?a*L+TS1Aq9#hqf0 z{}n@~4A?J5nR2GtFa3-;)oqs3#7A?_!~KvcXPS>DN7^fDtf71~Y8^kz7>{{9f3)4w zn~&yHuj^MbM%6!FAI;I<#E>-u_KQ*0oN4yU7syAm;s^bZJ7<`Wrmq|T%C<4*0*JP4 zdiBwq;+z1-D|cRdGltw5uyKrX=S;J4zCb>jnOh@rhxF=S&XPWw#HK;J5-iPnJk7_} z@?fZ;4%-530c`cy8n7+LwgTHqY$0r6Y;D-qVr$3Nfo%h}jY0Akk4kl!*`a6=Xi26i+Y%Hll3+Kl&dX1P=ma^=Osxx2K!fHe=PQiiZZp% z*9Yfnnkb9EBRRf@?+jB})4437RMu5o7Evlo#G{}r;;m&~qQv`E<|RtJA{GU_=6q^& z?$vxQQ9Ac)d@fNsSF9W7bED&Qlx=nIuPeqZ(O%m3p~a$&*0lYjEkD4o&@L)_2A55g z%BH-=^gM_Xk62I6_dwsXr)Ni$c&}kzqQooKk~8lt<|RtJ*D^0r;uY)2nRhnx5+&X_ z%u7_vC4(;%WhX?(_J0H4=GC+{yVQ42l%G>G`A>69{?ZWiFV%zIb8p`@632V_PPaJT zOMYH)yqE7)i@ts zfNDRn`nl=WZ|P{s z3!b1?mKmbs@BXJe?sZ#7nZNa-zV+Zmd}hEWkK_CnnyaDtWWwt&vyM{nE4}3%pf!*) zEY>tS7iH)%x#qPWvDQjP=MNBNjZPcYCClod`}hF&L3o`zZIs4wxgY!QTdCGN5~ch% zR_LVYZ&1JXdW`W$48Mnpo$V_gPQ}c|8)K_tX5)>qRWY;i#@H^${nIy@ge`*()sy_9 zdm=mK9M~X4v#?Kg$|$y1R-labQ@Ey4w_8v@stegGR2QOD7qVBVE<}k}#liKJ`-XVc zrrR^7`58LiTRs~)ZjP(f*H@q{i>4hBWoQQ>t8TSZow6>v)$WB|IAx|)%f9B~5p;YY zkBv6V*LafFA|e+M)*yQKN!ooa;=|&2-A9rx(6svwrF%8ecIaJ%u$D1Z(*o_1)@w;P zULHBV32PZ~ykWbf%jq18Cg<&c`A}p`cwBjGqzvsQ*J;h7YuK|{G32^8TCNjs>1fFd znc7XR({Zo+*{m3Hz1)TV9Pr8GIA53Rgx6p8Y*svSJyt#$vs{n0rY~BqcgqXIa-Hxx z_kpZvx$Z$7DaQ*@T6brb>ve}>$n_Y0B_Y=b^0}Ond@j^);>~MDywP&Kw>&PgKT6r& zpvI}DIWpquQ?B=smqnD(ms}sn$ui45*JCZfTDd}Oo)cL!hi56|x}@FZ`Ufo6C6#jB z6_MLrE-BadNh;-fLFD-1QYqIzlGGsA=lom$kqvedWSG@6~>`-#a)I_JR69%R$RQD?lqi{h)r(O3+HsD$pv>OF=IMrE7bAp`g## z6fF0JMVS@8mZ0C)7OeEG30C>m1~2uk3y!{aRxGOT@parPd$%61m`6?!~ZMU4%#5s4;BP0>Yzh(Js#4WcC}-*gtxWuI>wn z8vEOEkI`fOH-0%FJJkHdnL5^2ZI@_oV|hAm9%my94hUkS00SC^_QdOfsfZtQgbk7d+Ai#Np~IWUwbDRMR6bG zt0zjndW!cTUp-MZXAQDoy0pAqUKIcWG+O_r-=9sp0A*}T6zXV=^0Sm z20a6!YVJhDa4HZkHV4%GVJBDIM2JEEe^iAy#}4>(m(pC3%dD}`m&SMdRqF1HT4bi`dRb3So1fkd+HmZeKGpRfSqLQ8>i8_4r-%k zf}Qj~|9aegqyKhNj?^1!{ik`|r+Gc6`MbI}Owo5uWI&LE8;fQVBTl~Z( z#Q&m<8QUfOLc||G{@o}W=vlc&kJeut`B|a;2Cc}-CJz($e+j12{Fb-LMIHwCZCenWR-8Q zUpnD9zBMUy0=_LNbOOF7seEf@pR>>j1K*Pj7a>-z2(fa-pv9mipd}nT7eZXH&UIOvWik#}Y;mjn5QzyOp4`%?6{TYn-Ht1ug4}lI{vw@`6`;HqBYiC z9RJL=uZM7yqpzP)$M*eeiXtHFL@~HP2yv^QHdDoxc1MN-o9gmr#C;c>NN} zj}fn5LisUH#4oX&#_6Y#FH-q2l&@a-DcVra(BfQYEdPU&D`r2$KPxw_J*sbYo8yYF zU-f&gD@MQa$LLp8$J?(q?Tyi|VvO

Q@7CLZ>uNNR0&z{VMAFIp1&W|L4g*Cw=wOi!u6D3_Cflel-wdB=UO1>ql39_jr9K{nb}yJJ_tB zV$YfNHK(SI%el1$9g%)Ax8w7(?KJciLo8As-$LjsHYt{EckDk0`U>^`-g4Ru*mkl{ zoV+!0vTuF+JCVM1oI0rAww*bSiN{QSb37A|nX3AVYl_vc#>eVc1Gb$pe&aNXZTh0v zc0XBv!u_iMwq1@f21@ru0H<&FP;6B6w|n}EiHg;)u6kiWUwzT_)#qi~eRx--pS*up z(=G=RJkVcWx}<|8(a>(iUjow|d_0-=>zV%Lvtu{X&jvnjXS{>=c~>V>7caej;j&3N zzAOAWQ>XfK3JNB=Cc1L+F08+C>2(X2&1smsJg{``4GZVoeB+$ufw{})ENi&o<~j4{ zE}e7pHyRplT(%q>^RJ(K-QuO!;rPNi3l`4%#&vTRFI{xwoJBV-n=`k*e$I_I&uLh8 zQE$mf%&Ef{A!1-U2jm#9^na4dNf`17W6 z&SU%&6Q0j_PM-Lo|0`nrsEPk##nedMp51H`4GybXxw~L0GsBJgh z*AT`(HQ`9kh`&T_GV(}8|DDTthY8PPJjor+t!Zw?Q%v|I#`l@<3dWC^@TrW?Lf)$A zzppZ0XToPQ9ya0EF}~8&J{uTsFyXf_zEO5G{nAa0Pc`9pFuu>Dvx*-bjAxke`x(zO zmGgbZkDB=ZgYgX}{+*03FyXryuQB!Cml)44GuHDU<6Wgj{7uF;nDE~-e#nIXH{;c& z@;_mGp9xQ}()ALxLneGE*9mjZRk`9R<7clFX@o@ft05BOVnzlfYlE!WPE3(4v8NPjC-q$__r8;(S)}!zRy(tU5u|Z z;Tsv>V8Z{M@f{}oF~&XB#`-+Ncwnj#e^Hfh!Vfb3=@mx)w-`^EX2k!%c=|O){HVeY z8Q-sFBfdm!Q>~GIFymRKaz-%zsi~ceW_;G;#&Yr*_nXQoVSMLh#`=^q{^q?#{0hdG z-(oC(2IGNyjre@VYfaZ(%6QgnBmXUo@3_=>-LNX(blp1{-?Y$J&V7uJxxt8kkMXU` zjrexPJ(M9V>hGtFZ(kJ!;^zyD=igz(_cOkAy%B$t@x1$t_`8fhwAqM%%y`ZZjQBqo z-?-g~CkaC}QOkM4h@Zvy^QQKY&3KmyAJ4eQ)E+#HZ#R{H3FBENdm8yR2mq>+EE;y2;;KX{1e8HnB;>^41f}~4ii3v@$_F9 z>p7h93==+z@u?o^hcst{pO!#*hxBt>u&KAa>e$|Noknt`P z{|?5r*NptnFuuXW{{rJ)6aOoWKV;%R#CXWW|69hloA}>nyvxM@XT}@kz(KFc#aB!1k=_@jjGw=(|TQR8(VWqhLve~R&)e=+hu$M`2E{AI@5P53VvFZ~~5Id3t( z(Zv5Q<10-3A2Ghw#Q#^ucbfREXqwbM7yPgBx`P?dGvVose_|?U6ytR!{&9>SHsLPD z^FJ|OcakdSuSVR@_!wjdjsBa)xYuUHXE2_F45iV3^BDi5>Alx8e#nG>lkvQv#&SZ8 zA9Wh>HH^36>r&Bw9gKIG%6WkCr%m`)#?w=cJkxdmoAIqC{1e7km~d+nJr(#dOwT8U@wKCj_celXuL;j#ye`|wKY{U$ zCfvh#sR_TB@k1uOnsL7gpU!x?biC-VcoyR|CjLbVH}N+x{*ej4mGK=WJj{65gs)@# zk0yKrKP53s(^T!$apI|&a*NAs9zQe@-65|WT8~I;l{8Qx_n0dVT{|oMm&r0_DhWT`HZhM;RTFuGnMmY z#x)aO&G^%%`dr2MCzZzQ&SpGp!WT0hGU3Y^-&AER=XS=An)uf+zTGr#y_@m;DaLX( zF`i+>(LL>hZj4%JD5#P;tUaJv*nek21!Kk;BUoqZrm688##*fZ0;vX`e^>riu z3FF?GM%Aj}_;rjoTyMmeF}{7N5xN z9gL^kY{Yjl{-OzgiSaj0_%9jX_YGq?hZ#@(lM(+T<0&hQ`2R9KW~C9g4wmg?!R{{LpFen3LK=)-EG|3gccAp22waZX^E~#&wXaM33H{>lq*OqLF_E<11b^;$g-QzhcDO8Q1n1@r{gkneeTQ zA2H!O82{q|V>w-n&v?~{zs&dw6Ml&C(q9_+k1#&Vgnz{Nf?pZ=KV|&L`NsB^OfqelK3#>)@wPTI=9~s}7XT(2YJT>2l+lNx2iCW-7BmOPIF*xq>8Szx$X*LX| z$3;~JKmEXmX%5XhT;g7Jc;eC6LTbrTn9iX@NRWrU17>)Q4oYNh3c$Ab#;%2(xfgYV%73fjrcvLk!dO7NJkGj($b*M-cSfpxLqgjQU(KAFslLmm@lH*ppMKR`R#{Ow=c-!Ybiw8S%Ct&?FP}D39QFBqb1J9# zwfed97S_)xt*r9!PgkLV`@qufDWcFSoR~qJrA$w6CBzE~?b8KM=G%>eyA6O`CJo<(Jj)Pq*k@1X52Vs+g$% z(7v1c82zArBmnA<;)l+n&Ow`&hlB(p9w99SGxbn`=s&2d3PfGHKoy9m&E=2~Af7^X zmXK)zQKt);CJ>Sj^g~EOfv7qQ0VodXh`JUYFrTGPM{kozk-B=3VivNI$`P_r{7?m! zsMCdzq(eeBiXW<+5_K0sHi|>)EYT1MK`)Y5cSoMQf`m*YrczZ( zQMX$^!=v*Fy+}22>&?Mkpl;V)s9&u>uSenrdTvxA^P=w-K zp<~4_{dK$b7T^|oQXbXoqBj&o1FO3%(rbl&%XhA~2)Euy+(NmE@`O^Azw|l_Z7MLK zJmpdSX$Z|KkLp(w3RGfxDSF#<>n)J9p%jE=B@sDS>{8ONtO!ku8H!4k7%MB*Ra{vy z+J!1Xmvd$1RZ2>gm6z&;l|#%hZ-p-8!K*X)b*bue>Ehhy($yg(sVbqOLRU0?zb@o` z1$w!@GCV7JHD5$g`yv{y+yyBj8OkG4wY)&r!{tSl>MqKQ(U_UHMAri4B_8!i%JCqW z0j*OP*X4d)TvxdD`c!!I`>gQjlDi`EfGdjh+o&ihQ;(;jR5vFo%5<$>QC6X9Sy86z z;fiv-T(_r0uY5(NexAQTSFZjd-3&o})#d!f=sxnE{l&We_Lq9p<@^y%?T=_lf5edS z`}Ef1uh7eN7rD>}x!_9ug;o~$6UK({Pp@UAPj4NS z<+@sjde%$u>*b>F>F=(xQop;(N?n7yT|OvPS!1`mLVu4{F1^ZCC3@wm{Q6y2`E?tq zs!~_yRaJWJ-7dHOkliki{xI-cZ+nof`W@hRq~5Mlz25HP0$9^rLsx~~&)ja8U)?ff zZ{&{Lo`_8cVHUZ&BE3^UF6$4&?Jm_#8~lzcvrKQEZa4b6s)IYyQ&CLhe%+ONt8*6= zK;3bj3yLaLV{;coWD0&qo=HKZ0k{hyeZgJe(>ob{N0nI~c~%AGk^3$1>&@L=;@97s zyRabgE(%NZuH-I^*r|Agx)eme(tDoUyJ_ySxyu*d2yE#Ma~f{0Jf3qV zUcW$_NK^ENzrQNr7$VF zmYA*u&0w6|AOCag*p2G^hIDyA(J{=Yj;r$-)<)t7m936<@cD}7FrzxI>aXazIBttN ze+QqhXg!~=h|%k%Y3JhrVXA!HFJt!4Z~95&cxpFO_i$t0zpI?<$w)x!4c3PxvvJ8N_`5S1{&^ zu!vvk{QofF>U>47#&OdfiZc%L_4Ce?1yD7Ou*pZWa%5Ax8~S^xk5 literal 0 HcmV?d00001 diff --git a/hsa/gfx950/mla_v4/mla_v4_asm.csv b/hsa/gfx950/mla_v4/mla_v4_asm.csv index 0427a938720..5b96c10e846 100644 --- a/hsa/gfx950/mla_v4/mla_v4_asm.csv +++ b/hsa/gfx950/mla_v4/mla_v4_asm.csv @@ -13,16 +13,21 @@ // qType : Q dtype string (fp8 / bf16) // kvType : KV dtype string (fp8 / bf16) // Gqa : gqa_ratio -// ps : persistent scheduling flag (0 = non-persistent; v4 nm only -// ships non-persistent today) +// ps : persistent scheduling flag (0 = non-persistent, served by +// mla_decode_v4_asm; 1 = persistent, served by +// mla_decode_v4_ps_asm: in-kernel split plan + merge, one launch) // qSeqLen : max_seqlen_q (post-padding) baked into the kernel binary // (4 for the qseqlen4 variant) // prefill : 0 for decode, reserved for future prefill variants // causal : 0 for non-causal, reserved -// lse : final-LSE export flag (0 = no, v4 nm always 0 today) +// lse : final-LSE export flag (0 = no; the ps=1 kernel exports it when +// given an lse buffer) // knl_name : kernel symbol in the .co // co_name : .co file path relative to this CSV's directory // +// The ps=1 row is looked up by mla_decode_v4_ps_asm only (gqa=128, qSeqLen=1 +// callers, normalized to the same (Gqa=64, qSeqLen=1) key). +// // Dispatcher keys lookup on (qType, kvType, Gqa, ps, qSeqLen, prefill, // causal, lse) — exactly the same set as v3. // v4 kernel gqa*q_seq_logical=64 tile serves three entry points (gqa=16/qSeqLen=4, gqa=64/qSeqLen=1, gqa=128/qSeqLen=1) @@ -31,4 +36,5 @@ // // Required columns last (per codegen.py convention): knl_name, co_name. qType,kvType,Gqa,ps,qSeqLen,prefill,causal,lse,knl_name,co_name -fp8,fp8,64,0,1,0,0,0,_ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE,mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co \ No newline at end of file +fp8,fp8,64,0,1,0,0,0,_ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE,mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co +fp8,fp8,64,1,1,0,0,1,_ZN5aiter39mla_a8w8_qh64_qseqlen1_gqaratio64_nm_psE,mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co diff --git a/op_tests/test_mla_v4_nm.py b/op_tests/test_mla_v4_nm.py index c1dcd2bbc4b..cc0e819ae48 100644 --- a/op_tests/test_mla_v4_nm.py +++ b/op_tests/test_mla_v4_nm.py @@ -2615,6 +2615,313 @@ def test_v4_nm_kv_tail_not_tile_multiple(): ) +# --------------------------------------------------------------------------- +# Persistent kernel: mla_decode_fwd_v4_nm_ps (gqa=128, qlen=1, one launch). +# --------------------------------------------------------------------------- +_PS_HEADS = 128 +_PS_SENTINEL = -3.0 # rows with K=0 must keep it (the kernel leaves them unwritten) + + +def _ps_inputs(kv_lens, seed=0, pool=None, device="cuda"): + """Packed q / KV pool / tables for kv_lens; row j gathers kv_lens[j] random + rows of a shared pool (page_size 1).""" + g = torch.Generator(device=device).manual_seed(seed) + n, total = len(kv_lens), int(sum(kv_lens)) + pool = pool or max(1, min(total, 20000)) + 97 + qp, qr = _native_to_2buff_for_asm( + torch.randn(n, _PS_HEADS, _QUANT_D, generator=g, device=device).to(dtypes.bf16) + ) + kp, kr = _native_to_2buff_for_asm( + torch.randn( + pool, PAGE_SIZE, NUM_KV_HEADS, _QUANT_D, generator=g, device=device + ).to(dtypes.bf16) + ) + kv_indptr = torch.zeros(n + 1, dtype=torch.int32, device=device) + kv_indptr[1:] = torch.tensor(kv_lens, device=device).cumsum(0) + kv_page_indices = torch.randint( + 0, pool, (max(total, 1),), generator=g, device=device, dtype=torch.int32 + ) + sink = torch.randn(_PS_HEADS, generator=g, device=device) * 2.0 + return dict( + q_packed=qp, + q_rope=qr.contiguous(), + kv_packed=kp, + kv_rope=kr.contiguous(), + kv_indptr=kv_indptr, + kv_page_indices=kv_page_indices, + sink=sink, + ) + + +def _ps_call(inp, ws, out=None, lse=None, return_lse=True, kv_indptr=None): + n = inp["q_packed"].size(0) + if out is None: + out = torch.full( + (n, _PS_HEADS, V_HEAD_DIM), _PS_SENTINEL, dtype=dtypes.bf16, device="cuda" + ) + if lse is None and return_lse: + lse = torch.full((n, _PS_HEADS), _PS_SENTINEL, device="cuda") + return aiter.mla.mla_decode_fwd_v4_nm_ps( + inp["q_packed"], + inp["q_rope"], + inp["kv_packed"], + inp["kv_rope"], + inp["kv_indptr"] if kv_indptr is None else kv_indptr, + inp["kv_page_indices"], + inp["sink"], + ws, + out=out, + return_lse=return_lse, + lse=lse, + ) + + +def _ps_reference(inp, kv_indptr=None): + """(out bf16, lse fp32) of the fp8-dequant reference; lse includes the sink.""" + n = inp["q_packed"].size(0) + kv_indptr = inp["kv_indptr"] if kv_indptr is None else kv_indptr + ones = torch.ones(n, dtype=torch.int32, device="cuda") + qo_indptr = torch.arange(n + 1, dtype=torch.int32, device="cuda") + out, _ = _torch_attn_decode_fp8_dequant_ref( + inp["q_packed"], + inp["q_rope"], + inp["kv_packed"], + inp["kv_rope"], + qo_indptr, + kv_indptr, + inp["kv_page_indices"], + ones, + 1.0 / (_QUANT_D**0.5), + attn_sink=inp["sink"], + ) + q = _quant_2buff_to_native(inp["q_packed"], inp["q_rope"]).float() + kv = ( + _quant_2buff_to_native(inp["kv_packed"], inp["kv_rope"]) + .float() + .view(-1, _QUANT_D) + ) + lse = torch.empty(n, _PS_HEADS, dtype=torch.float32, device="cuda") + ip = kv_indptr.tolist() + for j in range(n): + k = kv[inp["kv_page_indices"][ip[j] : ip[j + 1]].long()] + s = (q[j] @ k.T) / (_QUANT_D**0.5) + lse[j] = torch.logsumexp(torch.cat([s, inp["sink"][:, None]], 1), 1) + return out, lse + + +def _ps_check(inp, out, lse, ref=None, msg=""): + ref_out, ref_lse = ref or _ps_reference(inp) + ip = inp["kv_indptr"] + live = (ip[1:] - ip[:-1]) > 0 + assert torch.all(out[~live] == _PS_SENTINEL), "K=0 rows must be left unwritten" + if lse is not None: + assert torch.all(lse[~live] == _PS_SENTINEL), "K=0 lse must be left unwritten" + if not bool(live.any()): + return + a, b = out[live].float(), ref_out[live].float() + assert torch.isfinite(a).all() + cos = torch.nn.functional.cosine_similarity(a.flatten(1), b.flatten(1), dim=1) + rel = (a - b).flatten(1).norm(dim=1) / b.flatten(1).norm(dim=1).clamp_min(1e-12) + assert float(cos.min()) >= 0.9999, f"{msg} min row cos {float(cos.min())}" + assert float(rel.max()) <= 1e-2, f"{msg} max row rel-L2 {float(rel.max())}" + err = checkAllclose( + b, a, rtol=3e-2, atol=3e-2, tol_err_ratio=0.02, msg=f"v4 nm ps {msg}" + ) + assert float(err or 0) < 0.02 + if lse is not None: + d = float((lse[live] - ref_lse[live]).abs().max()) + assert d <= 2e-2, f"{msg} lse max abs err {d}" + + +@pytest.fixture(scope="module") +def ps_workspace(): + if not torch.cuda.is_available() or not _on_gfx950(): + pytest.skip("v4 nm shader is shipped only for gfx950") + return aiter.mla.get_mla_v4_nm_ps_workspace() + + +def _ps_ragged(n, hi, seed, zero_every=0): + g = np.random.default_rng(seed) + lens = g.integers(1, hi + 1, size=n).tolist() + if zero_every: + lens[::zero_every] = [0] * len(lens[::zero_every]) + return lens + + +@needs_gfx950 +@pytest.mark.parametrize( + "kv_lens", + [ + [1], + [64] * 7, + [129] * 256, # HCA window + one compressed token + [8320] * 4, + [0, 1, 33, 8320, 0, 500, 7], + _ps_ragged(1000, 2000, 1, zero_every=37), + _ps_ragged(3000, 300, 2), + [8320] * 3 + [5] * 600, + ], + ids=lambda v: f"N{len(v)}_Kmax{max(v)}", +) +def test_v4_nm_ps_accuracy(ps_workspace, kv_lens): + inp = _ps_inputs(kv_lens, seed=len(kv_lens)) + out, lse = _ps_call(inp, ps_workspace) + torch.cuda.synchronize() + _ps_check(inp, out, lse, msg=f"N={len(kv_lens)}") + assert int(ps_workspace.cnt.abs().sum()) == 0 + + +@needs_gfx950 +def test_v4_nm_ps_indptr_offset_and_tail(ps_workspace): + """kv_indptr longer than N+1 and not starting at 0 (N comes from q).""" + kv_lens = [300, 0, 17, 1024, 64] + inp = _ps_inputs(kv_lens, seed=3) + base, base_lse = _ps_call(inp, ps_workspace) + off = 11 + idx = inp["kv_page_indices"] + pad = torch.zeros(off, dtype=torch.int32, device="cuda") + shifted = dict(inp, kv_page_indices=torch.cat([pad, idx, pad])) + kv_indptr = torch.cat( + [inp["kv_indptr"] + off, (inp["kv_indptr"][-1:] + off + 5).repeat(3)] + ) + out, lse = _ps_call(shifted, ps_workspace, kv_indptr=kv_indptr) + torch.cuda.synchronize() + assert torch.equal(out.view(torch.int16), base.view(torch.int16)) + assert torch.equal(lse.view(torch.int32), base_lse.view(torch.int32)) + _ps_check(inp, base, base_lse) + + +@needs_gfx950 +def test_v4_nm_ps_workspace_reuse_across_batches(ps_workspace): + """One workspace across batch sizes: counters return to zero and results + match a fresh workspace bit for bit.""" + for n, seed in ((1, 0), (700, 1), (5, 2), (2048, 3), (33, 4)): + inp = _ps_inputs(_ps_ragged(n, 1500, seed, zero_every=11), seed=seed) + out, lse = _ps_call(inp, ps_workspace) + fresh, fresh_lse = _ps_call(inp, aiter.mla.get_mla_v4_nm_ps_workspace()) + torch.cuda.synchronize() + assert int(ps_workspace.cnt.abs().sum()) == 0 + assert torch.equal(out.view(torch.int16), fresh.view(torch.int16)) + assert torch.equal(lse.view(torch.int32), fresh_lse.view(torch.int32)) + + +@needs_gfx950 +def test_v4_nm_ps_partitions(ps_workspace): + """Non-default partition counts agree with the default one.""" + inp = _ps_inputs(_ps_ragged(300, 3000, 5, zero_every=7), seed=5) + ref = _ps_reference(inp) + for p in (1, 37, 1024): + ws = aiter.mla.get_mla_v4_nm_ps_workspace(num_partitions=p) + out, lse = _ps_call(inp, ws) + torch.cuda.synchronize() + _ps_check(inp, out, lse, ref=ref, msg=f"P={p}") + assert int(ws.cnt.abs().sum()) == 0 + + +@needs_gfx950 +@pytest.mark.filterwarnings("ignore:The CUDA Graph is empty") +def test_v4_nm_ps_cudagraph_replay(ps_workspace): + """Captured once, replayed with changing KV lengths (incl. K=0 rows).""" + n, kv_cap = 96, 2048 + inp = _ps_inputs([kv_cap] * n, seed=6) + out = torch.empty(n, _PS_HEADS, V_HEAD_DIM, dtype=dtypes.bf16, device="cuda") + lse = torch.empty(n, _PS_HEADS, dtype=torch.float32, device="cuda") + + def run(): + _ps_call(inp, ps_workspace, out=out, lse=lse) + + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + _ps_call(inp, ps_workspace) + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run() + with pytest.raises(RuntimeError): + with torch.cuda.graph(torch.cuda.CUDAGraph()): + aiter.mla.get_mla_v4_nm_ps_workspace() + + g = np.random.default_rng(7) + for lens in ( + [kv_cap] * n, + g.integers(0, kv_cap + 1, size=n).tolist(), + [1] * n, + [0] * (n // 2) + [kv_cap] * (n - n // 2), + ): + # rows read a dense prefix of the capacity-sized index table + inp["kv_indptr"][1:].copy_(torch.tensor(lens, device="cuda").cumsum(0)) + out.fill_(_PS_SENTINEL) + lse.fill_(_PS_SENTINEL) + graph.replay() + torch.cuda.synchronize() + _ps_check(inp, out, lse, msg=f"graph lens={lens[:3]}") + eager, eager_lse = _ps_call(inp, ps_workspace) + torch.cuda.synchronize() + assert torch.equal(out.view(torch.int16), eager.view(torch.int16)) + assert torch.equal(lse.view(torch.int32), eager_lse.view(torch.int32)) + assert int(ps_workspace.cnt.abs().sum()) == 0 + + +@needs_gfx950 +@pytest.mark.parametrize( + "kv_lens", [[1152, 900, 64, 1152], [8320, 200], [100, 60, 128] * 50] +) +def test_v4_nm_ps_matches_split_kernel(ps_workspace, kv_lens): + """Agrees with mla_decode_fwd_v4_nm (non-persistent kernel + planned split).""" + inp = _ps_inputs(kv_lens, seed=8) + n = len(kv_lens) + out = _ps_call(inp, ps_workspace, return_lse=False) + ref = torch.empty(n, _PS_HEADS, V_HEAD_DIM, dtype=dtypes.bf16, device="cuda") + plan = aiter.mla.get_mla_v4_nm_split_plan(n, _PS_HEADS, max(kv_lens)) + logits, _ = aiter.mla.mla_decode_fwd_v4_nm( + inp["q_packed"], + inp["q_rope"], + inp["kv_packed"], + inp["kv_rope"], + ref, + torch.arange(n + 1, dtype=torch.int32, device="cuda"), + inp["kv_indptr"], + inp["kv_page_indices"], + 1, + sink=inp["sink"], + **plan._asdict(), + ) + if plan.num_kv_splits == 1: + ref = logits[:, 0] + torch.cuda.synchronize() + assert _cos_diff(out, ref) < 1e-5 + err = checkAllclose( + ref.float(), + out.float(), + rtol=3e-2, + atol=3e-2, + tol_err_ratio=0.02, + msg="v4 nm ps vs split kernel", + ) + assert float(err or 0) < 0.02 + + +@needs_gfx950 +def test_v4_nm_ps_rejects_bad_args(ps_workspace): + inp = _ps_inputs([64, 64], seed=9) + with pytest.raises(RuntimeError): # 64 heads + aiter.mla.mla_decode_fwd_v4_nm_ps( + inp["q_packed"][:, :64].contiguous(), + inp["q_rope"][:, :64].contiguous(), + inp["kv_packed"], + inp["kv_rope"], + inp["kv_indptr"], + inp["kv_page_indices"], + inp["sink"], + ps_workspace, + ) + with pytest.raises(RuntimeError): # kv_indptr shorter than N+1 + _ps_call(inp, ps_workspace, kv_indptr=inp["kv_indptr"][:2]) + with pytest.raises(ValueError): + aiter.mla.get_mla_v4_nm_ps_workspace(num_partitions=1025) + + if __name__ == "__main__": import argparse import itertools From e447b8bb7f5f63864fdc09049e9da8da5a6b2c50 Mon Sep 17 00:00:00 2001 From: yihonglie Date: Tue, 29 Sep 2026 09:56:45 -0500 Subject: [PATCH 05/13] [MLA v4 nm] Add the generator for the persistent decode code object mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co is now reproducible from sources in the repo: nm_ps_src/gen.py patches the disassembly of the #5195 decode kernel (asm/rt.s) into the persistent kernel, splices in the in-kernel split planner and last-arriver merge (splice/*.hip, compiled with hipcc -S), renames the kernel symbol, assembles for gfx950 and can check the sha256 against the shipped .co. README.md documents provenance, rebuild steps and the kernarg layout. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie --- hsa/gfx950/mla_v4/nm_ps_src/README.md | 142 + hsa/gfx950/mla_v4/nm_ps_src/asm/align.py | 65 + hsa/gfx950/mla_v4/nm_ps_src/asm/patch.py | 437 + hsa/gfx950/mla_v4/nm_ps_src/asm/rt.s | 15413 ++++++++++++++++ hsa/gfx950/mla_v4/nm_ps_src/gen.py | 154 + hsa/gfx950/mla_v4/nm_ps_src/splice/fin.hip | 273 + .../mla_v4/nm_ps_src/splice/hca_common.hip | 100 + hsa/gfx950/mla_v4/nm_ps_src/splice/pre.hip | 172 + hsa/gfx950/mla_v4/nm_ps_src/splice/splice.py | 95 + 9 files changed, 16851 insertions(+) create mode 100644 hsa/gfx950/mla_v4/nm_ps_src/README.md create mode 100644 hsa/gfx950/mla_v4/nm_ps_src/asm/align.py create mode 100644 hsa/gfx950/mla_v4/nm_ps_src/asm/patch.py create mode 100644 hsa/gfx950/mla_v4/nm_ps_src/asm/rt.s create mode 100644 hsa/gfx950/mla_v4/nm_ps_src/gen.py create mode 100644 hsa/gfx950/mla_v4/nm_ps_src/splice/fin.hip create mode 100644 hsa/gfx950/mla_v4/nm_ps_src/splice/hca_common.hip create mode 100644 hsa/gfx950/mla_v4/nm_ps_src/splice/pre.hip create mode 100644 hsa/gfx950/mla_v4/nm_ps_src/splice/splice.py diff --git a/hsa/gfx950/mla_v4/nm_ps_src/README.md b/hsa/gfx950/mla_v4/nm_ps_src/README.md new file mode 100644 index 00000000000..b6c25bd1406 --- /dev/null +++ b/hsa/gfx950/mla_v4/nm_ps_src/README.md @@ -0,0 +1,142 @@ + +# mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps: generator sources + +This directory regenerates the AITER code object +`hsa/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co` (kernel symbol +`_ZN5aiter39mla_a8w8_qh64_qseqlen1_gqaratio64_nm_psE`) from sources. It is the persistent +DeepSeek-V4 HCA decode kernel (fp8 KV "nm" layout, 128 query heads, qseqlen 1) for gfx950. + +## Provenance + +* **Base.** The kernel is derived from the AITER #5195 ASM kernel + `hsa/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co` + (`_ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE`). `asm/rt.s` is a disassembly of that code object, made + re-assemblable (llvm-objdump output plus the original kernel descriptor bytes and metadata). Assembled, it gives + the same `.text` and kernel descriptor (`.rodata`) as the #5195 `.co` (`gen.py --verify-rt` checks this). `rt.s` + is kept as generated: `asm/patch.py` finds some of its patch sites by line number, so the file carries no license + header, and `gen.py` checks its sha256 before it uses the file. +* **Persistent descriptor loop.** `asm/patch.py` rewrites the disassembly with scripts only; nobody edits the + output by hand. The grid becomes `(2P, 1, 1)`. Each workgroup owns one partition `p` and one head group `hg`, + and walks the rows `begin_row..end_row` of its descriptor. It runs one tile range `[tb, te)` of a row per pass, + re-entering the ASM's own prologue each time. The patches retarget the ASM's split-partial epilogue to the slot + layout of `o_acc` / `lse_acc`, apply the sink once, write the natural-log LSE of unsplit rows, and relax some + waits. Two never-executed `s_nop` pads keep the hot loop code at its original byte offset mod 256 + (`asm/align.py`). The docstring of `asm/patch.py` lists every patch. +* **In-kernel split planner.** `splice/pre.hip` holds the planner, written in HIP. It runs the closed-form cost-line + planner, a 64-way wave-cooperative boundary search, and an automatic split or no-split choice. The source is + compiled with `hipcc -S`, and `splice/splice.py` extracts the kernel body from that assembly. `asm/patch.py` + then splices the body into the kernel entry, so no separate planner launch is needed. Each workgroup publishes + its descriptor to `desc[p]`. +* **Fused last-arriver combine.** `splice/fin.hip` is also written in HIP and spliced the same way, at the end of + every split piece. Each piece counts its tiles into an agent-scope atomic counter (`cnt`). The piece that + completes a row merges all of that row's partials, adds the sink, and writes `out` / `lse_out`. Wide rows are + merged in two levels. The partial stores use sc1 (agent-scope write-through), so no separate combine launch is + needed, and the counters return to zero at the end of every completed call. + +## Layout + +| path | role | +|---|---| +| `gen.py` | the only build entry point (python3 stdlib only) | +| `asm/rt.s` | re-assemblable disassembly of the #5195 ASM (input, not edited) | +| `asm/patch.py` | rt.s -> persistent kernel (library, called by gen.py) | +| `asm/align.py` | alignment pads (assembles trial passes, reads label offsets with llvm-readelf) | +| `splice/splice.py` | `hipcc -S` + extraction of one kernel body as a splice block | +| `splice/pre.hip` | in-kernel split planner (`t8pre`) | +| `splice/fin.hip` | fused last-arriver combine (`t8fin`) | +| `splice/hca_common.hip` | shared definitions: cost line, `Desc`, wave-cooperative search, WG map | + +## Build and verify + +Run inside a ROCm container, for example with the toolchain listed below: + +```bash +python3 gen.py --out /tmp/x.co \ + --check /hsa/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co +# optional: keep the intermediates / check the rt.s provenance +python3 gen.py --out /tmp/x.co --keep /tmp/nm_ps_build \ + --verify-rt /hsa/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co +``` + +`gen.py` takes these steps: + +1. Check the sha256 of `asm/rt.s`. +2. Compile `splice/pre.hip` and `splice/fin.hip` with + `hipcc --offload-arch=gfx950 -O3 --offload-device-only -std=c++20 -w -S`. +3. Run `asm/patch.py`, including the alignment passes. +4. Rename the kernel symbol from `_ZN5aiter36..._nmE` to `_ZN5aiter39..._nm_psE`. This covers every occurrence: + the function symbol, `.kd`, and the metadata `.name` / `.symbol`. The Itanium length prefix is recomputed + (36 -> 39). +5. Assemble with `clang -x assembler -target amdgcn-amd-amdhsa -mcpu=gfx950 -c`. +6. Link with `ld.lld -shared`. +7. With `--check`, compare the sha256 against the given file. On a mismatch it exits with status 1. + +Tools are taken from `$ROCM_PATH` (default `/opt/rocm`): `llvm/bin/{clang,ld.lld,llvm-readelf,llvm-objcopy}` and +`bin/hipcc`. The output is byte-identical only with the same toolchain. `hipcc` codegen and `clang` / `lld` object +layout can change between ROCm releases. + +Toolchain used for the shipped `.co` (container image `rocm/atom-dev:nightly_202609161445`, `/opt/rocm` -> +`/opt/rocm-7.2.4`): + +``` +$ /opt/rocm/llvm/bin/clang --version +AMD clang version 22.0.0git (https://github.com/RadeonOpenCompute/llvm-project roc-7.2.4 26084 f58b06dce1f9c15707c5f808fd002e18c2accf7e) +Target: x86_64-unknown-linux-gnu +Thread model: posix +InstalledDir: /opt/rocm-7.2.4/lib/llvm/bin +$ hipcc --version +HIP version: 7.2.53211-97f5574fe2 +AMD clang version 22.0.0git (https://github.com/RadeonOpenCompute/llvm-project roc-7.2.4 26084 f58b06dce1f9c15707c5f808fd002e18c2accf7e) +$ /opt/rocm/llvm/bin/ld.lld --version +AMD LLD 22.0.0 (... f58b06dce1f9c15707c5f808fd002e18c2accf7e) (compatible with GNU linkers) +``` + +Expected sha256 of the output: +`b1a1944b57d006aba1fa91fff4d1ddc87029bef94d8bea066e59a4f7c5ab342c`. + +## Launch and kernarg ABI + +* Grid `(2P, 1, 1)` and block `256`, with no dynamic LDS. `P` is the number of partitions (`P <= 1024`, and + `P % 8 == 0` selects the XCD-aware WG map). Rows: `N <= 65536`. +* The kernarg block is 336 B: 21 slots of 16 B. Offsets are in bytes. Slots the kernel does not read are passed as + the values shown. + +| offset | type | content | +|---|---|---| +| 0x00 | ptr | `o_acc` fp32 `[2P][128][512]`: split-piece partial O (workspace) | +| 0x08 | ptr | `out` bf16 `[N][128][512]` | +| 0x10 | ptr | `lse_acc` fp32 `[2P][128]`: split-piece partial LSE, log2 domain, no sink (workspace) | +| 0x18 | ptr | `lse_out` fp32 `[N][128]`, natural log, sink included; 0 = not written | +| 0x20 | ptr | `q_packed` (Q NoPE, fp8 packed) | +| 0x30 | ptr | `kv_packed` (KV NoPE fp8 + scales, 512 B per token) | +| 0x40 | ptr | `kv_indptr` int32 `[N+1]` | +| 0x50 | ptr | `kv_page_indices` int32 | +| 0x60 | ptr | `desc` int32 `[P][8]`: per-partition descriptor, written by the kernel (workspace) | +| 0x68 | ptr | 0 (unused) | +| 0x70 | f32 | softmax scale `1/sqrt(512)` | +| 0x80 | u32 | 128 (gqa ratio) | +| 0x90 | u32 | 1 (num_kv_splits) | +| 0x98 | ptr | `cnt` uint32 `[2*65536 + 16*512 + 4*1024]`, zero at rest (row / group counters of the fused combine) | +| 0xa0 | u32 | `P` | +| 0xa4 | u32 | `N` (rows) | +| 0xb0 | u32 | planner config `F | MT << 8` (default `6 | 1 << 8`) | +| 0xb8 | ptr | `span` int32 `[P][2]` (passed by the host wrapper; not read by this build) | +| 0xc0 | u32 | 0 | +| 0xd0 | ptr | `qo_indptr` = int32 `arange(>= N+1)` | +| 0xe0 | ptr | `split_indptr` = int32 `arange(>= N+1)` (same buffer) | +| 0xf0 | u32 | 1 (bf16 output) | +| 0x100 | ptr | `q_rope` (bf16) | +| 0x110 | ptr | `kv_rope` (bf16) | +| 0x120 | ptr | `sink` fp32 `[128]` | +| 0x130 | ptr | 0 (valid_split) | +| 0x140 | u32 | 0 (use_valid_split) | + +Workspace rules: +* One workspace (`o_acc`, `lse_acc`, `desc`, `cnt`) can serve any number of calls that are ordered with respect to + each other. +* Calls that can run concurrently need separate workspaces. +* Rows with `K = 0` are not written. +* A kernel that aborts can leave `cnt` non-zero. Zero it again before the next call. diff --git a/hsa/gfx950/mla_v4/nm_ps_src/asm/align.py b/hsa/gfx950/mla_v4/nm_ps_src/asm/align.py new file mode 100644 index 00000000000..e47f34a79f3 --- /dev/null +++ b/hsa/gfx950/mla_v4/nm_ps_src/asm/align.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""Alignment-preserving pads for script-patched copies of rt.s. Two never-executed s_nop pads (@@PADA@@ before the +loop code, @@PADB@@ after the epilogue; each placed after an unconditional branch or s_endpgm) absorb inserted bytes so +the hot code keeps its original byte offset mod 256: loop copies label_1F54/3D74/5B68, drain label_795C, tail tile +label_8EFC, rescale subroutine label_9DB4. Offsets are read from the symbol table of an assembled pass. +""" + +import re +import subprocess +import tempfile + +A = 256 +PRE = ("label_1F54", "label_3D74", "label_5B68", "label_795C") +POST = ("label_8EFC", "label_9DB4") + + +def offsets(text, clang, readelf): + with tempfile.TemporaryDirectory() as d: + open(f"{d}/x.s", "w").write(text) + subprocess.run( + [ + clang, + "-x", + "assembler", + "-target", + "amdgcn-amd-amdhsa", + "-mcpu=gfx950", + "-c", + f"{d}/x.s", + "-o", + f"{d}/x.o", + ], + check=True, + ) + r = subprocess.run( + [readelf, "-s", f"{d}/x.o"], capture_output=True, text=True, check=True + ).stdout + return { + m[2]: int(m[1], 16) + for m in re.finditer( + r"^\s*\d+:\s+([0-9a-f]+)\s+0\s+NOTYPE\s+LOCAL\s+DEFAULT\s+\d+\s+(label_\w+)", + r, + re.M, + ) + } + + +def render(body, pads): + for k, n in pads.items(): + body = body.replace(k, "\n".join(["\ts_nop 0"] * (n // 4)) if n else "") + return body + + +def align(src_text, body, clang, readelf): + pads = {"@@PADA@@": 0, "@@PADB@@": 0} + orig = offsets(src_text, clang, readelf) + o = offsets(render(body, pads), clang, readelf) + pads["@@PADA@@"] = (-(o["label_1F54"] - orig["label_1F54"])) % A + o = offsets(render(body, pads), clang, readelf) + pads["@@PADB@@"] = (-(o["label_8EFC"] - orig["label_8EFC"])) % A + o = offsets(render(body, pads), clang, readelf) + for lab in PRE + POST: + assert (o[lab] - orig[lab]) % A == 0, (lab, o[lab], orig[lab]) + return render(body, pads), pads diff --git a/hsa/gfx950/mla_v4/nm_ps_src/asm/patch.py b/hsa/gfx950/mla_v4/nm_ps_src/asm/patch.py new file mode 100644 index 00000000000..44b003ee190 --- /dev/null +++ b/hsa/gfx950/mla_v4/nm_ps_src/asm/patch.py @@ -0,0 +1,437 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""rt.s (disassembly of the AITER #5195 v4 nm ASM, reassembles to the same .text / kernel descriptor) -> the persistent +HCA decode kernel. Script-driven rewrites only; the output is never hand-edited. Library: patch(src_text, ...) -> text. +The kernel symbol is left as in rt.s; gen.py renames it afterwards. + +Launch: grid (2P,1,1), 256 threads. WG w -> (p, hg) (xcd = w&7, loc = w>>3, p = xcd*(P/8) + loc/2, hg = loc&1 when +P%8 == 0, else p = w/2, hg = w&1). s2 := hg for the whole kernel. Kernarg: see README.md (21 x 16 B slots). +Entry (splice/pre.hip, the in-kernel split planner, spliced into the prologue): every WG plans its own partition p +(split and no-split plans from one search; the no-split plan is chosen when its max partition load stays within +NS_WANY of the split plan's), publishes desc[p] (hg == 0 WG) and leaves (begin_row, end_row, begin_tile, end_tile, +flags) in a243..a247, per / Pa / org in a248..a250. s0,s1,s2 are saved in a240..a242 around the splice. +Walk (per WG): rows r = begin_row .. end_row; piece (r, tb, te): + tb = r==begin_row ? begin_tile : 0, te = r==end_row ? end_tile : 2^20 (clamped by K), + split = (r==begin_row && flags&1) || (r==end_row && flags&2), slot = 2p + (r != begin_row). + Rows with K = 0 inside the walk are skipped without entering the ASM (they are left unwritten). + Each piece re-enters the ASM's own prologue with s3=r, s4=0, v0=thread id; at the ASM's s_endpgm the split piece + runs the fused combine (splice/fin.hip: last-arriver merge through agent-scope atomic counters at kernarg 0x98), + then all 4 waves barrier, r++, loop. Empty partition: begin_row=N > end_row=-1 -> exit. +Patches of the ASM body: + P1 chunk ([tb,te) contiguous tiles): s68 += 32tb, s71 = min(K-32tb, 32(te-tb)), s100=0 (no round-robin offset), + s65=s67, s94=4*s67 (index stride of consecutive tiles). + P2 s82 := split ? 2 : 1 (selects partial fp32 path vs bf16 path in the epilogue). + P3 sink: the ASM's "split 0 owns the sink" tests (s_cmp_eq_u32 s4, 0 in the epilogue) become "s82 == 1": sink + applied exactly once, in the unsplit epilogue; split pieces carry none (the fused combine adds it). + P4 partial LSE in the log2 domain: split pieces overwrite lse = m*s5 + log2(ln). + P5 partial store retargeted to the slot layout: at label_8068 o_acc base += slot*0x40000, lse_acc base += slot*512, + s89 (row) := 0 -> o_acc[slot][hg*64+h][:], lse_acc[slot][hg*64+h] (the ASM's own lane addressing). + P6 bf16 path (label_86D0): base = out (kernarg 0x08), row = s89 = r; plus (if lse_out != 0) + lse_out[r][hg*64+16w+ln%16] = lse (natural log, sink included). + P7 reloc: epilogue O-transpose LDS scratch 0x0..0x87ff -> 0x1f000..0x277ff. + sc1: the partial path's stores (label_8068 .. label_86CC) get sc1 (agent-scope write-through, read by fin.hip). + nowait / nowait2: epilogue store waits relaxed to lgkmcnt-only (stores need no completion wait before the next + piece; the fused combine waits vmcnt(0) itself before its counter atomic). + kfast: the entry touches the kernarg lines the ASM prologue reads (0x00, 0xc0, 0x100, 0x140) so its s_loads hit the + K$; the prologue's dependent indptr round trip becomes moves (kv_indptr[r], [r+1] from the wrapper's load in + s84/s85; split_indptr / qo_indptr = arange contract -> r, r+1). + kv0early: Q staging moved out of the KV ring (nope 0x2000.. -> 0x1f000.., rope 0x12800.. -> 0x17000..) so the KV + tile-0 loads (label_0A6C block) move up to right after the page indices land. + align: two never-executed s_nop pads keep the hot code at its original byte offset mod 256 (asm/align.py). +Registers used by the wrapper (never touched by rt.s): s30 (r), s31 (slot or -1), s38/s39 (tb/te), s42 s43 s84 s85 +(scratch), s101 (p), v248 (thread id), v249 (lse address), v250..v254 (readfirstlane temps), a230..a250. +""" + +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +sys.path.insert(0, os.path.join(HERE, "..", "splice")) +import align as AL # noqa: E402 +import splice as SP # noqa: E402 + +K = "_ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE" # kernel symbol of rt.s +KF_MAP = { + "s_load_dword s68, s[28:29], 0x0": "\ts_mov_b32 s68, s84", + "s_load_dword s69, s[28:29], 0x4": "\ts_mov_b32 s69, s85", + "s_load_dword s93, s[40:41], 0x0": "\ts_mov_b32 s93, s3", + "s_load_dword s92, s[40:41], 0x4": "\ts_add_u32 s92, s3, 1", + "s_load_dword s89, s[36:37], 0x0": "\ts_mov_b32 s89, s3", + "s_load_dword s90, s[36:37], 0x4": "\ts_add_u32 s90, s3, 1", +} + + +def patch(src_text, clang, readelf, hipcc, workdir): + """src_text: rt.s. Returns (patched text, marks, pads).""" + _pre, _kd = SP.body( + hipcc, + os.path.join(SP.HERE, "pre.hip"), + "t8pre", + "T8PRE", + workdir, + vmax=240, + smax=100, + allow_lds=True, + ) # uses <= 4.1 KB of LDS; ends with a barrier before the ASM + assert _kd["group_segment_fixed_size"] <= 8192, _kd["group_segment_fixed_size"] + _fin, _kf = SP.body( + hipcc, + os.path.join(SP.HERE, "fin.hip"), + "t8fin", + "T8FIN", + workdir, + vmax=240, + smax=100, + allow_lds=True, + ) + assert _kf["group_segment_fixed_size"] <= 0x1F000 + + ENTRY = [ + "\tv_mov_b32_e32 v248, v0", + "\tv_accvgpr_write_b32 a240, s0", + "\tv_accvgpr_write_b32 a241, s1", + "\tv_accvgpr_write_b32 a242, s2", + _pre, + "T8PRE_RET:", + "\ts_waitcnt vmcnt(0) lgkmcnt(0)", + "\ts_nop 4", + "\tv_accvgpr_read_b32 v250, a240", + "\tv_accvgpr_read_b32 v251, a241", + "\tv_accvgpr_read_b32 v252, a242", + "\ts_nop 4", + "\tv_readfirstlane_b32 s0, v250", + "\tv_readfirstlane_b32 s1, v251", + "\tv_readfirstlane_b32 s2, v252", + "\ts_load_dword s42, s[0:1], 0xa0", # P + "\ts_load_dword s43, s[0:1], 0x0", + "\ts_load_dword s84, s[0:1], 0xc0", + "\ts_load_dword s85, s[0:1], 0x100", + "\ts_load_dword s101, s[0:1], 0x140", # kfast: warm the K$ lines of the ASM prologue + "\ts_waitcnt lgkmcnt(0)", + "\ts_and_b32 s43, s42, 7", + "\ts_cmp_eq_u32 s43, 0", + "\ts_cbranch_scc0 T8_MAP_PLAIN", + "\ts_and_b32 s84, s2, 7", + "\ts_lshr_b32 s85, s2, 3", + "\ts_lshr_b32 s43, s42, 3", + "\ts_mul_i32 s84, s84, s43", + "\ts_lshr_b32 s43, s85, 1", + "\ts_add_u32 s101, s84, s43", + "\ts_and_b32 s2, s85, 1", + "\ts_branch T8_MAPPED", + "T8_MAP_PLAIN:", + "\ts_lshr_b32 s101, s2, 1", + "\ts_and_b32 s2, s2, 1", + "T8_MAPPED:", + "\tv_accvgpr_read_b32 v250, a243", + "\ts_nop 4", + "\tv_readfirstlane_b32 s30, v250", # r = begin_row + ] + PIECE = [ + "T8_PIECE_START:", + "\ts_and_b32 s1, s1, 0xffff", + "\ts_load_dwordx2 s[42:43], s[0:1], 0x60", + "\ts_load_dwordx2 s[76:77], s[0:1], 0x40", + "\ts_load_dwordx2 s[74:75], s[0:1], 0xa0", + "\ts_waitcnt lgkmcnt(0)", + "\tv_accvgpr_read_b32 v250, a243", + "\tv_accvgpr_read_b32 v251, a244", + "\tv_accvgpr_read_b32 v252, a245", + "\tv_accvgpr_read_b32 v253, a246", + "\tv_accvgpr_read_b32 v254, a247", + "\ts_nop 4", + "\tv_readfirstlane_b32 s64, v250", + "\tv_readfirstlane_b32 s65, v251", + "\tv_readfirstlane_b32 s66, v252", + "\tv_readfirstlane_b32 s67, v253", + "\tv_readfirstlane_b32 s68, v254", + "\ts_add_u32 s85, s75, -1", + "\ts_min_i32 s85, s30, s85", + "\ts_max_i32 s85, s85, 0", + "\ts_lshl_b32 s85, s85, 2", + "\ts_load_dwordx2 s[78:79], s[76:77], s85", # kv_indptr[min(r, N-1)], kv_indptr[..+1] + "\ts_waitcnt lgkmcnt(0)", + # s64 begin_row s65 end_row s66 begin_tile s67 end_tile s68 flags + "\ts_cmp_gt_i32 s30, s65", + "\ts_cbranch_scc1 T8_DONE", + # K = 0 row inside the walk (e.g. bucket padding): skip without entering the ASM (no LDS use, no barrier) + "\ts_cmp_eq_u32 s78, s79", + "\ts_cbranch_scc0 T8_KNZ", + "\ts_add_u32 s30, s30, 1", + "\ts_branch T8_PIECE_START", + "T8_KNZ:", + "\tv_accvgpr_write_b32 a236, s78", + "\tv_accvgpr_write_b32 a237, s79", + "\ts_and_b32 s69, s68, 1", + "\ts_bfe_u32 s70, s68, 0x10001", + "\ts_cmp_eq_u32 s30, s64", + "\ts_cselect_b32 s38, s66, 0", + "\ts_cselect_b32 s71, s69, 0", + "\ts_cselect_b32 s72, 0, 1", + "\ts_cmp_eq_u32 s30, s65", + "\ts_cselect_b32 s39, s67, 0x100000", + "\ts_cselect_b32 s73, s70, 0", + "\ts_or_b32 s71, s71, s73", + "\ts_lshl_b32 s73, s101, 1", + "\ts_add_u32 s72, s73, s72", + "\ts_cmp_eq_u32 s71, 0", + "\ts_cselect_b32 s31, -1, s72", + "\ts_mov_b32 s3, s30", + "\ts_mov_b32 s4, 0", + "\ts_mov_b32 s84, s78", + "\ts_mov_b32 s85, s79", + "\ts_mov_b32 s88, 0", + "\ts_mov_b32 s70, 0", + "\tv_mov_b32_e32 v0, v248", + "\ts_branch T8_PADA_END", + "@@PADA@@", + "T8_PADA_END:", # alignment pad (skipped), see align.py + ] + EPI_PART = [ # label_8068: o_acc / lse_acc slot bases + "\ts_mul_i32 s75, s31, 0x40000", + "\ts_add_u32 s8, s8, s75", + "\ts_addc_u32 s9, s9, 0", + "\ts_lshl_b32 s75, s31, 9", + "\ts_add_u32 s12, s12, s75", + "\ts_addc_u32 s13, s13, 0", + "\ts_mov_b32 s89, 0", + ] + EPI_BF16 = [ # label_86D0: out base + optional natural-log LSE of the unsplit row + "\ts_load_dwordx2 s[8:9], s[0:1], 0x8", + "\ts_load_dwordx2 s[42:43], s[0:1], 0x18", + "\ts_waitcnt lgkmcnt(0)", + "\ts_and_b32 s9, s9, 0xffff", + "\ts_or_b32 s9, s9, 0x40000", + "\ts_cmp_eq_u64 s[42:43], 0", + "\ts_cbranch_scc1 T8_NOLSE", + "\ts_lshl_b32 s84, s89, 9", + "\ts_lshl_b32 s85, s2, 8", + "\ts_add_u32 s84, s84, s85", + "\ts_lshl_b32 s85, s7, 6", + "\ts_add_u32 s84, s84, s85", + "\ts_add_u32 s42, s42, s84", + "\ts_addc_u32 s43, s43, 0", + "\tv_and_b32_e32 v249, 15, v0", + "\tv_lshlrev_b32_e32 v249, 2, v249", + "\tglobal_store_dword v249, v33, s[42:43]", + "T8_NOLSE:", + ] + END = [ + "\ts_mov_b64 exec, -1", + # fused combine (split pieces only) + "\ts_cmp_eq_u32 s31, -1", + "\ts_cbranch_scc1 T8FIN_SKIP", + "\tv_accvgpr_write_b32 a230, s30", + "\tv_accvgpr_write_b32 a231, s31", + "\tv_accvgpr_write_b32 a232, s101", + "\tv_accvgpr_write_b32 a233, s2", + "\tv_accvgpr_write_b32 a234, s38", + "\tv_accvgpr_write_b32 a235, s39", + "\tv_mov_b32_e32 v0, v248", + "\ts_nop 4", + _fin, + "T8FIN_RET:", + "\ts_waitcnt vmcnt(0) lgkmcnt(0)", + "\ts_mov_b64 exec, -1", + "\ts_nop 4", + "\tv_accvgpr_read_b32 v250, a240", + "\tv_accvgpr_read_b32 v251, a241", + "\tv_accvgpr_read_b32 v252, a233", + "\tv_accvgpr_read_b32 v253, a230", + "\tv_accvgpr_read_b32 v254, a232", + "\ts_nop 4", + "\tv_readfirstlane_b32 s0, v250", + "\tv_readfirstlane_b32 s1, v251", + "\tv_readfirstlane_b32 s2, v252", + "\tv_readfirstlane_b32 s30, v253", + "\tv_readfirstlane_b32 s101, v254", + "T8FIN_SKIP:", + "\ts_barrier", + "\ts_add_u32 s30, s30, 1", + "\ts_branch T8_PIECE_START", + "@@PADB@@", + "T8_DONE:", + "\ts_endpgm", + ] + + out, done = [], {} + + def mark(k): + done[k] = done.get(k, 0) + 1 + + lines = src_text.split("\n") + _sa, _sb = lines.index("label_8068:"), lines.index("label_86CC:") + # kv0early: Q-wait position and the label_0A6C KV tile-0 block + _qa = next(i for i, ln in enumerate(lines) if ln.strip() == "s_waitcnt vmcnt(2)") + _k0 = lines.index("label_0A6C:") + _k1 = next( + i + for i in range(_k0, _k0 + 80) + if lines[i].strip() == "s_waitcnt lgkmcnt(0)" + and lines[i + 1].strip() == "s_barrier" + ) + KV0BLK = lines[_k0 + 1 : _k1] + assert ( + lines[_qa + 1].strip() == "s_barrier" + and any("global_load_lds_dwordx4" in x for x in KV0BLK) + and len(KV0BLK) == 50 + ), len(KV0BLK) + for i, ln in enumerate(lines): + t = ln.strip() + if _sa < i < _sb and t.startswith("buffer_store"): + ln = ln + " sc1" + t = ln.strip() + mark("sc1") + if ln == f"{K}:": + out += [ln] + ENTRY + PIECE + mark("entry") + continue + # P1 chunk + if t == "s_sub_u32 s71, s69, s68": + out += [ + ln, + "\ts_lshl_b32 s85, s38, 5", + "\ts_add_u32 s68, s68, s85", + "\ts_sub_u32 s71, s71, s85", + "\ts_sub_u32 s84, s39, s38", + "\ts_lshl_b32 s84, s84, 5", + "\ts_min_u32 s71, s71, s84", + ] + mark("P1a") + continue + if t == "s_mul_i32 s100, s4, s67": + out += [ln, "\ts_mov_b32 s100, 0"] + mark("P1b") + continue + if t == "s_mul_i32 s65, s82, s67": + out += ["\ts_mov_b32 s65, s67"] + mark("P1c") + continue + if t == "s_mul_i32 s94, s94, s82": + out += [ln, "\ts_mul_i32 s94, s67, 4"] + mark("P1d") + continue + # P2 + if t == "s_mov_b32 s82, s98": + out += [ln, "\ts_cmp_eq_u32 s31, -1", "\ts_cselect_b32 s82, 1, 2"] + mark("P2") + continue + # P3 (both "s_cmp_eq_u32 s4, 0" inside the epilogue region) + if t == "s_cmp_eq_u32 s4, 0" and i > 4000: + out += ["\ts_cmp_eq_u32 s82, 1"] + mark("P3") + continue + # P4 + if t == "v_fma_f32 v33, v27, s6, v26": + out += [ + ln, + "\ts_cmp_eq_u32 s82, 1", + "\ts_cbranch_scc1 T8_LSE_NAT", + "\tv_fma_f32 v33, v45, s5, v27", + "T8_LSE_NAT:", + ] + mark("P4") + continue + # P5 / P6 + if ln == "label_8068:": + out += [ln] + EPI_PART + mark("P5") + continue + if ln == "label_86D0:": + out += [ln] + EPI_BF16 + mark("P6") + continue + # P7 reloc + if ( + t in ("v_lshlrev_b32_e32 v16, 2, v16", "v_lshlrev_b32_e32 v10, 2, v10") + and i > 4000 + ): + r = t.split()[1].rstrip(",") + out += [ln, f"\tv_add_u32_e32 {r}, 0x1f000, {r}"] + mark("P7") + continue + # kv0early + if i < 520: + if t == "s_add_u32 s56, 0x2000, s75" and i < 300: + out += ["\ts_add_u32 s56, 0x1f000, s75"] + mark("kv0e") + continue + if t == "v_add_u32_e32 v16, 0x12800, v16" and i < 330: + out += ["\tv_add_u32_e32 v16, 0x17000, v16"] + mark("kv0e") + continue + if t == "s_add_u32 m0, 0x12800, s75" and i < 330: + out += ["\ts_add_u32 m0, 0x17000, s75"] + mark("kv0e") + continue + if i == _qa: + out += ["\ts_waitcnt vmcnt(10)"] + KV0BLK + ["\ts_waitcnt vmcnt(8)"] + mark("kv0e") + continue + if i == _qa + 1: + out += [ + ln, + "\tv_add_u32_e32 v10, 0x1d000, v10", + "\tv_add_u32_e32 v34, 0x1d000, v34", + ] + mark("kv0e") + continue + if t == "s_waitcnt vmcnt(0)" and lines[i - 1] == "label_09EC:": + out += ["\ts_waitcnt vmcnt(6)"] + mark("kv0e") + continue + if _k0 < i < _k1: + mark("kv0drop") + continue + # kfast + if i < 80 and t in KF_MAP: + out += [KF_MAP[t]] + mark("kfast") + continue + # nowait2 / nowait + if ( + t == "s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)" + and lines[i - 1] == "label_8EF4:" + ): + out += ["\ts_waitcnt lgkmcnt(0)"] + mark("nowait2") + continue + if t == "s_waitcnt vmcnt(0) lgkmcnt(0)" and 4641 < i < 5431: + out += ["\ts_waitcnt lgkmcnt(0)"] + mark("nowait") + continue + if t == "s_endpgm": + out += END + mark("end") + continue + out.append(ln) + exp = dict( + entry=1, + P1a=1, + P1b=1, + P1c=1, + P1d=1, + P2=1, + P3=2, + P4=1, + P5=1, + P6=1, + end=1, + sc1=33, + nowait=2, + nowait2=1, + kfast=6, + kv0e=6, + kv0drop=50, + ) + exp["P7"] = sum( + 1 + for i, ln in enumerate(lines) + if ln.strip() + in ("v_lshlrev_b32_e32 v16, 2, v16", "v_lshlrev_b32_e32 v10, 2, v10") + and i > 4000 + ) + assert done == exp, (done, exp) + text, pads = AL.align(src_text, "\n".join(out), clang, readelf) + return text, done, pads diff --git a/hsa/gfx950/mla_v4/nm_ps_src/asm/rt.s b/hsa/gfx950/mla_v4/nm_ps_src/asm/rt.s new file mode 100644 index 00000000000..672bd13ec0c --- /dev/null +++ b/hsa/gfx950/mla_v4/nm_ps_src/asm/rt.s @@ -0,0 +1,15413 @@ +.amdgcn_target "amdgcn-amd-amdhsa--gfx950" +.amdhsa_code_object_version 5 +.text +.globl _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE +.protected _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE +.p2align 8 +.type _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE,@function +_ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE: + s_and_b32 s1, s1, 0xffff + s_mov_b32 s70, 0 + s_mov_b32 s73, 0x7060302 + s_mov_b32 s74, 0x5040100 + s_mov_b32 s78, 0 + s_load_dwordx2 s[8:9], s[0:1], 0x0 + s_load_dwordx2 s[12:13], s[0:1], 0x10 + s_load_dwordx2 s[16:17], s[0:1], 0x20 + s_load_dwordx2 s[20:21], s[0:1], 0x30 + s_load_dwordx2 s[28:29], s[0:1], 0x40 + s_load_dwordx2 s[24:25], s[0:1], 0x50 + s_load_dword s79, s[0:1], 0x70 + s_load_dword s80, s[0:1], 0x80 + s_load_dword s81, s[0:1], 0x90 + s_load_dword s86, s[0:1], 0xc0 + s_load_dwordx2 s[36:37], s[0:1], 0xd0 + s_load_dwordx2 s[40:41], s[0:1], 0xe0 + s_load_dword s83, s[0:1], 0xf0 + s_load_dwordx2 s[44:45], s[0:1], 0x100 + s_load_dwordx2 s[48:49], s[0:1], 0x110 + s_load_dwordx2 s[52:53], s[0:1], 0x120 + s_load_dwordx2 s[32:33], s[0:1], 0x130 + s_load_dword s99, s[0:1], 0x140 + v_lshrrev_b32_e32 v1, 10, v0 + v_lshrrev_b32_e32 v2, 10, v1 + v_and_b32_e32 v2, 0x3ff, v2 + v_and_b32_e32 v1, 0x3ff, v1 + v_and_b32_e32 v0, 0x3ff, v0 + v_lshrrev_b32_e32 v3, 6, v0 + v_and_b32_e32 v0, 63, v0 + s_mov_b32 s2, s2 + s_mov_b32 s3, s3 + s_mov_b32 s4, s4 + v_readfirstlane_b32 s7, v3 + s_waitcnt lgkmcnt(0) + s_mul_i32 s75, s3, 4 + s_and_b32 s29, s29, 0xffff + s_add_u32 s28, s75, s28 + s_addc_u32 s29, 0, s29 + s_load_dword s68, s[28:29], 0x0 + s_load_dword s69, s[28:29], 0x4 + s_mul_i32 s75, s3, 4 + s_and_b32 s41, s41, 0xffff + s_add_u32 s40, s75, s40 + s_addc_u32 s41, 0, s41 + s_and_b32 s37, s37, 0xffff + s_add_u32 s36, s75, s36 + s_addc_u32 s37, 0, s37 + s_load_dword s93, s[40:41], 0x0 + s_load_dword s92, s[40:41], 0x4 + s_load_dword s89, s[36:37], 0x0 + s_load_dword s90, s[36:37], 0x4 + s_waitcnt lgkmcnt(0) + s_sub_u32 s98, s92, s93 + s_cmp_le_u32 s98, s4 + s_cbranch_scc1 label_8EF4 + s_mov_b32 s82, s98 + s_sub_u32 s91, s90, s89 + s_cmp_eq_u32 s91, 0 + s_cbranch_scc1 label_8EF4 + s_mul_i32 s72, s91, s80 + s_mul_i32 s75, 0x800, s72 + s_mul_i32 s76, 0x400, s72 + s_cmp_eq_u32 s82, 1 + s_cselect_b32 s95, s76, s75 + s_mul_i32 s76, 0x200, s72 + s_mul_i32 s77, 0x80, s72 + s_mul_i32 s75, 4, s72 + s_mov_b32 s10, s95 + s_mov_b32 s18, s76 + s_mov_b32 s46, s77 + s_mov_b32 s14, s75 + s_mov_b32 s22, -16 + s_mov_b32 s50, -16 + s_mov_b32 s26, -16 + s_mov_b32 s11, 0x20000 + s_mov_b32 s19, 0x20000 + s_mov_b32 s47, 0x20000 + s_mov_b32 s15, 0x20000 + s_mov_b32 s23, 0x20000 + s_mov_b32 s51, 0x20000 + s_mov_b32 s27, 0x20000 + s_and_b32 s9, s9, 0xffff + s_and_b32 s17, s17, 0xffff + s_and_b32 s45, s45, 0xffff + s_and_b32 s13, s13, 0xffff + s_and_b32 s21, s21, 0xffff + s_and_b32 s49, s49, 0xffff + s_and_b32 s25, s25, 0xffff + s_or_b32 s9, s9, 0x40000 + s_or_b32 s17, s17, 0x40000 + s_or_b32 s45, s45, 0x40000 + s_or_b32 s13, s13, 0x40000 + s_or_b32 s21, s21, 0x40000 + s_or_b32 s49, s49, 0x40000 + s_or_b32 s25, s25, 0x40000 + s_and_b32 s53, s53, 0xffff + s_or_b32 s53, s53, 0x40000 + s_lshl_b32 s75, s80, 2 + s_mov_b32 s54, s75 + s_mov_b32 s55, 0x20000 + s_and_b32 s33, s33, 0xffff + s_or_b32 s33, s33, 0x40000 + s_mov_b32 s34, -16 + s_mov_b32 s35, 0x20000 + s_waitcnt lgkmcnt(0) + s_mov_b32 s86, 0 + s_lshr_b32 s67, 32, s86 + s_sub_u32 s71, s69, s68 + s_cmp_eq_u32 s71, 0 + s_cbranch_scc1 label_8EF4 + s_mul_i32 s100, s4, s67 + s_cmp_eq_u32 s99, 1 + s_cbranch_scc0 label_030C + s_sub_u32 s76, 5, s86 + s_sub_u32 s77, s67, 1 + s_add_u32 s77, s71, s77 + s_lshr_b32 s77, s77, s76 + s_cmp_lt_u32 s77, s82 + s_cselect_b32 s77, s77, s82 + s_cmp_eq_u32 s4, 0 + s_cbranch_scc0 label_030C + s_cmp_eq_u32 s2, 0 + s_cbranch_scc0 label_030C + s_cmp_eq_u32 s7, 0 + s_cbranch_scc0 label_030C + v_mov_b32_e32 v26, s77 + v_mov_b32_e32 v27, s3 + v_lshlrev_b32_e32 v27, 2, v27 + buffer_store_dword v26, v27, s[32:35], 0 offen +label_030C: + s_cmp_le_u32 s71, s100 + s_cbranch_scc1 label_8DAC + s_mov_b32 s87, 0 + s_sub_u32 s88, s71, s100 + s_mul_i32 s65, s82, s67 + s_mov_b32 s64, s88 + s_mov_b32 s66, 0 + s_mov_b32 s66, 0 + v_cvt_f32_u32_e32 v26, s65 + s_sub_i32 s75, 0, s65 + v_rcp_iflag_f32_e32 v26, v26 + s_nop 0 + v_mul_f32_e32 v26, 0x4f7ffffe, v26 + v_cvt_u32_f32_e32 v26, v26 + v_mul_lo_u32 v27, s75, v26 + v_mul_hi_u32 v27, v26, v27 + v_add_u32_e32 v26, v26, v27 + v_mul_hi_u32 v26, s64, v26 + v_mul_lo_u32 v27, v26, s65 + v_sub_u32_e32 v29, s64, v27 + v_add_u32_e32 v28, 1, v26 + v_cmp_le_u32_e32 vcc, s65, v29 + v_subrev_u32_e32 v27, s65, v29 + s_nop 0 + v_cndmask_b32_e32 v26, v26, v28, vcc + v_cndmask_b32_e32 v29, v29, v27, vcc + v_add_u32_e32 v27, 1, v26 + v_cmp_le_u32_e32 vcc, s65, v29 + s_nop 1 + v_cndmask_b32_e32 v29, v26, v27, vcc + s_nop 3 + v_readfirstlane_b32 s66, v29 + s_nop 3 + s_mov_b32 s88, s66 + s_mul_i32 s75, s88, s65 + s_sub_u32 s75, s64, s75 + s_mov_b32 s76, 0 + s_cmp_lt_u32 s75, s67 + s_cselect_b32 s76, s76, 1 + s_add_u32 s88, s76, s88 + s_cmpk_eq_u32 s76, 0x1 + s_cselect_b32 s70, 0, s75 + s_mul_i32 s94, s67, 4 + s_mul_i32 s94, s94, s82 + s_mul_i32 s75, s71, 4 + s_mov_b32 s26, s75 + s_mul_i32 s75, s68, 4 + s_add_u32 s24, s75, s24 + s_addc_u32 s25, 0, s25 + v_lshlrev_b32_e32 v26, 4, v0 + v_and_b32_e32 v27, 32, v26 + v_lshlrev_b32_e32 v26, 3, v0 + v_and_or_b32 v27, v26, 64, v27 + v_and_or_b32 v27, v0, 4, v27 + s_mul_i32 s75, 8, s7 + s_mul_i32 s76, 4, s100 + s_add_u32 s75, s76, s75 + v_add_u32_e32 v38, s75, v27 + v_bfe_u32 v44, v0, 1, 3 + v_lshlrev_b32_e32 v44, 2, v44 + s_mul_i32 s75, 32, s7 + v_add_u32_e32 v44, s75, v44 + s_mul_i32 s76, 4, s100 + v_add_u32_e32 v44, s76, v44 + buffer_load_dword v39, v38, s[24:27], 0 offen + buffer_load_dword v242, v44, s[24:27], 0 offen + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + buffer_load_dword v40, v38, s[24:27], 0 offen + buffer_load_dword v243, v44, s[24:27], 0 offen + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + s_cmp_eq_u32 s80, 16 + s_cselect_b32 s69, 16, 64 + s_cmp_eq_u32 s80, 32 + s_cselect_b32 s69, 32, s69 + s_cmp_eq_u32 s91, 1 + s_cselect_b32 s69, s69, 64 + s_lshr_b32 s72, s69, 4 + s_cmp_eq_u32 s70, 0 + s_cselect_b32 s72, s72, 4 + s_cmp_gt_u32 s88, 0 + s_cselect_b32 s72, s72, 4 + s_mul_i32 s75, s89, s80 + s_mul_i32 s75, s75, 0x200 + s_add_u32 s16, s75, s16 + s_addc_u32 s17, 0, s17 + s_mov_b32 s75, 0x8000 + s_mul_i32 s75, s75, s2 + s_mov_b32 s76, 0x200 + s_mul_i32 s76, s7, s76 + s_add_u32 s75, s75, s76 + v_and_b32_e32 v1, 31, v0 + v_lshlrev_b32_e32 v1, 4, v1 + v_bfe_u32 v26, v0, 5, 1 + v_lshlrev_b32_e32 v26, 11, v26 + v_add_u32_e32 v1, v26, v1 + v_add_u32_e32 v1, s75, v1 + s_mul_i32 s75, s7, 0x820 + s_add_u32 s56, 0x2000, s75 + s_add_u32 s57, 0x2080, s56 + s_add_u32 s58, 0x2080, s57 + s_add_u32 s59, 0x2080, s58 + s_mov_b32 m0, s56 + v_add_u32_e32 v2, 0, v1 + buffer_load_dwordx4 v2, s[16:19], 0 offen lds + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v2, 0x1000, v2 + buffer_load_dwordx4 v2, s[16:19], 0 offen lds + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v2, 0x1000, v2 + s_mov_b32 m0, s57 + v_add_u32_e32 v2, 0x2000, v1 + buffer_load_dwordx4 v2, s[16:19], 0 offen lds + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v2, 0x1000, v2 + buffer_load_dwordx4 v2, s[16:19], 0 offen lds + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v2, 0x1000, v2 + s_mov_b32 s6, 0x3fb8aa3b + v_mov_b32_e32 v27, s6 + v_mov_b32_e32 v26, s79 + v_mul_f32_e32 v26, s6, v26 + v_rcp_f32_e32 v27, v27 + v_mov_b32_e32 v45, 0xff7fffff + v_mov_b32_e32 v33, 0 + v_mov_b32_e32 v46, 0 + s_mov_b32 m0, s58 + v_add_u32_e32 v2, 0x4000, v1 + buffer_load_dwordx4 v2, s[16:19], 0 offen lds + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v2, 0x1000, v2 + buffer_load_dwordx4 v2, s[16:19], 0 offen lds + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v2, 0x1000, v2 + s_mov_b32 m0, s59 + v_add_u32_e32 v2, 0x6000, v1 + buffer_load_dwordx4 v2, s[16:19], 0 offen lds + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v2, 0x1000, v2 + buffer_load_dwordx4 v2, s[16:19], 0 offen lds + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v2, 0x1000, v2 + v_readfirstlane_b32 s5, v26 + v_readfirstlane_b32 s6, v27 + s_mul_i32 s75, s89, s80 + s_mul_i32 s75, s75, 0x80 + s_add_u32 s44, s75, s44 + s_addc_u32 s45, 0, s45 + s_mov_b32 s75, 0x2000 + s_mul_i32 s75, s75, s2 + v_bfe_u32 v26, v0, 1, 3 + v_lshlrev_b32_e32 v12, 7, v26 + v_lshrrev_b32_e32 v26, 4, v0 + v_lshl_add_u32 v12, v26, 5, v12 + v_and_b32_e32 v26, 1, v0 + v_lshl_add_u32 v12, v26, 4, v12 + v_add_u32_e32 v12, s75, v12 + s_add_u32 s60, 0x12800, 0 + s_add_u32 s61, 0x8c0, s60 + s_add_u32 s62, 0x8c0, s61 + s_add_u32 s63, 0x8c0, s62 + v_and_b32_e32 v26, 7, v0 + v_lshlrev_b32_e32 v26, 5, v26 + v_lshrrev_b32_e32 v29, 4, v0 + v_lshl_add_u32 v26, v29, 8, v26 + v_bfe_u32 v29, v0, 3, 1 + v_mul_i32_i24_e32 v29, 0x410, v29 + v_add_u32_e32 v16, v29, v26 + v_add_u32_e32 v16, 0x12800, v16 + s_mul_i32 s75, s7, 0x8c0 + s_add_u32 m0, 0x12800, s75 + s_mul_i32 s75, s7, 0x800 + v_add_u32_e32 v2, s75, v12 + buffer_load_dwordx4 v2, s[44:47], 0 offen lds + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v2, 0x400, v2 + buffer_load_dwordx4 v2, s[44:47], 0 offen lds + v_bfe_u32 v26, v0, 3, 1 + v_and_b32_e32 v27, 3, v0 + v_lshl_or_b32 v26, v27, 1, v26 + v_mul_i32_i24_e32 v27, 0x410, v26 + v_bfe_u32 v26, v0, 2, 1 + v_mul_i32_i24_e32 v26, 0x200, v26 + v_add_u32_e32 v27, v26, v27 + v_bfe_u32 v26, v0, 4, 2 + v_lshl_add_u32 v10, v26, 5, v27 + v_lshrrev_b32_e32 v28, 4, v0 + v_mul_i32_i24_e32 v29, 32, v28 + v_sub_u32_e32 v34, v10, v29 + v_add_u32_e32 v34, 0x1c0, v34 + v_lshlrev_b32_e32 v29, 1, v28 + v_lshrrev_b32_e32 v28, 1, v28 + v_or_b32_e32 v28, v28, v29 + v_and_b32_e32 v28, 3, v28 + v_add_u32_e32 v30, v28, v34 + v_sub_u32_e32 v25, v30, v34 + v_lshlrev_b32_e32 v25, 3, v25 + v_lshlrev_b32_e32 v43, 2, v0 + v_lshlrev_b32_e32 v17, 2, v0 + s_mul_i32 s75, 0x100, s7 + v_add_u32_e32 v17, s75, v17 + v_and_b32_e32 v26, 31, v0 + v_lshlrev_b32_e32 v27, 3, v26 + v_bfe_u32 v26, v0, 4, 1 + v_lshl_add_u32 v27, v26, 8, v27 + v_lshrrev_b32_e32 v26, 5, v0 + v_lshl_add_u32 v18, v26, 11, v27 + v_xor_b32_e32 v19, 0x80, v18 + v_lshlrev_b32_e32 v26, 2, v0 + v_and_b32_e32 v26, 0x80, v26 + s_mul_i32 s75, 0x200, s7 + v_lshl_add_u32 v20, v0, 3, s75 + v_xor_b32_e32 v20, v20, v26 + s_waitcnt vmcnt(2) + s_barrier + s_cmp_ge_u32 s7, s72 + s_cbranch_scc1 label_09EC + s_cmp_eq_i32 s7, 0 + s_cbranch_scc0 label_0854 + ds_read_b128 a[0:3], v10 offset:8192 + ds_read_b128 a[4:7], v10 offset:8208 + ds_read_b128 a[8:11], v10 offset:8320 + ds_read_b128 a[12:15], v10 offset:8336 + ds_read_b128 a[16:19], v10 offset:8448 + ds_read_b128 a[20:23], v10 offset:8464 + ds_read_b128 a[24:27], v10 offset:8576 + ds_read_b128 a[28:31], v10 offset:8592 + ds_read_b128 v[26:29], v34 offset:8192 + s_waitcnt lgkmcnt(0) + v_bfe_u32 v224, v26, v25, 8 + v_bfe_u32 v225, v27, v25, 8 + v_bfe_u32 v226, v28, v25, 8 + v_bfe_u32 v227, v29, v25, 8 + v_cmp_ge_u32_e64 vcc, v25, 16 + v_cndmask_b32_e64 v227, v227, 0, vcc + s_branch label_09EC +label_0854: + s_cmp_eq_i32 s7, 1 + s_cbranch_scc0 label_08DC + ds_read_b128 a[0:3], v10 offset:16640 + ds_read_b128 a[4:7], v10 offset:16656 + ds_read_b128 a[8:11], v10 offset:16512 + ds_read_b128 a[12:15], v10 offset:16528 + ds_read_b128 a[16:19], v10 offset:16768 + ds_read_b128 a[20:23], v10 offset:16784 + ds_read_b128 a[24:27], v10 offset:16896 + ds_read_b128 a[28:31], v10 offset:16912 + ds_read_b128 v[26:29], v34 offset:16512 + s_waitcnt lgkmcnt(0) + v_bfe_u32 v224, v27, v25, 8 + v_bfe_u32 v225, v26, v25, 8 + v_bfe_u32 v226, v28, v25, 8 + v_bfe_u32 v227, v29, v25, 8 + v_cmp_ge_u32_e64 vcc, v25, 16 + v_cndmask_b32_e64 v227, v227, 0, vcc + s_branch label_09EC +label_08DC: + s_cmp_eq_i32 s7, 2 + s_cbranch_scc0 label_0964 + ds_read_b128 a[0:3], v10 offset:24832 + ds_read_b128 a[4:7], v10 offset:24848 + ds_read_b128 a[8:11], v10 offset:24960 + ds_read_b128 a[12:15], v10 offset:24976 + ds_read_b128 a[16:19], v10 offset:25088 + ds_read_b128 a[20:23], v10 offset:25104 + ds_read_b128 a[24:27], v10 offset:25216 + ds_read_b128 a[28:31], v10 offset:25232 + ds_read_b128 v[26:29], v34 offset:24832 + s_waitcnt lgkmcnt(0) + v_bfe_u32 v224, v26, v25, 8 + v_bfe_u32 v225, v27, v25, 8 + v_bfe_u32 v226, v28, v25, 8 + v_bfe_u32 v227, v29, v25, 8 + v_cmp_ge_u32_e64 vcc, v25, 16 + v_cndmask_b32_e64 v227, v227, 0, vcc + s_branch label_09EC +label_0964: + s_cmp_eq_i32 s7, 3 + s_cbranch_scc0 label_09EC + ds_read_b128 a[0:3], v10 offset:33152 + ds_read_b128 a[4:7], v10 offset:33168 + ds_read_b128 a[8:11], v10 offset:33280 + ds_read_b128 a[12:15], v10 offset:33296 + ds_read_b128 a[16:19], v10 offset:33408 + ds_read_b128 a[20:23], v10 offset:33424 + ds_read_b128 a[24:27], v10 offset:33536 + ds_read_b128 a[28:31], v10 offset:33552 + ds_read_b128 v[26:29], v34 offset:33152 + s_waitcnt lgkmcnt(0) + v_bfe_u32 v224, v26, v25, 8 + v_bfe_u32 v225, v27, v25, 8 + v_bfe_u32 v226, v28, v25, 8 + v_bfe_u32 v227, v29, v25, 8 + v_cmp_ge_u32_e64 vcc, v25, 16 + v_cndmask_b32_e64 v227, v227, 0, vcc + s_branch label_09EC +label_09EC: + s_waitcnt vmcnt(0) + s_barrier + s_cmp_ge_u32 s7, s72 + s_cbranch_scc1 label_0A6C + s_cmp_eq_i32 s7, 0 + s_cbranch_scc0 label_0A18 + ds_read_b128 a[32:35], v16 + ds_read_b128 a[36:39], v16 offset:16 + s_branch label_0A6C +label_0A18: + s_cmp_eq_i32 s7, 1 + s_cbranch_scc0 label_0A34 + ds_read_b128 a[32:35], v16 offset:2240 + ds_read_b128 a[36:39], v16 offset:2256 + s_branch label_0A6C +label_0A34: + s_cmp_eq_i32 s7, 2 + s_cbranch_scc0 label_0A50 + ds_read_b128 a[32:35], v16 offset:4480 + ds_read_b128 a[36:39], v16 offset:4496 + s_branch label_0A6C +label_0A50: + s_cmp_eq_i32 s7, 3 + s_cbranch_scc0 label_0A6C + ds_read_b128 a[32:35], v16 offset:6720 + ds_read_b128 a[36:39], v16 offset:6736 + s_branch label_0A6C +label_0A6C: + v_bfe_u32 v26, v0, 2, 2 + v_lshlrev_b32_e32 v1, 7, v26 + v_lshrrev_b32_e32 v26, 4, v0 + v_lshl_add_u32 v1, v26, 5, v1 + v_and_b32_e32 v26, 1, v0 + v_lshl_add_u32 v1, v26, 4, v1 + s_mov_b32 s92, 0xcccccccc + s_mov_b32 s93, 0xcccccccc + s_mov_b32 s74, 0xff7fffff + s_mul_i32 s75, s7, 0x1040 + s_add_u32 s56, 0x2000, s75 + s_add_u32 s57, 0x4100, s56 + s_add_u32 s58, 0x4100, s57 + s_add_u32 s59, 0x4100, s58 + s_movk_i32 s73, 0x200 + s_and_b32 s75, s21, 0xffff + v_add_co_u32_e32 v14, vcc, s20, v1 + v_mov_b32_e32 v15, s75 + v_addc_co_u32_e64 v15, vcc, v15, 0, vcc + v_max_i32_e32 v39, 0, v39 + s_nop 1 + v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + s_mov_b32 m0, s56 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + buffer_load_dword v41, v38, s[24:27], 0 offen + buffer_load_dword v244, v44, s[24:27], 0 offen + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + s_waitcnt lgkmcnt(0) + s_barrier + s_cmp_ge_u32 s7, s72 + s_cbranch_scc1 label_0C34 + v_lshrrev_b32_e32 v32, 4, v0 + v_cmp_ge_u32_e64 vcc, v32, 2 + s_mov_b64 exec, vcc + v_accvgpr_write_b32 a24, 0 + v_accvgpr_write_b32 a25, 0 + v_accvgpr_write_b32 a26, 0 + v_accvgpr_write_b32 a27, 0 + v_accvgpr_write_b32 a28, 0 + v_accvgpr_write_b32 a29, 0 + v_accvgpr_write_b32 a30, 0 + v_accvgpr_write_b32 a31, 0 + s_mov_b64 exec, -1 +label_0C34: + v_and_b32_e32 v26, 48, v0 + v_lshlrev_b32_e32 v12, 1, v26 + v_and_b32_e32 v26, 1, v0 + v_lshl_or_b32 v12, v26, 4, v12 + s_mul_i32 s75, s7, 0x410 + s_add_u32 s60, 0x12800, s75 + s_add_u32 s61, 0x1040, s60 + s_add_u32 s62, 0x1040, s61 + s_add_u32 s63, 0x1040, s62 + s_movk_i32 s86, 0x80 + s_and_b32 s75, s49, 0xffff + v_add_co_u32_e32 v246, vcc, s48, v12 + v_mov_b32_e32 v247, s75 + v_addc_co_u32_e64 v247, vcc, v247, 0, vcc + v_max_i32_e32 v242, 0, v242 + v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] + s_mov_b32 m0, s60 + global_load_lds_dwordx4 v[2:3], off + v_lshrrev_b32_e32 v26, 4, v0 + v_lshlrev_b32_e32 v27, 8, v26 + v_and_b32_e32 v26, 7, v0 + v_lshlrev_b32_e32 v28, 5, v26 + v_add_u32_e32 v27, v28, v27 + v_bfe_u32 v26, v0, 3, 1 + v_mul_i32_i24_e32 v28, 0x410, v26 + v_add_u32_e32 v27, v28, v27 + v_add_u32_e32 v16, 0x12800, v27 + v_and_b32_e32 v26, 6, v0 + v_lshlrev_b32_e32 v26, 1, v26 + v_and_or_b32 v26, v0, 1, v26 + v_mul_i32_i24_e32 v27, 0x410, v26 + v_bfe_u32 v26, v0, 3, 1 + v_mul_i32_i24_e32 v26, 32, v26 + v_add_u32_e32 v27, v26, v27 + v_bfe_u32 v26, v0, 4, 2 + v_lshl_add_u32 v13, v26, 8, v27 + v_max_i32_e32 v40, 0, v40 + s_nop 1 + v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + s_mov_b32 m0, s57 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_lshrrev_b32_e32 v28, 4, v0 + v_mul_i32_i24_e32 v29, 0x100, v28 + v_sub_u32_e32 v1, v13, v29 + v_add_u32_e32 v1, 0x2c0, v1 + v_lshlrev_b32_e32 v29, 1, v28 + v_lshrrev_b32_e32 v28, 1, v28 + v_or_b32_e32 v28, v28, v29 + v_and_b32_e32 v28, 3, v28 + v_add_u32_e32 v30, v28, v1 + v_sub_u32_e32 v25, v30, v1 + v_lshlrev_b32_e32 v25, 3, v25 + s_mul_i32 s75, s7, 64 + s_lshl_b32 s76, s7, 2 + v_lshrrev_b32_e32 v32, 4, v0 + s_cmp_eq_i32 s7, 3 + s_cselect_b64 vcc, -1, 0 + v_and_b32_e32 v33, 32, v0 + v_lshlrev_b32_e32 v33, 4, v33 + v_sub_u32_e32 v241, v13, v33 + v_cndmask_b32_e32 v33, v13, v241, vcc + v_add_u32_e32 v10, s75, v33 + v_add3_u32 v238, s76, v1, v32 + s_xor_b32 s78, s75, 64 + s_cmp_lt_u32 s7, 2 + s_cselect_b32 s76, s75, 0 + s_cselect_b32 s78, s78, 64 + v_add_u32_e32 v239, s76, v13 + v_add_u32_e32 v240, s78, v13 + s_cmp_eq_i32 s7, 1 + s_cselect_b32 s78, 4, 0 + s_or_b32 s78, s78, 0xc0c0c00 + v_lshrrev_b32_e32 v12, 3, v25 + v_or_b32_e32 v12, s78, v12 + v_max_i32_e32 v243, 0, v243 + v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] + s_mov_b32 m0, s61 + global_load_lds_dwordx4 v[2:3], off + v_and_b32_e32 v26, 15, v0 + v_mul_i32_i24_e32 v26, 0x390, v26 + v_lshrrev_b32_e32 v27, 4, v0 + v_lshl_add_u32 v26, v27, 6, v26 + s_mul_i32 s75, 0x100, s7 + s_add_u32 s75, 0x17000, s75 + v_add_u32_e32 v11, s75, v26 + v_cmp_ge_u32_e64 vcc, v27, 2 + s_cmp_lg_u32 s7, 3 + s_nop 4 + s_cselect_b64 vcc, 0, vcc + v_and_b32_e32 v28, 31, v0 + v_lshlrev_b32_e32 v28, 4, v28 + v_add_u32_e32 v28, 0x1e200, v28 + v_cndmask_b32_e32 v11, v11, v28, vcc + v_bfe_u32 v26, v0, 2, 2 + v_bfe_u32 v27, v0, 4, 2 + v_and_b32_e32 v28, 3, v0 + v_lshlrev_b32_e32 v28, 3, v28 + v_lshl_add_u32 v29, v27, 3, v26 + v_mul_i32_i24_e32 v29, 0x390, v29 + v_add_u32_e32 v29, v28, v29 + s_mul_i32 s75, 0x100, s7 + s_add_u32 s75, 0x17000, s75 + v_add_u32_e32 v21, s75, v29 + v_add_u32_e32 v22, 32, v21 + v_mul_i32_i24_e32 v29, 0x410, v27 + v_add_u32_e32 v29, v28, v29 + v_lshl_add_u32 v29, v26, 5, v29 + v_add_u32_e32 v23, 0x12800, v29 + v_add_u32_e32 v24, 0x100, v23 + buffer_load_dword v42, v38, s[24:27], 0 offen + buffer_load_dword v245, v44, s[24:27], 0 offen + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mov_b64_e32 v[96:97], 0 + v_mov_b64_e32 v[98:99], 0 + v_mov_b64_e32 v[100:101], 0 + v_mov_b64_e32 v[102:103], 0 + v_mov_b64_e32 v[104:105], 0 + v_mov_b64_e32 v[106:107], 0 + v_mov_b64_e32 v[108:109], 0 + v_mov_b64_e32 v[110:111], 0 + v_mov_b64_e32 v[112:113], 0 + v_mov_b64_e32 v[114:115], 0 + v_mov_b64_e32 v[116:117], 0 + v_mov_b64_e32 v[118:119], 0 + v_mov_b64_e32 v[120:121], 0 + v_mov_b64_e32 v[122:123], 0 + v_mov_b64_e32 v[124:125], 0 + v_mov_b64_e32 v[126:127], 0 + s_cmp_le_u32 s72, 1 + s_cbranch_scc1 label_1068 + v_mov_b64_e32 v[128:129], 0 + v_mov_b64_e32 v[130:131], 0 + v_mov_b64_e32 v[132:133], 0 + v_mov_b64_e32 v[134:135], 0 + v_mov_b64_e32 v[136:137], 0 + v_mov_b64_e32 v[138:139], 0 + v_mov_b64_e32 v[140:141], 0 + v_mov_b64_e32 v[142:143], 0 + v_mov_b64_e32 v[144:145], 0 + v_mov_b64_e32 v[146:147], 0 + v_mov_b64_e32 v[148:149], 0 + v_mov_b64_e32 v[150:151], 0 + v_mov_b64_e32 v[152:153], 0 + v_mov_b64_e32 v[154:155], 0 + v_mov_b64_e32 v[156:157], 0 + v_mov_b64_e32 v[158:159], 0 + s_cmp_le_u32 s72, 2 + s_cbranch_scc1 label_1068 + v_mov_b64_e32 v[160:161], 0 + v_mov_b64_e32 v[162:163], 0 + v_mov_b64_e32 v[164:165], 0 + v_mov_b64_e32 v[166:167], 0 + v_mov_b64_e32 v[168:169], 0 + v_mov_b64_e32 v[170:171], 0 + v_mov_b64_e32 v[172:173], 0 + v_mov_b64_e32 v[174:175], 0 + v_mov_b64_e32 v[176:177], 0 + v_mov_b64_e32 v[178:179], 0 + v_mov_b64_e32 v[180:181], 0 + v_mov_b64_e32 v[182:183], 0 + v_mov_b64_e32 v[184:185], 0 + v_mov_b64_e32 v[186:187], 0 + v_mov_b64_e32 v[188:189], 0 + v_mov_b64_e32 v[190:191], 0 + v_mov_b64_e32 v[192:193], 0 + v_mov_b64_e32 v[194:195], 0 + v_mov_b64_e32 v[196:197], 0 + v_mov_b64_e32 v[198:199], 0 + v_mov_b64_e32 v[200:201], 0 + v_mov_b64_e32 v[202:203], 0 + v_mov_b64_e32 v[204:205], 0 + v_mov_b64_e32 v[206:207], 0 + v_mov_b64_e32 v[208:209], 0 + v_mov_b64_e32 v[210:211], 0 + v_mov_b64_e32 v[212:213], 0 + v_mov_b64_e32 v[214:215], 0 + v_mov_b64_e32 v[216:217], 0 + v_mov_b64_e32 v[218:219], 0 + v_mov_b64_e32 v[220:221], 0 + v_mov_b64_e32 v[222:223], 0 +label_1068: + s_waitcnt vmcnt(6) + s_barrier + s_cmp_ge_u32 s7, s72 + s_cbranch_scc1 label_1098 + ds_read_b128 a[104:107], v16 + ds_read_b128 a[108:111], v16 offset:16 + ds_read_b128 a[112:115], v16 offset:2080 + ds_read_b128 a[116:119], v16 offset:2096 +label_1098: + ds_read_b128 v[80:83], v10 offset:8192 + ds_read_b128 v[84:87], v10 offset:8208 + ds_read_b128 v[88:91], v10 offset:10272 + ds_read_b128 v[92:95], v10 offset:10288 + s_cmp_ge_u32 s7, s72 + s_cbranch_scc1 label_11FC + v_accvgpr_write_b32 a64, 0 + v_accvgpr_write_b32 a65, 0 + v_accvgpr_write_b32 a66, 0 + v_accvgpr_write_b32 a67, 0 + v_accvgpr_write_b32 a68, 0 + v_accvgpr_write_b32 a69, 0 + v_accvgpr_write_b32 a70, 0 + v_accvgpr_write_b32 a71, 0 + v_accvgpr_write_b32 a96, 0 + v_accvgpr_write_b32 a97, 0 + v_accvgpr_write_b32 a98, 0 + v_accvgpr_write_b32 a99, 0 + v_accvgpr_write_b32 a100, 0 + v_accvgpr_write_b32 a101, 0 + v_accvgpr_write_b32 a102, 0 + v_accvgpr_write_b32 a103, 0 + v_lshrrev_b32_e32 v32, 4, v0 + v_cmp_eq_u32_e32 vcc, 3, v32 + v_mov_b32_e32 v47, 0xff + v_cndmask_b32_e64 v47, v47, 0, vcc + ds_read_b128 a[40:43], v239 offset:8192 + ds_read_b128 a[44:47], v239 offset:8208 + ds_read_b128 a[72:75], v239 offset:10272 + ds_read_b128 a[76:79], v239 offset:10288 + ds_read_b128 a[48:51], v240 offset:8192 + ds_read_b128 a[52:55], v240 offset:8208 + ds_read_b128 a[80:83], v240 offset:10272 + ds_read_b128 a[84:87], v240 offset:10288 + ds_read_b128 a[56:59], v13 offset:8320 + ds_read_b128 a[60:63], v13 offset:8336 + ds_read_b128 a[88:91], v13 offset:10400 + ds_read_b128 a[92:95], v13 offset:10416 + ds_read_b128 a[64:67], v241 offset:8384 + ds_read_b128 a[68:71], v241 offset:8400 + ds_read_b128 a[96:99], v241 offset:10464 + ds_read_b128 a[100:103], v241 offset:10480 + ds_read_b128 v[228:231], v1 offset:8192 + ds_read_u8 v236, v238 offset:8192 + ds_read_b128 v[232:235], v1 offset:10272 + ds_read_u8 v237, v238 offset:10272 + s_branch label_120C +label_11FC: + ds_read_u8 v236, v238 offset:8192 + ds_read_u8 v237, v238 offset:10272 +label_120C: + s_cmp_eq_u32 s88, 0 + s_cbranch_scc1 label_1F14 + s_cmp_ge_u32 s7, s72 + s_cbranch_scc1 label_12E14 + s_cmp_lt_u32 s7, 2 + s_cbranch_scc0 label_18A0 + s_waitcnt lgkmcnt(0) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + buffer_load_dword v39, v38, s[24:27], 0 offen + buffer_load_dword v242, v44, s[24:27], 0 offen + v_max_i32_e32 v41, 0, v41 + v_bfe_u32 v230, v230, v25, 8 + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + s_mov_b32 m0, s58 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_cvt_scalef32_pk_bf16_fp8 v34, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + ds_write_b128 v11, v[34:37] + v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cvt_scalef32_pk_bf16_fp8 v34, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + ds_write_b128 v11, v[34:37] offset:16 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_cvt_scalef32_pk_bf16_fp8 v34, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v85, v33 op_sel:[1,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_write_b128 v11, v[34:37] offset:32 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_bfe_u32 v234, v234, v25, 8 + v_cvt_scalef32_pk_bf16_fp8 v34, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v87, v33 op_sel:[1,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + ds_write_b128 v11, v[34:37] offset:48 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_and_b32_e32 v235, v47, v235 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + v_cvt_scalef32_pk_bf16_fp8 v34, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v89, v33 op_sel:[1,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + ds_write_b128 v11, v[34:37] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v34, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + ds_write_b128 v11, v[34:37] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v34, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + ds_write_b128 v11, v[34:37] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v34, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v95, v33 op_sel:[1,0,0] + ds_write_b128 v11, v[34:37] offset:14640 + v_max_i32_e32 v244, 0, v244 + v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] + s_mov_b32 m0, s62 + global_load_lds_dwordx4 v[2:3], off + s_waitcnt vmcnt(5) + s_barrier + ds_read_b128 v[228:231], v1 offset:24832 + ds_read_u8 v236, v238 offset:24832 + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + ds_read_b128 v[232:235], v1 offset:26912 + ds_read_u8 v237, v238 offset:26912 + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + ds_read_b128 v[80:83], v10 offset:24832 + ds_read_b128 v[84:87], v10 offset:24848 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + ds_read_b128 v[88:91], v10 offset:26912 + ds_read_b128 v[92:95], v10 offset:26928 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + ds_read_b128 a[40:43], v239 offset:24832 + ds_read_b128 a[44:47], v239 offset:24848 + v_exp_f32_e32 v33, v33 + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + ds_read_b128 a[72:75], v239 offset:26912 + ds_read_b128 a[76:79], v239 offset:26928 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + ds_read_b128 a[48:51], v240 offset:24832 + ds_read_b128 a[52:55], v240 offset:24848 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + ds_read_b128 a[80:83], v240 offset:26912 + ds_read_b128 a[84:87], v240 offset:26928 + v_exp_f32_e32 v50, v50 + v_exp_f32_e32 v51, v51 + ds_read_b128 a[56:59], v13 offset:24960 + ds_read_b128 a[60:63], v13 offset:24976 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + ds_read_b128 a[88:91], v13 offset:27040 + ds_read_b128 a[92:95], v13 offset:27056 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + ds_read_b128 a[64:67], v241 offset:25024 + ds_read_b128 a[68:71], v241 offset:25040 + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + ds_read_b128 a[96:99], v241 offset:27104 + ds_read_b128 a[100:103], v241 offset:27120 + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_add_f32_e32 v26, v55, v26 + ds_read_b128 a[104:107], v16 offset:4160 + ds_read_b128 a[108:111], v16 offset:4176 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + ds_write_b32 v17, v33 offset:4096 + ds_write_b64 v20, v[48:49] + ds_write_b64 v20, v[50:51] offset:2048 + v_max_i32_e32 v42, 0, v42 + s_nop 1 + v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + ds_read_b128 a[112:115], v16 offset:6240 + ds_read_b128 a[116:119], v16 offset:6256 + buffer_load_dword v40, v38, s[24:27], 0 offen + buffer_load_dword v243, v44, s[24:27], 0 offen + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + s_mov_b32 m0, s59 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_max_i32_e32 v245, 0, v245 + v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] + s_mov_b32 m0, s63 + global_load_lds_dwordx4 v[2:3], off + s_branch label_1F1C +label_18A0: + s_waitcnt lgkmcnt(0) + s_barrier + v_bfe_u32 v228, v228, v25, 8 + v_bfe_u32 v229, v229, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + buffer_load_dword v39, v38, s[24:27], 0 offen + buffer_load_dword v242, v44, s[24:27], 0 offen + v_max_i32_e32 v41, 0, v41 + v_bfe_u32 v230, v230, v25, 8 + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + s_mov_b32 m0, s58 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_cvt_scalef32_pk_bf16_fp8 v34, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + ds_write_b128 v11, v[34:37] + v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cvt_scalef32_pk_bf16_fp8 v34, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + ds_write_b128 v11, v[34:37] offset:16 + v_bfe_u32 v232, v232, v25, 8 + v_bfe_u32 v233, v233, v25, 8 + v_cvt_scalef32_pk_bf16_fp8 v34, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v85, v33 op_sel:[1,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_write_b128 v11, v[34:37] offset:32 + v_bfe_u32 v234, v234, v25, 8 + v_cvt_scalef32_pk_bf16_fp8 v34, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v87, v33 op_sel:[1,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + ds_write_b128 v11, v[34:37] offset:48 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_and_b32_e32 v235, v47, v235 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_cvt_scalef32_pk_bf16_fp8 v34, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v89, v33 op_sel:[1,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + ds_write_b128 v11, v[34:37] offset:14592 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_cvt_scalef32_pk_bf16_fp8 v34, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + ds_write_b128 v11, v[34:37] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v34, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + ds_write_b128 v11, v[34:37] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v34, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v95, v33 op_sel:[1,0,0] + ds_write_b128 v11, v[34:37] offset:14640 + v_max_i32_e32 v244, 0, v244 + v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] + s_mov_b32 m0, s62 + global_load_lds_dwordx4 v[2:3], off + s_waitcnt vmcnt(5) + s_barrier + ds_read_b128 v[228:231], v1 offset:24832 + ds_read_u8 v236, v238 offset:24832 + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + ds_read_b128 v[232:235], v1 offset:26912 + ds_read_u8 v237, v238 offset:26912 + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + ds_read_b128 v[80:83], v10 offset:24832 + ds_read_b128 v[84:87], v10 offset:24848 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + ds_read_b128 v[88:91], v10 offset:26912 + ds_read_b128 v[92:95], v10 offset:26928 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + ds_read_b128 a[40:43], v239 offset:24832 + ds_read_b128 a[44:47], v239 offset:24848 + v_exp_f32_e32 v33, v33 + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + ds_read_b128 a[72:75], v239 offset:26912 + ds_read_b128 a[76:79], v239 offset:26928 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + ds_read_b128 a[48:51], v240 offset:24832 + ds_read_b128 a[52:55], v240 offset:24848 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + ds_read_b128 a[80:83], v240 offset:26912 + ds_read_b128 a[84:87], v240 offset:26928 + v_exp_f32_e32 v50, v50 + v_exp_f32_e32 v51, v51 + ds_read_b128 a[56:59], v13 offset:24960 + ds_read_b128 a[60:63], v13 offset:24976 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + ds_read_b128 a[88:91], v13 offset:27040 + ds_read_b128 a[92:95], v13 offset:27056 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + ds_read_b128 a[64:67], v241 offset:25024 + ds_read_b128 a[68:71], v241 offset:25040 + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + ds_read_b128 a[96:99], v241 offset:27104 + ds_read_b128 a[100:103], v241 offset:27120 + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_add_f32_e32 v26, v55, v26 + ds_read_b128 a[104:107], v16 offset:4160 + ds_read_b128 a[108:111], v16 offset:4176 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + ds_write_b32 v17, v33 offset:4096 + ds_write_b64 v20, v[48:49] + ds_write_b64 v20, v[50:51] offset:2048 + v_max_i32_e32 v42, 0, v42 + s_nop 1 + v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + ds_read_b128 a[112:115], v16 offset:6240 + ds_read_b128 a[116:119], v16 offset:6256 + buffer_load_dword v40, v38, s[24:27], 0 offen + buffer_load_dword v243, v44, s[24:27], 0 offen + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + s_mov_b32 m0, s59 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_max_i32_e32 v245, 0, v245 + v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] + s_mov_b32 m0, s63 + global_load_lds_dwordx4 v[2:3], off + s_branch label_1F1C +label_1F14: + s_mov_b32 s72, s69 + s_branch label_8EFC +label_1F1C: + s_mov_b32 s72, s69 + s_cmp_eq_u32 s88, 1 + s_cbranch_scc1 label_795C + s_mov_b32 s87, 1 + s_cmp_eq_u32 s72, 64 + s_cbranch_scc1 label_1F44 + s_cmp_eq_u32 s72, 16 + s_cbranch_scc1 label_9FB8 + s_cmp_eq_u32 s72, 32 + s_cbranch_scc1 label_EDDC +label_1F44: + s_cmp_eq_i32 s7, 2 + s_cbranch_scc1 label_3D74 + s_cmp_eq_i32 s7, 3 + s_cbranch_scc1 label_5B68 +label_1F54: + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b32 v36, v43 offset:4608 + ds_read_b32 v37, v43 offset:4864 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v39, 0, v39 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s56 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[50:51], v19 + v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] + buffer_load_dword v41, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] + buffer_load_dword v244, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[52:53], v18 offset:512 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64 v[54:55], v19 offset:512 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64 v[56:57], v18 offset:1024 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + ds_read_b64 v[58:59], v19 offset:1024 + ds_read_b64 v[60:61], v18 offset:1536 + v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_min_f32_e32 v26, v26, v36 + v_min_f32_e32 v26, v26, v37 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] + ds_read_b64 v[62:63], v19 offset:1536 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] + s_cbranch_vccz label_221C + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9DB4 +label_221C: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v64, v65, v64 + v_max3_f32 v30, v66, v67, v30 + v_max3_f32 v30, v68, v69, v30 + v_max3_f32 v30, v70, v71, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v242, 0, v242 + v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + s_mov_b32 m0, s60 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_waitcnt vmcnt(12) + s_barrier + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:41472 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] + ds_read_b128 a[48:51], v240 offset:41472 + ds_read_b128 a[52:55], v240 offset:41488 + ds_read_u8 v236, v238 offset:41472 + v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] + ds_read_b128 a[80:83], v240 offset:43552 + ds_read_b128 a[84:87], v240 offset:43568 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:43552 + v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] + v_fma_f32 v64, v64, s5, -v32 + v_fma_f32 v65, v65, s5, -v32 + v_fma_f32 v66, v66, s5, -v32 + v_fma_f32 v67, v67, s5, -v32 + v_fma_f32 v68, v68, s5, -v32 + v_fma_f32 v69, v69, s5, -v32 + v_fma_f32 v70, v70, s5, -v32 + v_fma_f32 v71, v71, s5, -v32 + v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] + ds_read_b128 a[56:59], v13 offset:41600 + ds_read_b128 a[60:63], v13 offset:41616 + v_exp_f32_e32 v64, v64 + v_exp_f32_e32 v65, v65 + v_exp_f32_e32 v66, v66 + ds_read_u8 v237, v238 offset:43552 + v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] + v_exp_f32_e32 v67, v67 + v_exp_f32_e32 v68, v68 + v_exp_f32_e32 v69, v69 + v_exp_f32_e32 v70, v70 + v_exp_f32_e32 v71, v71 + v_mul_f32_e32 v46, v33, v46 + v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] + ds_read_b128 a[88:91], v13 offset:43680 + ds_read_b128 a[92:95], v13 offset:43696 + v_add_f32_e32 v26, v65, v64 + v_add_f32_e32 v26, v66, v26 + v_add_f32_e32 v26, v67, v26 + v_add_f32_e32 v26, v68, v26 + v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] + v_add_f32_e32 v26, v69, v26 + v_add_f32_e32 v26, v70, v26 + v_add_f32_e32 v26, v71, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v64, v64, v65 + v_cvt_pk_bf16_f32 v65, v66, v67 + v_cvt_pk_bf16_f32 v66, v68, v69 + v_cvt_pk_bf16_f32 v67, v70, v71 + v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] + ds_read_b128 a[64:67], v241 offset:41664 + ds_read_b128 a[68:71], v241 offset:41680 + ds_write_b32 v17, v33 offset:4096 + v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] + v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] + ds_read_b128 a[96:99], v241 offset:43744 + ds_read_b128 a[100:103], v241 offset:43760 + ds_write_b64 v20, v[64:65] + v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] + v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] + ds_read_b128 a[104:107], v16 offset:8320 + ds_read_b128 a[108:111], v16 offset:8336 + ds_write_b64 v20, v[66:67] offset:2048 + v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] + v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] + ds_read_b128 a[112:115], v16 offset:10400 + ds_read_b128 a[116:119], v16 offset:10416 + v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b32 v36, v43 offset:4608 + ds_read_b32 v37, v43 offset:4864 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v40, 0, v40 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s57 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[66:67], v19 + v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + buffer_load_dword v42, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + buffer_load_dword v245, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[68:69], v18 offset:512 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64 v[70:71], v19 offset:512 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64 v[72:73], v18 offset:1024 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + ds_read_b64 v[74:75], v19 offset:1024 + ds_read_b64 v[76:77], v18 offset:1536 + v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_min_f32_e32 v26, v26, v36 + v_min_f32_e32 v26, v26, v37 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + ds_read_b64 v[78:79], v19 offset:1536 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + s_cbranch_vccz label_29A0 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9DB4 +label_29A0: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v243, 0, v243 + v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] + s_mov_b32 m0, s61 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] + s_waitcnt vmcnt(12) + s_barrier + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:58112 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[72:75], v[160:163] + ds_read_b128 a[48:51], v240 offset:58112 + ds_read_b128 a[52:55], v240 offset:58128 + ds_read_u8 v236, v238 offset:58112 + v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[72:75], v[164:167] + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[72:75], v[168:171] + ds_read_b128 a[80:83], v240 offset:60192 + ds_read_b128 a[84:87], v240 offset:60208 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:60192 + v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[72:75], v[172:175] + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[72:75], v[176:179] + ds_read_b128 a[56:59], v13 offset:58240 + ds_read_b128 a[60:63], v13 offset:58256 + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + v_exp_f32_e32 v50, v50 + ds_read_u8 v237, v238 offset:60192 + v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[72:75], v[180:183] + v_exp_f32_e32 v51, v51 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[72:75], v[184:187] + ds_read_b128 a[88:91], v13 offset:60320 + ds_read_b128 a[92:95], v13 offset:60336 + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[72:75], v[188:191] + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_add_f32_e32 v26, v55, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[76:79], v[192:195] + ds_read_b128 a[64:67], v241 offset:58304 + ds_read_b128 a[68:71], v241 offset:58320 + ds_write_b32 v17, v33 offset:4096 + v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[76:79], v[196:199] + v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[76:79], v[200:203] + ds_read_b128 a[96:99], v241 offset:60384 + ds_read_b128 a[100:103], v241 offset:60400 + ds_write_b64 v20, v[48:49] + v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[76:79], v[204:207] + v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[76:79], v[208:211] + ds_read_b128 a[104:107], v16 offset:12480 + ds_read_b128 a[108:111], v16 offset:12496 + ds_write_b64 v20, v[50:51] offset:2048 + v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[76:79], v[212:215] + v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[76:79], v[216:219] + ds_read_b128 a[112:115], v16 offset:14560 + ds_read_b128 a[116:119], v16 offset:14576 + v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[76:79], v[220:223] + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b32 v36, v43 offset:4608 + ds_read_b32 v37, v43 offset:4864 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v41, 0, v41 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s58 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[50:51], v19 + v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] + buffer_load_dword v39, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] + buffer_load_dword v242, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[52:53], v18 offset:512 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64 v[54:55], v19 offset:512 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64 v[56:57], v18 offset:1024 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + ds_read_b64 v[58:59], v19 offset:1024 + ds_read_b64 v[60:61], v18 offset:1536 + v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_min_f32_e32 v26, v26, v36 + v_min_f32_e32 v26, v26, v37 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] + ds_read_b64 v[62:63], v19 offset:1536 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] + s_cbranch_vccz label_3124 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9DB4 +label_3124: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v64, v65, v64 + v_max3_f32 v30, v66, v67, v30 + v_max3_f32 v30, v68, v69, v30 + v_max3_f32 v30, v70, v71, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v244, 0, v244 + v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + s_mov_b32 m0, s62 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_waitcnt vmcnt(12) + s_barrier + ds_read_b128 v[80:83], v10 offset:8192 + ds_read_b128 v[84:87], v10 offset:8208 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + ds_read_b128 v[88:91], v10 offset:10272 + ds_read_b128 v[92:95], v10 offset:10288 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:8192 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] + ds_read_b128 a[48:51], v240 offset:8192 + ds_read_b128 a[52:55], v240 offset:8208 + ds_read_u8 v236, v238 offset:8192 + v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] + ds_read_b128 a[80:83], v240 offset:10272 + ds_read_b128 a[84:87], v240 offset:10288 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:10272 + v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] + v_fma_f32 v64, v64, s5, -v32 + v_fma_f32 v65, v65, s5, -v32 + v_fma_f32 v66, v66, s5, -v32 + v_fma_f32 v67, v67, s5, -v32 + v_fma_f32 v68, v68, s5, -v32 + v_fma_f32 v69, v69, s5, -v32 + v_fma_f32 v70, v70, s5, -v32 + v_fma_f32 v71, v71, s5, -v32 + v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] + ds_read_b128 a[56:59], v13 offset:8320 + ds_read_b128 a[60:63], v13 offset:8336 + v_exp_f32_e32 v64, v64 + v_exp_f32_e32 v65, v65 + v_exp_f32_e32 v66, v66 + ds_read_u8 v237, v238 offset:10272 + v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] + v_exp_f32_e32 v67, v67 + v_exp_f32_e32 v68, v68 + v_exp_f32_e32 v69, v69 + v_exp_f32_e32 v70, v70 + v_exp_f32_e32 v71, v71 + v_mul_f32_e32 v46, v33, v46 + v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] + ds_read_b128 a[88:91], v13 offset:10400 + ds_read_b128 a[92:95], v13 offset:10416 + v_add_f32_e32 v26, v65, v64 + v_add_f32_e32 v26, v66, v26 + v_add_f32_e32 v26, v67, v26 + v_add_f32_e32 v26, v68, v26 + v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] + v_add_f32_e32 v26, v69, v26 + v_add_f32_e32 v26, v70, v26 + v_add_f32_e32 v26, v71, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v64, v64, v65 + v_cvt_pk_bf16_f32 v65, v66, v67 + v_cvt_pk_bf16_f32 v66, v68, v69 + v_cvt_pk_bf16_f32 v67, v70, v71 + v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] + ds_read_b128 a[64:67], v241 offset:8384 + ds_read_b128 a[68:71], v241 offset:8400 + ds_write_b32 v17, v33 offset:4096 + v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] + v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] + ds_read_b128 a[96:99], v241 offset:10464 + ds_read_b128 a[100:103], v241 offset:10480 + ds_write_b64 v20, v[64:65] + v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] + v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] + ds_read_b128 a[104:107], v16 + ds_read_b128 a[108:111], v16 offset:16 + ds_write_b64 v20, v[66:67] offset:2048 + v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] + v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] + ds_read_b128 a[112:115], v16 offset:2080 + ds_read_b128 a[116:119], v16 offset:2096 + v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b32 v36, v43 offset:4608 + ds_read_b32 v37, v43 offset:4864 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v42, 0, v42 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s59 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[66:67], v19 + v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + buffer_load_dword v40, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + buffer_load_dword v243, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[68:69], v18 offset:512 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64 v[70:71], v19 offset:512 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64 v[72:73], v18 offset:1024 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + ds_read_b64 v[74:75], v19 offset:1024 + ds_read_b64 v[76:77], v18 offset:1536 + v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_min_f32_e32 v26, v26, v36 + v_min_f32_e32 v26, v26, v37 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + ds_read_b64 v[78:79], v19 offset:1536 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + s_cbranch_vccz label_38A8 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9DB4 +label_38A8: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v245, 0, v245 + v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] + s_mov_b32 m0, s63 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] + s_waitcnt vmcnt(12) + s_barrier + ds_read_b128 v[80:83], v10 offset:24832 + ds_read_b128 v[84:87], v10 offset:24848 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] + ds_read_b128 v[88:91], v10 offset:26912 + ds_read_b128 v[92:95], v10 offset:26928 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:24832 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[72:75], v[160:163] + ds_read_b128 a[48:51], v240 offset:24832 + ds_read_b128 a[52:55], v240 offset:24848 + ds_read_u8 v236, v238 offset:24832 + v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[72:75], v[164:167] + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[72:75], v[168:171] + ds_read_b128 a[80:83], v240 offset:26912 + ds_read_b128 a[84:87], v240 offset:26928 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:26912 + v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[72:75], v[172:175] + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[72:75], v[176:179] + ds_read_b128 a[56:59], v13 offset:24960 + ds_read_b128 a[60:63], v13 offset:24976 + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + v_exp_f32_e32 v50, v50 + ds_read_u8 v237, v238 offset:26912 + v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[72:75], v[180:183] + v_exp_f32_e32 v51, v51 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[72:75], v[184:187] + ds_read_b128 a[88:91], v13 offset:27040 + ds_read_b128 a[92:95], v13 offset:27056 + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[72:75], v[188:191] + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_add_f32_e32 v26, v55, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[76:79], v[192:195] + ds_read_b128 a[64:67], v241 offset:25024 + ds_read_b128 a[68:71], v241 offset:25040 + ds_write_b32 v17, v33 offset:4096 + v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[76:79], v[196:199] + v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[76:79], v[200:203] + ds_read_b128 a[96:99], v241 offset:27104 + ds_read_b128 a[100:103], v241 offset:27120 + ds_write_b64 v20, v[48:49] + v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[76:79], v[204:207] + v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[76:79], v[208:211] + ds_read_b128 a[104:107], v16 offset:4160 + ds_read_b128 a[108:111], v16 offset:4176 + ds_write_b64 v20, v[50:51] offset:2048 + v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[76:79], v[212:215] + v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[76:79], v[216:219] + ds_read_b128 a[112:115], v16 offset:6240 + ds_read_b128 a[116:119], v16 offset:6256 + v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[76:79], v[220:223] + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_branch label_1F54 +label_3D68: + s_nop 0 + s_nop 0 + s_branch label_795C +label_3D74: + s_waitcnt lgkmcnt(2) + s_barrier + v_bfe_u32 v228, v228, v25, 8 + v_bfe_u32 v229, v229, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b32 v36, v43 offset:4608 + ds_read_b32 v37, v43 offset:4864 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v39, 0, v39 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s56 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] + buffer_load_dword v41, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] + buffer_load_dword v244, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[52:53], v18 offset:512 + v_bfe_u32 v232, v232, v25, 8 + v_bfe_u32 v233, v233, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64 v[54:55], v19 offset:512 + v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64 v[56:57], v18 offset:1024 + ds_read_b64 v[58:59], v19 offset:1024 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + ds_read_b64 v[60:61], v18 offset:1536 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_min_f32_e32 v26, v26, v36 + v_min_f32_e32 v26, v26, v37 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] + ds_read_b64 v[62:63], v19 offset:1536 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_cbranch_vccz label_4034 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9DB4 +label_4034: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + v_max3_f32 v30, v64, v65, v64 + v_max3_f32 v30, v66, v67, v30 + v_max3_f32 v30, v68, v69, v30 + v_max3_f32 v30, v70, v71, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v242, 0, v242 + v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + s_mov_b32 m0, s60 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_waitcnt vmcnt(12) + s_barrier + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + ds_read_b128 v[228:231], v1 offset:41472 + v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] + ds_read_b128 a[40:43], v239 offset:41472 + ds_read_b128 a[44:47], v239 offset:41488 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_u8 v236, v238 offset:41472 + v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] + v_fma_f32 v64, v64, s5, -v32 + v_fma_f32 v65, v65, s5, -v32 + v_fma_f32 v66, v66, s5, -v32 + v_fma_f32 v67, v67, s5, -v32 + v_fma_f32 v68, v68, s5, -v32 + v_fma_f32 v69, v69, s5, -v32 + v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] + ds_read_b128 a[72:75], v239 offset:43552 + ds_read_b128 a[76:79], v239 offset:43568 + v_fma_f32 v70, v70, s5, -v32 + v_fma_f32 v71, v71, s5, -v32 + ds_read_b128 v[232:235], v1 offset:43552 + v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] + v_exp_f32_e32 v64, v64 + v_exp_f32_e32 v65, v65 + v_exp_f32_e32 v66, v66 + v_exp_f32_e32 v67, v67 + v_exp_f32_e32 v68, v68 + v_exp_f32_e32 v69, v69 + v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] + ds_read_b128 a[48:51], v240 offset:41472 + ds_read_b128 a[52:55], v240 offset:41488 + v_exp_f32_e32 v70, v70 + v_exp_f32_e32 v71, v71 + v_mul_f32_e32 v46, v33, v46 + ds_read_u8 v237, v238 offset:43552 + v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] + v_add_f32_e32 v26, v65, v64 + v_add_f32_e32 v26, v66, v26 + v_add_f32_e32 v26, v67, v26 + v_add_f32_e32 v26, v68, v26 + v_add_f32_e32 v26, v69, v26 + v_add_f32_e32 v26, v70, v26 + v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] + ds_read_b128 a[80:83], v240 offset:43552 + ds_read_b128 a[84:87], v240 offset:43568 + v_add_f32_e32 v26, v71, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v64, v64, v65 + v_cvt_pk_bf16_f32 v65, v66, v67 + v_cvt_pk_bf16_f32 v66, v68, v69 + v_cvt_pk_bf16_f32 v67, v70, v71 + v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] + v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] + ds_read_b128 a[64:67], v241 offset:41664 + ds_read_b128 a[68:71], v241 offset:41680 + ds_write_b32 v17, v33 offset:4096 + v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] + v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] + ds_read_b128 a[96:99], v241 offset:43744 + ds_read_b128 a[100:103], v241 offset:43760 + ds_write_b64 v20, v[64:65] + v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] + v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] + ds_read_b128 a[104:107], v16 offset:8320 + ds_read_b128 a[108:111], v16 offset:8336 + ds_write_b64 v20, v[66:67] offset:2048 + v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] + v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] + ds_read_b128 a[112:115], v16 offset:10400 + ds_read_b128 a[116:119], v16 offset:10416 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_bfe_u32 v228, v228, v25, 8 + v_bfe_u32 v229, v229, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b32 v36, v43 offset:4608 + ds_read_b32 v37, v43 offset:4864 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v40, 0, v40 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s57 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + buffer_load_dword v42, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + buffer_load_dword v245, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[68:69], v18 offset:512 + v_bfe_u32 v232, v232, v25, 8 + v_bfe_u32 v233, v233, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64 v[70:71], v19 offset:512 + v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64 v[72:73], v18 offset:1024 + ds_read_b64 v[74:75], v19 offset:1024 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + ds_read_b64 v[76:77], v18 offset:1536 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_min_f32_e32 v26, v26, v36 + v_min_f32_e32 v26, v26, v37 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + ds_read_b64 v[78:79], v19 offset:1536 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_cbranch_vccz label_47B0 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9DB4 +label_47B0: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v243, 0, v243 + v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] + s_mov_b32 m0, s61 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] + s_waitcnt vmcnt(12) + s_barrier + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + ds_read_b128 v[228:231], v1 offset:58112 + v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[72:75], v[160:163] + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[72:75], v[164:167] + ds_read_b128 a[40:43], v239 offset:58112 + ds_read_b128 a[44:47], v239 offset:58128 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_u8 v236, v238 offset:58112 + v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[72:75], v[168:171] + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[72:75], v[172:175] + ds_read_b128 a[72:75], v239 offset:60192 + ds_read_b128 a[76:79], v239 offset:60208 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + ds_read_b128 v[232:235], v1 offset:60192 + v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[72:75], v[176:179] + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + v_exp_f32_e32 v50, v50 + v_exp_f32_e32 v51, v51 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[72:75], v[180:183] + ds_read_b128 a[48:51], v240 offset:58112 + ds_read_b128 a[52:55], v240 offset:58128 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + ds_read_u8 v237, v238 offset:60192 + v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[72:75], v[184:187] + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[72:75], v[188:191] + ds_read_b128 a[80:83], v240 offset:60192 + ds_read_b128 a[84:87], v240 offset:60208 + v_add_f32_e32 v26, v55, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[76:79], v[192:195] + v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[76:79], v[196:199] + ds_read_b128 a[64:67], v241 offset:58304 + ds_read_b128 a[68:71], v241 offset:58320 + ds_write_b32 v17, v33 offset:4096 + v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[76:79], v[200:203] + v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[76:79], v[204:207] + ds_read_b128 a[96:99], v241 offset:60384 + ds_read_b128 a[100:103], v241 offset:60400 + ds_write_b64 v20, v[48:49] + v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[76:79], v[208:211] + v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[76:79], v[212:215] + ds_read_b128 a[104:107], v16 offset:12480 + ds_read_b128 a[108:111], v16 offset:12496 + ds_write_b64 v20, v[50:51] offset:2048 + v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[76:79], v[216:219] + v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[76:79], v[220:223] + ds_read_b128 a[112:115], v16 offset:14560 + ds_read_b128 a[116:119], v16 offset:14576 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_bfe_u32 v228, v228, v25, 8 + v_bfe_u32 v229, v229, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b32 v36, v43 offset:4608 + ds_read_b32 v37, v43 offset:4864 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v41, 0, v41 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s58 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] + buffer_load_dword v39, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] + buffer_load_dword v242, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[52:53], v18 offset:512 + v_bfe_u32 v232, v232, v25, 8 + v_bfe_u32 v233, v233, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64 v[54:55], v19 offset:512 + v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64 v[56:57], v18 offset:1024 + ds_read_b64 v[58:59], v19 offset:1024 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + ds_read_b64 v[60:61], v18 offset:1536 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_min_f32_e32 v26, v26, v36 + v_min_f32_e32 v26, v26, v37 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] + ds_read_b64 v[62:63], v19 offset:1536 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_cbranch_vccz label_4F2C + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9DB4 +label_4F2C: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + v_max3_f32 v30, v64, v65, v64 + v_max3_f32 v30, v66, v67, v30 + v_max3_f32 v30, v68, v69, v30 + v_max3_f32 v30, v70, v71, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v244, 0, v244 + v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + s_mov_b32 m0, s62 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_waitcnt vmcnt(12) + s_barrier + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + ds_read_b128 v[80:83], v10 offset:8192 + ds_read_b128 v[84:87], v10 offset:8208 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + ds_read_b128 v[88:91], v10 offset:10272 + ds_read_b128 v[92:95], v10 offset:10288 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + ds_read_b128 v[228:231], v1 offset:8192 + v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] + ds_read_b128 a[40:43], v239 offset:8192 + ds_read_b128 a[44:47], v239 offset:8208 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_u8 v236, v238 offset:8192 + v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] + v_fma_f32 v64, v64, s5, -v32 + v_fma_f32 v65, v65, s5, -v32 + v_fma_f32 v66, v66, s5, -v32 + v_fma_f32 v67, v67, s5, -v32 + v_fma_f32 v68, v68, s5, -v32 + v_fma_f32 v69, v69, s5, -v32 + v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] + ds_read_b128 a[72:75], v239 offset:10272 + ds_read_b128 a[76:79], v239 offset:10288 + v_fma_f32 v70, v70, s5, -v32 + v_fma_f32 v71, v71, s5, -v32 + ds_read_b128 v[232:235], v1 offset:10272 + v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] + v_exp_f32_e32 v64, v64 + v_exp_f32_e32 v65, v65 + v_exp_f32_e32 v66, v66 + v_exp_f32_e32 v67, v67 + v_exp_f32_e32 v68, v68 + v_exp_f32_e32 v69, v69 + v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] + ds_read_b128 a[48:51], v240 offset:8192 + ds_read_b128 a[52:55], v240 offset:8208 + v_exp_f32_e32 v70, v70 + v_exp_f32_e32 v71, v71 + v_mul_f32_e32 v46, v33, v46 + ds_read_u8 v237, v238 offset:10272 + v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] + v_add_f32_e32 v26, v65, v64 + v_add_f32_e32 v26, v66, v26 + v_add_f32_e32 v26, v67, v26 + v_add_f32_e32 v26, v68, v26 + v_add_f32_e32 v26, v69, v26 + v_add_f32_e32 v26, v70, v26 + v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] + ds_read_b128 a[80:83], v240 offset:10272 + ds_read_b128 a[84:87], v240 offset:10288 + v_add_f32_e32 v26, v71, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v64, v64, v65 + v_cvt_pk_bf16_f32 v65, v66, v67 + v_cvt_pk_bf16_f32 v66, v68, v69 + v_cvt_pk_bf16_f32 v67, v70, v71 + v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] + v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] + ds_read_b128 a[64:67], v241 offset:8384 + ds_read_b128 a[68:71], v241 offset:8400 + ds_write_b32 v17, v33 offset:4096 + v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] + v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] + ds_read_b128 a[96:99], v241 offset:10464 + ds_read_b128 a[100:103], v241 offset:10480 + ds_write_b64 v20, v[64:65] + v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] + v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] + ds_read_b128 a[104:107], v16 + ds_read_b128 a[108:111], v16 offset:16 + ds_write_b64 v20, v[66:67] offset:2048 + v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] + v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] + ds_read_b128 a[112:115], v16 offset:2080 + ds_read_b128 a[116:119], v16 offset:2096 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_bfe_u32 v228, v228, v25, 8 + v_bfe_u32 v229, v229, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b32 v36, v43 offset:4608 + ds_read_b32 v37, v43 offset:4864 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v42, 0, v42 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s59 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + buffer_load_dword v40, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + buffer_load_dword v243, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[68:69], v18 offset:512 + v_bfe_u32 v232, v232, v25, 8 + v_bfe_u32 v233, v233, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64 v[70:71], v19 offset:512 + v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64 v[72:73], v18 offset:1024 + ds_read_b64 v[74:75], v19 offset:1024 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + ds_read_b64 v[76:77], v18 offset:1536 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_min_f32_e32 v26, v26, v36 + v_min_f32_e32 v26, v26, v37 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + ds_read_b64 v[78:79], v19 offset:1536 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_cbranch_vccz label_56A8 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9DB4 +label_56A8: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v245, 0, v245 + v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] + s_mov_b32 m0, s63 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] + s_waitcnt vmcnt(12) + s_barrier + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] + ds_read_b128 v[80:83], v10 offset:24832 + ds_read_b128 v[84:87], v10 offset:24848 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] + ds_read_b128 v[88:91], v10 offset:26912 + ds_read_b128 v[92:95], v10 offset:26928 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + ds_read_b128 v[228:231], v1 offset:24832 + v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[72:75], v[160:163] + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[72:75], v[164:167] + ds_read_b128 a[40:43], v239 offset:24832 + ds_read_b128 a[44:47], v239 offset:24848 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_u8 v236, v238 offset:24832 + v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[72:75], v[168:171] + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[72:75], v[172:175] + ds_read_b128 a[72:75], v239 offset:26912 + ds_read_b128 a[76:79], v239 offset:26928 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + ds_read_b128 v[232:235], v1 offset:26912 + v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[72:75], v[176:179] + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + v_exp_f32_e32 v50, v50 + v_exp_f32_e32 v51, v51 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[72:75], v[180:183] + ds_read_b128 a[48:51], v240 offset:24832 + ds_read_b128 a[52:55], v240 offset:24848 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + ds_read_u8 v237, v238 offset:26912 + v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[72:75], v[184:187] + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[72:75], v[188:191] + ds_read_b128 a[80:83], v240 offset:26912 + ds_read_b128 a[84:87], v240 offset:26928 + v_add_f32_e32 v26, v55, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[76:79], v[192:195] + v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[76:79], v[196:199] + ds_read_b128 a[64:67], v241 offset:25024 + ds_read_b128 a[68:71], v241 offset:25040 + ds_write_b32 v17, v33 offset:4096 + v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[76:79], v[200:203] + v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[76:79], v[204:207] + ds_read_b128 a[96:99], v241 offset:27104 + ds_read_b128 a[100:103], v241 offset:27120 + ds_write_b64 v20, v[48:49] + v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[76:79], v[208:211] + v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[76:79], v[212:215] + ds_read_b128 a[104:107], v16 offset:4160 + ds_read_b128 a[108:111], v16 offset:4176 + ds_write_b64 v20, v[50:51] offset:2048 + v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[76:79], v[216:219] + v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[76:79], v[220:223] + ds_read_b128 a[112:115], v16 offset:6240 + ds_read_b128 a[116:119], v16 offset:6256 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_branch label_3D74 +label_5B68: + s_waitcnt lgkmcnt(2) + s_barrier + v_bfe_u32 v228, v228, v25, 8 + v_bfe_u32 v229, v229, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b32 v36, v43 offset:4608 + ds_read_b32 v37, v43 offset:4864 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v39, 0, v39 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s56 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] + buffer_load_dword v41, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 + v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] + buffer_load_dword v244, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v23 offset:128 + ds_read_b64 v[52:53], v18 offset:512 + v_bfe_u32 v232, v232, v25, 8 + v_bfe_u32 v233, v233, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v24 + ds_read_b64_tr_b16 a[142:143], v24 offset:128 + ds_read_b64 v[54:55], v19 offset:512 + v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v23 offset:512 + ds_read_b64_tr_b16 a[146:147], v23 offset:640 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v24 offset:512 + ds_read_b64 v[56:57], v18 offset:1024 + ds_read_b64 v[58:59], v19 offset:1024 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v24 offset:640 + ds_read_b64 v[60:61], v18 offset:1536 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_min_f32_e32 v26, v26, v36 + v_min_f32_e32 v26, v26, v37 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] + ds_read_b64 v[62:63], v19 offset:1536 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_cbranch_vccz label_5E28 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9DB4 +label_5E28: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + v_max3_f32 v30, v64, v65, v64 + v_max3_f32 v30, v66, v67, v30 + v_max3_f32 v30, v68, v69, v30 + v_max3_f32 v30, v70, v71, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v242, 0, v242 + v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + s_mov_b32 m0, s60 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_waitcnt vmcnt(12) + s_barrier + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + ds_read_b128 v[228:231], v1 offset:41472 + v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] + ds_read_b128 a[40:43], v239 offset:41472 + ds_read_b128 a[44:47], v239 offset:41488 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_u8 v236, v238 offset:41472 + v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] + v_fma_f32 v64, v64, s5, -v32 + v_fma_f32 v65, v65, s5, -v32 + v_fma_f32 v66, v66, s5, -v32 + v_fma_f32 v67, v67, s5, -v32 + v_fma_f32 v68, v68, s5, -v32 + v_fma_f32 v69, v69, s5, -v32 + v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] + ds_read_b128 a[72:75], v239 offset:43552 + ds_read_b128 a[76:79], v239 offset:43568 + v_fma_f32 v70, v70, s5, -v32 + v_fma_f32 v71, v71, s5, -v32 + ds_read_b128 v[232:235], v1 offset:43552 + v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] + v_exp_f32_e32 v64, v64 + v_exp_f32_e32 v65, v65 + v_exp_f32_e32 v66, v66 + v_exp_f32_e32 v67, v67 + v_exp_f32_e32 v68, v68 + v_exp_f32_e32 v69, v69 + v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] + ds_read_b128 a[48:51], v240 offset:41472 + ds_read_b128 a[52:55], v240 offset:41488 + v_exp_f32_e32 v70, v70 + v_exp_f32_e32 v71, v71 + v_mul_f32_e32 v46, v33, v46 + ds_read_u8 v237, v238 offset:43552 + v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] + v_add_f32_e32 v26, v65, v64 + v_add_f32_e32 v26, v66, v26 + v_add_f32_e32 v26, v67, v26 + v_add_f32_e32 v26, v68, v26 + v_add_f32_e32 v26, v69, v26 + v_add_f32_e32 v26, v70, v26 + v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] + ds_read_b128 a[80:83], v240 offset:43552 + ds_read_b128 a[84:87], v240 offset:43568 + v_add_f32_e32 v26, v71, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v64, v64, v65 + v_cvt_pk_bf16_f32 v65, v66, v67 + v_cvt_pk_bf16_f32 v66, v68, v69 + v_cvt_pk_bf16_f32 v67, v70, v71 + v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] + v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] + ds_read_b128 a[56:59], v13 offset:41600 + ds_read_b128 a[60:63], v13 offset:41616 + ds_write_b32 v17, v33 offset:4096 + v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] + v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] + ds_read_b128 a[88:91], v13 offset:43680 + ds_read_b128 a[92:95], v13 offset:43696 + ds_write_b64 v20, v[64:65] + v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] + v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] + ds_read_b128 a[104:107], v16 offset:8320 + ds_read_b128 a[108:111], v16 offset:8336 + ds_write_b64 v20, v[66:67] offset:2048 + v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] + v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] + ds_read_b128 a[112:115], v16 offset:10400 + ds_read_b128 a[116:119], v16 offset:10416 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_bfe_u32 v228, v228, v25, 8 + v_bfe_u32 v229, v229, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b32 v36, v43 offset:4608 + ds_read_b32 v37, v43 offset:4864 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v40, 0, v40 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s57 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + buffer_load_dword v42, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 offset:4160 + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + buffer_load_dword v245, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v23 offset:4288 + ds_read_b64 v[68:69], v18 offset:512 + v_bfe_u32 v232, v232, v25, 8 + v_bfe_u32 v233, v233, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v24 offset:4160 + ds_read_b64_tr_b16 a[142:143], v24 offset:4288 + ds_read_b64 v[70:71], v19 offset:512 + v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v23 offset:4672 + ds_read_b64_tr_b16 a[146:147], v23 offset:4800 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v24 offset:4672 + ds_read_b64 v[72:73], v18 offset:1024 + ds_read_b64 v[74:75], v19 offset:1024 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v24 offset:4800 + ds_read_b64 v[76:77], v18 offset:1536 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_min_f32_e32 v26, v26, v36 + v_min_f32_e32 v26, v26, v37 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + ds_read_b64 v[78:79], v19 offset:1536 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_cbranch_vccz label_65A4 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9DB4 +label_65A4: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v243, 0, v243 + v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] + s_mov_b32 m0, s61 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] + s_waitcnt vmcnt(12) + s_barrier + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + ds_read_b128 v[228:231], v1 offset:58112 + v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[72:75], v[160:163] + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[72:75], v[164:167] + ds_read_b128 a[40:43], v239 offset:58112 + ds_read_b128 a[44:47], v239 offset:58128 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_u8 v236, v238 offset:58112 + v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[72:75], v[168:171] + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[72:75], v[172:175] + ds_read_b128 a[72:75], v239 offset:60192 + ds_read_b128 a[76:79], v239 offset:60208 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + ds_read_b128 v[232:235], v1 offset:60192 + v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[72:75], v[176:179] + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + v_exp_f32_e32 v50, v50 + v_exp_f32_e32 v51, v51 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[72:75], v[180:183] + ds_read_b128 a[48:51], v240 offset:58112 + ds_read_b128 a[52:55], v240 offset:58128 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + ds_read_u8 v237, v238 offset:60192 + v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[72:75], v[184:187] + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[72:75], v[188:191] + ds_read_b128 a[80:83], v240 offset:60192 + ds_read_b128 a[84:87], v240 offset:60208 + v_add_f32_e32 v26, v55, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[76:79], v[192:195] + v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[76:79], v[196:199] + ds_read_b128 a[56:59], v13 offset:58240 + ds_read_b128 a[60:63], v13 offset:58256 + ds_write_b32 v17, v33 offset:4096 + v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[76:79], v[200:203] + v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[76:79], v[204:207] + ds_read_b128 a[88:91], v13 offset:60320 + ds_read_b128 a[92:95], v13 offset:60336 + ds_write_b64 v20, v[48:49] + v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[76:79], v[208:211] + v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[76:79], v[212:215] + ds_read_b128 a[104:107], v16 offset:12480 + ds_read_b128 a[108:111], v16 offset:12496 + ds_write_b64 v20, v[50:51] offset:2048 + v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[76:79], v[216:219] + v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[76:79], v[220:223] + ds_read_b128 a[112:115], v16 offset:14560 + ds_read_b128 a[116:119], v16 offset:14576 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_bfe_u32 v228, v228, v25, 8 + v_bfe_u32 v229, v229, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b32 v36, v43 offset:4608 + ds_read_b32 v37, v43 offset:4864 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v41, 0, v41 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s58 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] + buffer_load_dword v39, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 offset:8320 + v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] + buffer_load_dword v242, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v23 offset:8448 + ds_read_b64 v[52:53], v18 offset:512 + v_bfe_u32 v232, v232, v25, 8 + v_bfe_u32 v233, v233, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v24 offset:8320 + ds_read_b64_tr_b16 a[142:143], v24 offset:8448 + ds_read_b64 v[54:55], v19 offset:512 + v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v23 offset:8832 + ds_read_b64_tr_b16 a[146:147], v23 offset:8960 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v24 offset:8832 + ds_read_b64 v[56:57], v18 offset:1024 + ds_read_b64 v[58:59], v19 offset:1024 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v24 offset:8960 + ds_read_b64 v[60:61], v18 offset:1536 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_min_f32_e32 v26, v26, v36 + v_min_f32_e32 v26, v26, v37 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] + ds_read_b64 v[62:63], v19 offset:1536 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_cbranch_vccz label_6D20 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9DB4 +label_6D20: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + v_max3_f32 v30, v64, v65, v64 + v_max3_f32 v30, v66, v67, v30 + v_max3_f32 v30, v68, v69, v30 + v_max3_f32 v30, v70, v71, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v244, 0, v244 + v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + s_mov_b32 m0, s62 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_waitcnt vmcnt(12) + s_barrier + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + ds_read_b128 v[80:83], v10 offset:8192 + ds_read_b128 v[84:87], v10 offset:8208 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + ds_read_b128 v[88:91], v10 offset:10272 + ds_read_b128 v[92:95], v10 offset:10288 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + ds_read_b128 v[228:231], v1 offset:8192 + v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] + ds_read_b128 a[40:43], v239 offset:8192 + ds_read_b128 a[44:47], v239 offset:8208 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_u8 v236, v238 offset:8192 + v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] + v_fma_f32 v64, v64, s5, -v32 + v_fma_f32 v65, v65, s5, -v32 + v_fma_f32 v66, v66, s5, -v32 + v_fma_f32 v67, v67, s5, -v32 + v_fma_f32 v68, v68, s5, -v32 + v_fma_f32 v69, v69, s5, -v32 + v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] + ds_read_b128 a[72:75], v239 offset:10272 + ds_read_b128 a[76:79], v239 offset:10288 + v_fma_f32 v70, v70, s5, -v32 + v_fma_f32 v71, v71, s5, -v32 + ds_read_b128 v[232:235], v1 offset:10272 + v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] + v_exp_f32_e32 v64, v64 + v_exp_f32_e32 v65, v65 + v_exp_f32_e32 v66, v66 + v_exp_f32_e32 v67, v67 + v_exp_f32_e32 v68, v68 + v_exp_f32_e32 v69, v69 + v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] + ds_read_b128 a[48:51], v240 offset:8192 + ds_read_b128 a[52:55], v240 offset:8208 + v_exp_f32_e32 v70, v70 + v_exp_f32_e32 v71, v71 + v_mul_f32_e32 v46, v33, v46 + ds_read_u8 v237, v238 offset:10272 + v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] + v_add_f32_e32 v26, v65, v64 + v_add_f32_e32 v26, v66, v26 + v_add_f32_e32 v26, v67, v26 + v_add_f32_e32 v26, v68, v26 + v_add_f32_e32 v26, v69, v26 + v_add_f32_e32 v26, v70, v26 + v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] + ds_read_b128 a[80:83], v240 offset:10272 + ds_read_b128 a[84:87], v240 offset:10288 + v_add_f32_e32 v26, v71, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v64, v64, v65 + v_cvt_pk_bf16_f32 v65, v66, v67 + v_cvt_pk_bf16_f32 v66, v68, v69 + v_cvt_pk_bf16_f32 v67, v70, v71 + v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] + v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] + ds_read_b128 a[56:59], v13 offset:8320 + ds_read_b128 a[60:63], v13 offset:8336 + ds_write_b32 v17, v33 offset:4096 + v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] + v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] + ds_read_b128 a[88:91], v13 offset:10400 + ds_read_b128 a[92:95], v13 offset:10416 + ds_write_b64 v20, v[64:65] + v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] + v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] + ds_read_b128 a[104:107], v16 + ds_read_b128 a[108:111], v16 offset:16 + ds_write_b64 v20, v[66:67] offset:2048 + v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] + v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] + ds_read_b128 a[112:115], v16 offset:2080 + ds_read_b128 a[116:119], v16 offset:2096 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_bfe_u32 v228, v228, v25, 8 + v_bfe_u32 v229, v229, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b32 v36, v43 offset:4608 + ds_read_b32 v37, v43 offset:4864 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v42, 0, v42 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s59 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + buffer_load_dword v40, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 offset:12480 + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + buffer_load_dword v243, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v23 offset:12608 + ds_read_b64 v[68:69], v18 offset:512 + v_bfe_u32 v232, v232, v25, 8 + v_bfe_u32 v233, v233, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v24 offset:12480 + ds_read_b64_tr_b16 a[142:143], v24 offset:12608 + ds_read_b64 v[70:71], v19 offset:512 + v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v23 offset:12992 + ds_read_b64_tr_b16 a[146:147], v23 offset:13120 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v24 offset:12992 + ds_read_b64 v[72:73], v18 offset:1024 + ds_read_b64 v[74:75], v19 offset:1024 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v24 offset:13120 + ds_read_b64 v[76:77], v18 offset:1536 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_min_f32_e32 v26, v26, v36 + v_min_f32_e32 v26, v26, v37 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + ds_read_b64 v[78:79], v19 offset:1536 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_cbranch_vccz label_749C + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9DB4 +label_749C: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v245, 0, v245 + v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] + s_mov_b32 m0, s63 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] + s_waitcnt vmcnt(12) + s_barrier + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] + ds_read_b128 v[80:83], v10 offset:24832 + ds_read_b128 v[84:87], v10 offset:24848 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] + ds_read_b128 v[88:91], v10 offset:26912 + ds_read_b128 v[92:95], v10 offset:26928 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + ds_read_b128 v[228:231], v1 offset:24832 + v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[72:75], v[160:163] + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[72:75], v[164:167] + ds_read_b128 a[40:43], v239 offset:24832 + ds_read_b128 a[44:47], v239 offset:24848 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_u8 v236, v238 offset:24832 + v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[72:75], v[168:171] + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[72:75], v[172:175] + ds_read_b128 a[72:75], v239 offset:26912 + ds_read_b128 a[76:79], v239 offset:26928 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + ds_read_b128 v[232:235], v1 offset:26912 + v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[72:75], v[176:179] + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + v_exp_f32_e32 v50, v50 + v_exp_f32_e32 v51, v51 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[72:75], v[180:183] + ds_read_b128 a[48:51], v240 offset:24832 + ds_read_b128 a[52:55], v240 offset:24848 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + ds_read_u8 v237, v238 offset:26912 + v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[72:75], v[184:187] + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[72:75], v[188:191] + ds_read_b128 a[80:83], v240 offset:26912 + ds_read_b128 a[84:87], v240 offset:26928 + v_add_f32_e32 v26, v55, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[76:79], v[192:195] + v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[76:79], v[196:199] + ds_read_b128 a[56:59], v13 offset:24960 + ds_read_b128 a[60:63], v13 offset:24976 + ds_write_b32 v17, v33 offset:4096 + v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[76:79], v[200:203] + v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[76:79], v[204:207] + ds_read_b128 a[88:91], v13 offset:27040 + ds_read_b128 a[92:95], v13 offset:27056 + ds_write_b64 v20, v[48:49] + v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[76:79], v[208:211] + v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[76:79], v[212:215] + ds_read_b128 a[104:107], v16 offset:4160 + ds_read_b128 a[108:111], v16 offset:4176 + ds_write_b64 v20, v[50:51] offset:2048 + v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[76:79], v[216:219] + v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[76:79], v[220:223] + ds_read_b128 a[112:115], v16 offset:6240 + ds_read_b128 a[116:119], v16 offset:6256 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_branch label_5B68 +label_795C: + s_add_i32 s75, s88, -1 + s_and_b32 s75, s75, 3 + s_mul_i32 s75, s75, 0x1040 + v_add_u32_e32 v23, s75, v23 + v_add_u32_e32 v24, s75, v24 + s_cmp_eq_u32 s72, 64 + s_cbranch_scc1 label_7A20 + s_cmp_eq_u32 s72, 16 + s_cbranch_scc0 label_79C4 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b64 v[48:49], v18 + ds_read_b64 v[50:51], v19 + ds_read_b32 v34, v43 offset:4096 + s_waitcnt lgkmcnt(0) + s_barrier + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_cbranch_vccz label_79C0 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_79C0: + s_branch label_7AB0 +label_79C4: + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b64 v[48:49], v18 + ds_read_b64 v[50:51], v19 + ds_read_b64 v[52:53], v18 offset:512 + ds_read_b64 v[54:55], v19 offset:512 + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + s_waitcnt lgkmcnt(0) + s_barrier + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_cbranch_vccz label_7A1C + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_7A1C: + s_branch label_7AB0 +label_7A20: + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b64 v[48:49], v18 + ds_read_b64 v[50:51], v19 + ds_read_b64 v[52:53], v18 offset:512 + ds_read_b64 v[54:55], v19 offset:512 + ds_read_b64 v[56:57], v18 offset:1024 + ds_read_b64 v[58:59], v19 offset:1024 + ds_read_b64 v[60:61], v18 offset:1536 + ds_read_b64 v[62:63], v19 offset:1536 + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b32 v36, v43 offset:4608 + ds_read_b32 v37, v43 offset:4864 + s_waitcnt lgkmcnt(0) + s_barrier + v_min_f32_e32 v26, v34, v35 + v_min_f32_e32 v26, v26, v36 + v_min_f32_e32 v26, v26, v37 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_cbranch_vccz label_7AB0 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9DB4 +label_7AB0: + s_cmp_eq_i32 s7, 3 + s_cbranch_scc1 label_7B3C + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + s_branch label_7BBC +label_7B3C: + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 + ds_read_b64_tr_b16 a[138:139], v23 offset:128 + ds_read_b64_tr_b16 a[140:141], v24 + ds_read_b64_tr_b16 a[142:143], v24 offset:128 + ds_read_b64_tr_b16 a[144:145], v23 offset:512 + ds_read_b64_tr_b16 a[146:147], v23 offset:640 + ds_read_b64_tr_b16 a[148:149], v24 offset:512 + ds_read_b64_tr_b16 a[150:151], v24 offset:640 +label_7BBC: + s_waitcnt lgkmcnt(0) + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + s_cmp_le_u32 s72, 16 + s_cbranch_scc1 label_7CD0 + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + s_cmp_le_u32 s72, 32 + s_cbranch_scc1 label_7CD0 + v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] + v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] + v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] + v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] + v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] + v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] + v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] + v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] + v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] + v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] + v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] + v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] + v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] + v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] + v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] + v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] +label_7CD0: + s_add_i32 s75, s88, -1 + s_and_b32 s75, s75, 3 + s_mul_i32 s75, s75, 0xffffefc0 + v_add_u32_e32 v23, s75, v23 + v_add_u32_e32 v24, s75, v24 + s_cmp_eq_i32 s70, 0 + s_cbranch_scc1 label_7CF4 + s_branch label_8EFC +label_7CF4: + s_cmp_eq_u32 s4, 0 + s_cbranch_scc0 label_7D78 + v_and_b32_e64 v33, v0, 15 + v_lshlrev_b32_e32 v33, 2, v33 + s_lshl_b32 s75, s7, 4 + s_sub_u32 s76, s80, 1 + s_min_u32 s76, s76, 63 + s_and_b32 s75, s75, s76 + s_lshl_b32 s76, s2, 6 + s_add_u32 s75, s75, s76 + s_lshl_b32 s75, s75, 2 + v_add_u32_e32 v33, s75, v33 + buffer_load_dword v30, v33, s[52:55], 0 offen + s_waitcnt vmcnt(0) + v_mul_f32_e32 v30, 0x41b504f3, v30 + v_max_f32_e32 v31, v45, v30 + v_sub_f32_e32 v33, v45, v31 + v_mul_f32_e64 v33, v33, s5 + v_exp_f32_e32 v32, v33 + v_sub_f32_e32 v30, v30, v31 + v_mul_f32_e64 v30, v30, s5 + v_exp_f32_e32 v30, v30 + s_nop 0 + v_mul_f32_e32 v30, 0x3e800000, v30 + v_fma_f32 v46, v46, v32, v30 + v_mov_b32_e32 v45, v31 +label_7D78: + v_mov_b32_e32 v26, v46 + v_mov_b32_e32 v27, v46 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + v_mov_b32_e32 v46, 0 + v_add_f32_e32 v46, v26, v46 + v_add_f32_e32 v46, v27, v46 + v_add_f32_e32 v46, v28, v46 + v_add_f32_e32 v46, v29, v46 + v_mov_b32_e32 v26, 0 + v_cmp_eq_u32_e64 s[64:65], v26, v46 + v_mul_f32_e64 v26, v45, s79 + v_log_f32_e32 v27, v46 + v_cndmask_b32_e64 v46, v46, 1.0, s[64:65] + s_nop 1 + v_rcp_f32_e32 v46, v46 + s_nop 1 + v_fma_f32 v33, v27, s6, v26 + s_cmp_eq_u32 s4, 0 + s_cbranch_scc0 label_7DF0 + v_mul_f32_e32 v46, v46, v32 +label_7DF0: + s_mov_b32 s87, s72 + v_lshlrev_b32_e32 v30, 2, v0 + s_mul_i32 s75, 0x100, s7 + v_add_u32_e32 v30, s75, v30 + ds_write_b32 v30, v46 offset:4096 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v26, v43 offset:4096 + s_cmp_le_u32 s87, 16 + s_cbranch_scc1 label_7E44 + ds_read_b32 v27, v43 offset:4352 + s_cmp_le_u32 s87, 32 + s_cbranch_scc1 label_7E44 + ds_read_b32 v28, v43 offset:4608 + ds_read_b32 v29, v43 offset:4864 +label_7E44: + s_waitcnt lgkmcnt(0) + v_pk_mul_f32 v[96:97], v[26:27], v[96:97] op_sel_hi:[0,1] + v_pk_mul_f32 v[98:99], v[26:27], v[98:99] op_sel_hi:[0,1] + v_pk_mul_f32 v[100:101], v[26:27], v[100:101] op_sel_hi:[0,1] + v_pk_mul_f32 v[102:103], v[26:27], v[102:103] op_sel_hi:[0,1] + v_pk_mul_f32 v[104:105], v[26:27], v[104:105] op_sel_hi:[0,1] + v_pk_mul_f32 v[106:107], v[26:27], v[106:107] op_sel_hi:[0,1] + v_pk_mul_f32 v[108:109], v[26:27], v[108:109] op_sel_hi:[0,1] + v_pk_mul_f32 v[110:111], v[26:27], v[110:111] op_sel_hi:[0,1] + v_pk_mul_f32 v[112:113], v[26:27], v[112:113] op_sel_hi:[0,1] + v_pk_mul_f32 v[114:115], v[26:27], v[114:115] op_sel_hi:[0,1] + v_pk_mul_f32 v[116:117], v[26:27], v[116:117] op_sel_hi:[0,1] + v_pk_mul_f32 v[118:119], v[26:27], v[118:119] op_sel_hi:[0,1] + v_pk_mul_f32 v[120:121], v[26:27], v[120:121] op_sel_hi:[0,1] + v_pk_mul_f32 v[122:123], v[26:27], v[122:123] op_sel_hi:[0,1] + v_pk_mul_f32 v[124:125], v[26:27], v[124:125] op_sel_hi:[0,1] + v_pk_mul_f32 v[126:127], v[26:27], v[126:127] op_sel_hi:[0,1] + s_cmp_le_u32 s87, 16 + s_cbranch_scc1 label_8058 + v_pk_mul_f32 v[128:129], v[26:27], v[128:129] op_sel:[1,0] + v_pk_mul_f32 v[130:131], v[26:27], v[130:131] op_sel:[1,0] + v_pk_mul_f32 v[132:133], v[26:27], v[132:133] op_sel:[1,0] + v_pk_mul_f32 v[134:135], v[26:27], v[134:135] op_sel:[1,0] + v_pk_mul_f32 v[136:137], v[26:27], v[136:137] op_sel:[1,0] + v_pk_mul_f32 v[138:139], v[26:27], v[138:139] op_sel:[1,0] + v_pk_mul_f32 v[140:141], v[26:27], v[140:141] op_sel:[1,0] + v_pk_mul_f32 v[142:143], v[26:27], v[142:143] op_sel:[1,0] + v_pk_mul_f32 v[144:145], v[26:27], v[144:145] op_sel:[1,0] + v_pk_mul_f32 v[146:147], v[26:27], v[146:147] op_sel:[1,0] + v_pk_mul_f32 v[148:149], v[26:27], v[148:149] op_sel:[1,0] + v_pk_mul_f32 v[150:151], v[26:27], v[150:151] op_sel:[1,0] + v_pk_mul_f32 v[152:153], v[26:27], v[152:153] op_sel:[1,0] + v_pk_mul_f32 v[154:155], v[26:27], v[154:155] op_sel:[1,0] + v_pk_mul_f32 v[156:157], v[26:27], v[156:157] op_sel:[1,0] + v_pk_mul_f32 v[158:159], v[26:27], v[158:159] op_sel:[1,0] + s_cmp_le_u32 s87, 32 + s_cbranch_scc1 label_8058 + v_pk_mul_f32 v[160:161], v[28:29], v[160:161] op_sel_hi:[0,1] + v_pk_mul_f32 v[162:163], v[28:29], v[162:163] op_sel_hi:[0,1] + v_pk_mul_f32 v[164:165], v[28:29], v[164:165] op_sel_hi:[0,1] + v_pk_mul_f32 v[166:167], v[28:29], v[166:167] op_sel_hi:[0,1] + v_pk_mul_f32 v[168:169], v[28:29], v[168:169] op_sel_hi:[0,1] + v_pk_mul_f32 v[170:171], v[28:29], v[170:171] op_sel_hi:[0,1] + v_pk_mul_f32 v[172:173], v[28:29], v[172:173] op_sel_hi:[0,1] + v_pk_mul_f32 v[174:175], v[28:29], v[174:175] op_sel_hi:[0,1] + v_pk_mul_f32 v[176:177], v[28:29], v[176:177] op_sel_hi:[0,1] + v_pk_mul_f32 v[178:179], v[28:29], v[178:179] op_sel_hi:[0,1] + v_pk_mul_f32 v[180:181], v[28:29], v[180:181] op_sel_hi:[0,1] + v_pk_mul_f32 v[182:183], v[28:29], v[182:183] op_sel_hi:[0,1] + v_pk_mul_f32 v[184:185], v[28:29], v[184:185] op_sel_hi:[0,1] + v_pk_mul_f32 v[186:187], v[28:29], v[186:187] op_sel_hi:[0,1] + v_pk_mul_f32 v[188:189], v[28:29], v[188:189] op_sel_hi:[0,1] + v_pk_mul_f32 v[190:191], v[28:29], v[190:191] op_sel_hi:[0,1] + v_pk_mul_f32 v[192:193], v[28:29], v[192:193] op_sel:[1,0] + v_pk_mul_f32 v[194:195], v[28:29], v[194:195] op_sel:[1,0] + v_pk_mul_f32 v[196:197], v[28:29], v[196:197] op_sel:[1,0] + v_pk_mul_f32 v[198:199], v[28:29], v[198:199] op_sel:[1,0] + v_pk_mul_f32 v[200:201], v[28:29], v[200:201] op_sel:[1,0] + v_pk_mul_f32 v[202:203], v[28:29], v[202:203] op_sel:[1,0] + v_pk_mul_f32 v[204:205], v[28:29], v[204:205] op_sel:[1,0] + v_pk_mul_f32 v[206:207], v[28:29], v[206:207] op_sel:[1,0] + v_pk_mul_f32 v[208:209], v[28:29], v[208:209] op_sel:[1,0] + v_pk_mul_f32 v[210:211], v[28:29], v[210:211] op_sel:[1,0] + v_pk_mul_f32 v[212:213], v[28:29], v[212:213] op_sel:[1,0] + v_pk_mul_f32 v[214:215], v[28:29], v[214:215] op_sel:[1,0] + v_pk_mul_f32 v[216:217], v[28:29], v[216:217] op_sel:[1,0] + v_pk_mul_f32 v[218:219], v[28:29], v[218:219] op_sel:[1,0] + v_pk_mul_f32 v[220:221], v[28:29], v[220:221] op_sel:[1,0] + v_pk_mul_f32 v[222:223], v[28:29], v[222:223] op_sel:[1,0] +label_8058: + s_cmp_gt_u32 s82, 1 + s_cbranch_scc1 label_8068 + s_cmp_eq_u32 s83, 1 + s_cbranch_scc1 label_86D0 +label_8068: + s_mul_i32 s95, 0x800, s80 + s_mul_i32 s96, s81, s95 + s_mul_i32 s75, s89, s96 + s_add_u32 s8, s75, s8 + s_addc_u32 s9, 0, s9 + s_mov_b32 s75, 0x20000 + s_mul_i32 s75, s75, s2 + s_add_u32 s8, s75, s8 + s_addc_u32 s9, 0, s9 + s_mul_i32 s75, s96, s91 + s_mov_b32 s10, s75 + v_and_b32_e32 v26, 15, v0 + v_lshlrev_b32_e32 v2, 4, v26 + v_lshrrev_b32_e32 v26, 4, v0 + v_mul_i32_i24_e32 v26, 0x800, v26 + v_add_u32_e32 v2, v2, v26 + s_mul_i32 s75, s4, s95 + v_add_u32_e64 v2, v2, s75 + s_mul_i32 s75, s7, 0x200 + v_add_u32_e64 v2, v2, s75 + v_mov_b32_e32 v1, v2 + s_lshl_b32 s64, s7, 4 + s_ff1_i32_b32 s78, s80 + s_lshr_b32 s66, s64, s78 + s_sub_u32 s78, s80, 1 + s_and_b32 s76, s64, s78 + s_mul_i32 s97, 4, s80 + s_mul_i32 s77, s81, s97 + s_mul_i32 s75, s89, s77 + s_mul_i32 s78, s4, s97 + s_add_u32 s75, s75, s78 + s_mov_b32 s78, 0x100 + s_mul_i32 s78, s78, s2 + s_add_u32 s75, s75, s78 + s_add_u32 s12, s75, s12 + s_addc_u32 s13, 0, s13 + s_mul_i32 s75, s91, s77 + s_mov_b32 s14, s75 + v_and_b32_e32 v26, 15, v0 + v_lshlrev_b32_e32 v26, 2, v26 + s_lshl_b32 s76, s76, 2 + v_add_u32_e64 v26, v26, s76 + s_mul_i32 s76, s66, s77 + v_add_u32_e64 v26, v26, s76 + buffer_store_dword v33, v26, s[12:15], 0 offen + s_waitcnt vmcnt(0) lgkmcnt(0) + s_barrier + v_and_b32_e32 v26, 3, v0 + v_lshlrev_b32_e32 v16, 2, v26 + v_bfe_u32 v26, v0, 2, 2 + v_lshl_add_u32 v16, v26, 4, v16 + v_lshrrev_b32_e32 v26, 4, v0 + v_mul_i32_i24_e32 v26, 0x44, v26 + v_add_u32_e32 v16, v26, v16 + s_mul_i32 s75, s7, 0x880 + v_add_u32_e32 v16, s75, v16 + v_lshlrev_b32_e32 v16, 2, v16 + v_lshrrev_b32_e32 v26, 4, v0 + v_mul_i32_i24_e32 v10, 4, v26 + v_and_b32_e32 v26, 3, v0 + v_mul_i32_i24_e32 v26, 0x44, v26 + v_add_u32_e32 v10, v26, v10 + v_and_b32_e32 v26, 15, v0 + v_lshrrev_b32_e32 v26, 2, v26 + v_mul_i32_i24_e32 v26, 0x110, v26 + v_add_u32_e32 v10, v26, v10 + s_mul_i32 s75, s7, 0x880 + v_add_u32_e32 v10, s75, v10 + v_lshlrev_b32_e32 v10, 2, v10 + v_mov_b32_e32 v2, v1 + v_mov_b64_e32 v[26:27], v[96:97] + v_mov_b64_e32 v[28:29], v[98:99] + ds_write_b128 v16, v[26:29] + v_mov_b64_e32 v[26:27], v[100:101] + v_mov_b64_e32 v[28:29], v[102:103] + ds_write_b128 v16, v[26:29] offset:1088 + v_mov_b64_e32 v[26:27], v[104:105] + v_mov_b64_e32 v[28:29], v[106:107] + ds_write_b128 v16, v[26:29] offset:2176 + v_mov_b64_e32 v[26:27], v[108:109] + v_mov_b64_e32 v[28:29], v[110:111] + ds_write_b128 v16, v[26:29] offset:3264 + v_mov_b64_e32 v[26:27], v[112:113] + v_mov_b64_e32 v[28:29], v[114:115] + ds_write_b128 v16, v[26:29] offset:4352 + v_mov_b64_e32 v[26:27], v[116:117] + v_mov_b64_e32 v[28:29], v[118:119] + ds_write_b128 v16, v[26:29] offset:5440 + v_mov_b64_e32 v[26:27], v[120:121] + v_mov_b64_e32 v[28:29], v[122:123] + ds_write_b128 v16, v[26:29] offset:6528 + v_mov_b64_e32 v[26:27], v[124:125] + v_mov_b64_e32 v[28:29], v[126:127] + ds_write_b128 v16, v[26:29] offset:7616 + s_waitcnt lgkmcnt(4) + ds_read_b128 v[96:99], v10 + ds_read_b128 v[100:103], v10 offset:64 + ds_read_b128 v[104:107], v10 offset:128 + ds_read_b128 v[108:111], v10 offset:192 + s_waitcnt lgkmcnt(4) + ds_read_b128 v[112:115], v10 offset:4352 + ds_read_b128 v[116:119], v10 offset:4416 + ds_read_b128 v[120:123], v10 offset:4480 + ds_read_b128 v[124:127], v10 offset:4544 + s_waitcnt lgkmcnt(0) + buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[112:115], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[116:119], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[120:123], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[124:127], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 + s_cmp_le_u32 s87, 16 + s_cbranch_scc1 label_86CC + s_sub_u32 s76, s96, s95 + s_sub_u32 s77, s80, 1 + s_and_b32 s75, s77, 16 + s_cmp_eq_u32 s75, 0 + s_cselect_b32 s75, s76, 0 + v_add_u32_e32 v2, s75, v2 + v_mov_b64_e32 v[26:27], v[128:129] + v_mov_b64_e32 v[28:29], v[130:131] + ds_write_b128 v16, v[26:29] + v_mov_b64_e32 v[26:27], v[132:133] + v_mov_b64_e32 v[28:29], v[134:135] + ds_write_b128 v16, v[26:29] offset:1088 + v_mov_b64_e32 v[26:27], v[136:137] + v_mov_b64_e32 v[28:29], v[138:139] + ds_write_b128 v16, v[26:29] offset:2176 + v_mov_b64_e32 v[26:27], v[140:141] + v_mov_b64_e32 v[28:29], v[142:143] + ds_write_b128 v16, v[26:29] offset:3264 + v_mov_b64_e32 v[26:27], v[144:145] + v_mov_b64_e32 v[28:29], v[146:147] + ds_write_b128 v16, v[26:29] offset:4352 + v_mov_b64_e32 v[26:27], v[148:149] + v_mov_b64_e32 v[28:29], v[150:151] + ds_write_b128 v16, v[26:29] offset:5440 + v_mov_b64_e32 v[26:27], v[152:153] + v_mov_b64_e32 v[28:29], v[154:155] + ds_write_b128 v16, v[26:29] offset:6528 + v_mov_b64_e32 v[26:27], v[156:157] + v_mov_b64_e32 v[28:29], v[158:159] + ds_write_b128 v16, v[26:29] offset:7616 + s_waitcnt lgkmcnt(4) + ds_read_b128 v[96:99], v10 + ds_read_b128 v[100:103], v10 offset:64 + ds_read_b128 v[104:107], v10 offset:128 + ds_read_b128 v[108:111], v10 offset:192 + s_waitcnt lgkmcnt(4) + ds_read_b128 v[112:115], v10 offset:4352 + ds_read_b128 v[116:119], v10 offset:4416 + ds_read_b128 v[120:123], v10 offset:4480 + ds_read_b128 v[124:127], v10 offset:4544 + s_waitcnt lgkmcnt(0) + buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[112:115], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[116:119], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[120:123], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[124:127], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 + s_cmp_le_u32 s87, 32 + s_cbranch_scc1 label_86CC + s_sub_u32 s76, s96, s95 + s_sub_u32 s77, s80, 1 + s_and_b32 s75, s77, 32 + s_cmp_eq_u32 s75, 0 + s_cselect_b32 s75, s76, 0 + v_add_u32_e32 v2, s75, v2 + v_mov_b64_e32 v[26:27], v[160:161] + v_mov_b64_e32 v[28:29], v[162:163] + ds_write_b128 v16, v[26:29] + v_mov_b64_e32 v[26:27], v[164:165] + v_mov_b64_e32 v[28:29], v[166:167] + ds_write_b128 v16, v[26:29] offset:1088 + v_mov_b64_e32 v[26:27], v[168:169] + v_mov_b64_e32 v[28:29], v[170:171] + ds_write_b128 v16, v[26:29] offset:2176 + v_mov_b64_e32 v[26:27], v[172:173] + v_mov_b64_e32 v[28:29], v[174:175] + ds_write_b128 v16, v[26:29] offset:3264 + v_mov_b64_e32 v[26:27], v[176:177] + v_mov_b64_e32 v[28:29], v[178:179] + ds_write_b128 v16, v[26:29] offset:4352 + v_mov_b64_e32 v[26:27], v[180:181] + v_mov_b64_e32 v[28:29], v[182:183] + ds_write_b128 v16, v[26:29] offset:5440 + v_mov_b64_e32 v[26:27], v[184:185] + v_mov_b64_e32 v[28:29], v[186:187] + ds_write_b128 v16, v[26:29] offset:6528 + v_mov_b64_e32 v[26:27], v[188:189] + v_mov_b64_e32 v[28:29], v[190:191] + ds_write_b128 v16, v[26:29] offset:7616 + s_waitcnt lgkmcnt(4) + ds_read_b128 v[96:99], v10 + ds_read_b128 v[100:103], v10 offset:64 + ds_read_b128 v[104:107], v10 offset:128 + ds_read_b128 v[108:111], v10 offset:192 + s_waitcnt lgkmcnt(4) + ds_read_b128 v[112:115], v10 offset:4352 + ds_read_b128 v[116:119], v10 offset:4416 + ds_read_b128 v[120:123], v10 offset:4480 + ds_read_b128 v[124:127], v10 offset:4544 + s_waitcnt lgkmcnt(0) + buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[112:115], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[116:119], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[120:123], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[124:127], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 + s_sub_u32 s76, s96, s95 + s_sub_u32 s77, s80, 1 + s_and_b32 s75, s77, 48 + s_cmp_eq_u32 s75, 0 + s_cselect_b32 s75, s76, 0 + v_add_u32_e32 v2, s75, v2 + v_mov_b64_e32 v[26:27], v[192:193] + v_mov_b64_e32 v[28:29], v[194:195] + ds_write_b128 v16, v[26:29] + v_mov_b64_e32 v[26:27], v[196:197] + v_mov_b64_e32 v[28:29], v[198:199] + ds_write_b128 v16, v[26:29] offset:1088 + v_mov_b64_e32 v[26:27], v[200:201] + v_mov_b64_e32 v[28:29], v[202:203] + ds_write_b128 v16, v[26:29] offset:2176 + v_mov_b64_e32 v[26:27], v[204:205] + v_mov_b64_e32 v[28:29], v[206:207] + ds_write_b128 v16, v[26:29] offset:3264 + v_mov_b64_e32 v[26:27], v[208:209] + v_mov_b64_e32 v[28:29], v[210:211] + ds_write_b128 v16, v[26:29] offset:4352 + v_mov_b64_e32 v[26:27], v[212:213] + v_mov_b64_e32 v[28:29], v[214:215] + ds_write_b128 v16, v[26:29] offset:5440 + v_mov_b64_e32 v[26:27], v[216:217] + v_mov_b64_e32 v[28:29], v[218:219] + ds_write_b128 v16, v[26:29] offset:6528 + v_mov_b64_e32 v[26:27], v[220:221] + v_mov_b64_e32 v[28:29], v[222:223] + ds_write_b128 v16, v[26:29] offset:7616 + s_waitcnt lgkmcnt(4) + ds_read_b128 v[96:99], v10 + ds_read_b128 v[100:103], v10 offset:64 + ds_read_b128 v[104:107], v10 offset:128 + ds_read_b128 v[108:111], v10 offset:192 + s_waitcnt lgkmcnt(4) + ds_read_b128 v[112:115], v10 offset:4352 + ds_read_b128 v[116:119], v10 offset:4416 + ds_read_b128 v[120:123], v10 offset:4480 + ds_read_b128 v[124:127], v10 offset:4544 + s_waitcnt lgkmcnt(0) + buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[112:115], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[116:119], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[120:123], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[124:127], v2, s[8:11], 0 offen offset:256 + v_add_u32_e32 v2, 0x2000, v2 +label_86CC: + s_branch label_8EF4 +label_86D0: + s_mul_i32 s95, 0x400, s80 + s_mul_i32 s96, s82, s95 + s_mul_i32 s75, s89, s96 + s_add_u32 s8, s75, s8 + s_addc_u32 s9, 0, s9 + s_mov_b32 s75, 0x10000 + s_mul_i32 s75, s75, s2 + s_add_u32 s8, s75, s8 + s_addc_u32 s9, 0, s9 + s_mul_i32 s75, s96, s91 + s_mov_b32 s10, s75 + v_and_b32_e32 v26, 7, v0 + v_lshlrev_b32_e32 v2, 4, v26 + v_lshrrev_b32_e32 v26, 3, v0 + v_mul_i32_i24_e32 v26, 0x400, v26 + v_add_u32_e32 v2, v2, v26 + s_mul_i32 s75, s4, s95 + v_add_u32_e64 v2, v2, s75 + s_mul_i32 s75, s7, 0x100 + v_add_u32_e64 v2, v2, s75 + v_mov_b32_e32 v1, v2 + s_waitcnt vmcnt(0) lgkmcnt(0) + s_barrier + v_lshrrev_b32_e32 v26, 4, v0 + v_mul_i32_i24_e32 v16, 0x48, v26 + v_and_b32_e32 v26, 15, v0 + v_mul_i32_i24_e32 v26, 2, v26 + v_add_u32_e32 v16, v26, v16 + s_mul_i32 s75, s7, 0x480 + v_add_u32_e32 v16, s75, v16 + v_lshlrev_b32_e32 v16, 2, v16 + v_lshrrev_b32_e32 v26, 3, v0 + v_mul_i32_i24_e32 v10, 2, v26 + v_and_b32_e32 v26, 3, v0 + v_mul_i32_i24_e32 v26, 0x90, v26 + v_add_u32_e32 v10, v26, v10 + v_lshrrev_b32_e32 v26, 2, v0 + v_and_b32_e32 v26, 1, v26 + v_mul_i32_i24_e32 v26, 36, v26 + v_add_u32_e32 v10, v26, v10 + s_mul_i32 s75, s7, 0x480 + v_add_u32_e32 v10, s75, v10 + v_lshlrev_b32_e32 v10, 2, v10 + v_mov_b32_e32 v2, v1 + v_mov_b64_e32 v[30:31], v[96:97] + v_mov_b64_e32 v[32:33], v[98:99] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] + v_mov_b64_e32 v[30:31], v[100:101] + v_mov_b64_e32 v[32:33], v[102:103] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:1152 + v_mov_b64_e32 v[30:31], v[104:105] + v_mov_b64_e32 v[32:33], v[106:107] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:144 + v_mov_b64_e32 v[30:31], v[108:109] + v_mov_b64_e32 v[32:33], v[110:111] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:1296 + v_mov_b64_e32 v[30:31], v[112:113] + v_mov_b64_e32 v[32:33], v[114:115] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:2304 + v_mov_b64_e32 v[30:31], v[116:117] + v_mov_b64_e32 v[32:33], v[118:119] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:3456 + v_mov_b64_e32 v[30:31], v[120:121] + v_mov_b64_e32 v[32:33], v[122:123] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:2448 + v_mov_b64_e32 v[30:31], v[124:125] + v_mov_b64_e32 v[32:33], v[126:127] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:3600 + s_waitcnt lgkmcnt(4) + ds_read_b64 v[96:97], v10 + ds_read_b64 v[100:101], v10 offset:64 + ds_read_b64 v[98:99], v10 offset:288 + ds_read_b64 v[102:103], v10 offset:352 + s_waitcnt lgkmcnt(4) + ds_read_b64 v[104:105], v10 offset:2304 + ds_read_b64 v[108:109], v10 offset:2368 + ds_read_b64 v[106:107], v10 offset:2592 + ds_read_b64 v[110:111], v10 offset:2656 + s_waitcnt lgkmcnt(0) + buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen offset:128 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen offset:128 + v_add_u32_e32 v2, 0x2000, v2 + s_cmp_le_u32 s87, 16 + s_cbranch_scc1 label_8DA8 + v_mov_b64_e32 v[30:31], v[128:129] + v_mov_b64_e32 v[32:33], v[130:131] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] + v_mov_b64_e32 v[30:31], v[132:133] + v_mov_b64_e32 v[32:33], v[134:135] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:1152 + v_mov_b64_e32 v[30:31], v[136:137] + v_mov_b64_e32 v[32:33], v[138:139] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:144 + v_mov_b64_e32 v[30:31], v[140:141] + v_mov_b64_e32 v[32:33], v[142:143] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:1296 + v_mov_b64_e32 v[30:31], v[144:145] + v_mov_b64_e32 v[32:33], v[146:147] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:2304 + v_mov_b64_e32 v[30:31], v[148:149] + v_mov_b64_e32 v[32:33], v[150:151] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:3456 + v_mov_b64_e32 v[30:31], v[152:153] + v_mov_b64_e32 v[32:33], v[154:155] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:2448 + v_mov_b64_e32 v[30:31], v[156:157] + v_mov_b64_e32 v[32:33], v[158:159] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:3600 + s_waitcnt lgkmcnt(4) + ds_read_b64 v[96:97], v10 + ds_read_b64 v[100:101], v10 offset:64 + ds_read_b64 v[98:99], v10 offset:288 + ds_read_b64 v[102:103], v10 offset:352 + s_waitcnt lgkmcnt(4) + ds_read_b64 v[104:105], v10 offset:2304 + ds_read_b64 v[108:109], v10 offset:2368 + ds_read_b64 v[106:107], v10 offset:2592 + ds_read_b64 v[110:111], v10 offset:2656 + s_waitcnt lgkmcnt(0) + buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen offset:128 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen offset:128 + v_add_u32_e32 v2, 0x2000, v2 + s_cmp_le_u32 s87, 32 + s_cbranch_scc1 label_8DA8 + v_mov_b64_e32 v[30:31], v[160:161] + v_mov_b64_e32 v[32:33], v[162:163] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] + v_mov_b64_e32 v[30:31], v[164:165] + v_mov_b64_e32 v[32:33], v[166:167] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:1152 + v_mov_b64_e32 v[30:31], v[168:169] + v_mov_b64_e32 v[32:33], v[170:171] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:144 + v_mov_b64_e32 v[30:31], v[172:173] + v_mov_b64_e32 v[32:33], v[174:175] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:1296 + v_mov_b64_e32 v[30:31], v[176:177] + v_mov_b64_e32 v[32:33], v[178:179] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:2304 + v_mov_b64_e32 v[30:31], v[180:181] + v_mov_b64_e32 v[32:33], v[182:183] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:3456 + v_mov_b64_e32 v[30:31], v[184:185] + v_mov_b64_e32 v[32:33], v[186:187] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:2448 + v_mov_b64_e32 v[30:31], v[188:189] + v_mov_b64_e32 v[32:33], v[190:191] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:3600 + s_waitcnt lgkmcnt(4) + ds_read_b64 v[96:97], v10 + ds_read_b64 v[100:101], v10 offset:64 + ds_read_b64 v[98:99], v10 offset:288 + ds_read_b64 v[102:103], v10 offset:352 + s_waitcnt lgkmcnt(4) + ds_read_b64 v[104:105], v10 offset:2304 + ds_read_b64 v[108:109], v10 offset:2368 + ds_read_b64 v[106:107], v10 offset:2592 + ds_read_b64 v[110:111], v10 offset:2656 + s_waitcnt lgkmcnt(0) + buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen offset:128 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen offset:128 + v_add_u32_e32 v2, 0x2000, v2 + v_mov_b64_e32 v[30:31], v[192:193] + v_mov_b64_e32 v[32:33], v[194:195] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] + v_mov_b64_e32 v[30:31], v[196:197] + v_mov_b64_e32 v[32:33], v[198:199] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:1152 + v_mov_b64_e32 v[30:31], v[200:201] + v_mov_b64_e32 v[32:33], v[202:203] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:144 + v_mov_b64_e32 v[30:31], v[204:205] + v_mov_b64_e32 v[32:33], v[206:207] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:1296 + v_mov_b64_e32 v[30:31], v[208:209] + v_mov_b64_e32 v[32:33], v[210:211] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:2304 + v_mov_b64_e32 v[30:31], v[212:213] + v_mov_b64_e32 v[32:33], v[214:215] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:3456 + v_mov_b64_e32 v[30:31], v[216:217] + v_mov_b64_e32 v[32:33], v[218:219] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:2448 + v_mov_b64_e32 v[30:31], v[220:221] + v_mov_b64_e32 v[32:33], v[222:223] + v_cvt_pk_bf16_f32 v30, v30, v31 + v_cvt_pk_bf16_f32 v31, v32, v33 + ds_write_b64 v16, v[30:31] offset:3600 + s_waitcnt lgkmcnt(4) + ds_read_b64 v[96:97], v10 + ds_read_b64 v[100:101], v10 offset:64 + ds_read_b64 v[98:99], v10 offset:288 + ds_read_b64 v[102:103], v10 offset:352 + s_waitcnt lgkmcnt(4) + ds_read_b64 v[104:105], v10 offset:2304 + ds_read_b64 v[108:109], v10 offset:2368 + ds_read_b64 v[106:107], v10 offset:2592 + ds_read_b64 v[110:111], v10 offset:2656 + s_waitcnt lgkmcnt(0) + buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen offset:128 + v_add_u32_e32 v2, 0x2000, v2 + buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen + buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen offset:128 + v_add_u32_e32 v2, 0x2000, v2 +label_8DA8: + s_branch label_8EF4 +label_8DAC: + s_cmp_eq_u32 s99, 1 + s_cbranch_scc0 label_8DC4 + s_waitcnt vmcnt(0) + s_waitcnt lgkmcnt(0) + s_barrier + s_branch label_8EF4 +label_8DC4: + s_cmp_eq_u32 s80, 16 + s_cselect_b32 s69, 16, 64 + s_cmp_eq_u32 s80, 32 + s_cselect_b32 s69, 32, s69 + s_cmp_eq_u32 s91, 1 + s_cselect_b32 s69, s69, 64 + s_mov_b32 s72, s69 + v_mov_b32_e32 v33, 0xe0ad78ec + v_mov_b32_e32 v46, 0 + v_mov_b32_e32 v43, 0 + v_mov_b64_e32 v[96:97], 0 + v_mov_b64_e32 v[98:99], 0 + v_mov_b64_e32 v[100:101], 0 + v_mov_b64_e32 v[102:103], 0 + v_mov_b64_e32 v[104:105], 0 + v_mov_b64_e32 v[106:107], 0 + v_mov_b64_e32 v[108:109], 0 + v_mov_b64_e32 v[110:111], 0 + v_mov_b64_e32 v[112:113], 0 + v_mov_b64_e32 v[114:115], 0 + v_mov_b64_e32 v[116:117], 0 + v_mov_b64_e32 v[118:119], 0 + v_mov_b64_e32 v[120:121], 0 + v_mov_b64_e32 v[122:123], 0 + v_mov_b64_e32 v[124:125], 0 + v_mov_b64_e32 v[126:127], 0 + v_mov_b64_e32 v[128:129], 0 + v_mov_b64_e32 v[130:131], 0 + v_mov_b64_e32 v[132:133], 0 + v_mov_b64_e32 v[134:135], 0 + v_mov_b64_e32 v[136:137], 0 + v_mov_b64_e32 v[138:139], 0 + v_mov_b64_e32 v[140:141], 0 + v_mov_b64_e32 v[142:143], 0 + v_mov_b64_e32 v[144:145], 0 + v_mov_b64_e32 v[146:147], 0 + v_mov_b64_e32 v[148:149], 0 + v_mov_b64_e32 v[150:151], 0 + v_mov_b64_e32 v[152:153], 0 + v_mov_b64_e32 v[154:155], 0 + v_mov_b64_e32 v[156:157], 0 + v_mov_b64_e32 v[158:159], 0 + v_mov_b64_e32 v[160:161], 0 + v_mov_b64_e32 v[162:163], 0 + v_mov_b64_e32 v[164:165], 0 + v_mov_b64_e32 v[166:167], 0 + v_mov_b64_e32 v[168:169], 0 + v_mov_b64_e32 v[170:171], 0 + v_mov_b64_e32 v[172:173], 0 + v_mov_b64_e32 v[174:175], 0 + v_mov_b64_e32 v[176:177], 0 + v_mov_b64_e32 v[178:179], 0 + v_mov_b64_e32 v[180:181], 0 + v_mov_b64_e32 v[182:183], 0 + v_mov_b64_e32 v[184:185], 0 + v_mov_b64_e32 v[186:187], 0 + v_mov_b64_e32 v[188:189], 0 + v_mov_b64_e32 v[190:191], 0 + v_mov_b64_e32 v[192:193], 0 + v_mov_b64_e32 v[194:195], 0 + v_mov_b64_e32 v[196:197], 0 + v_mov_b64_e32 v[198:199], 0 + v_mov_b64_e32 v[200:201], 0 + v_mov_b64_e32 v[202:203], 0 + v_mov_b64_e32 v[204:205], 0 + v_mov_b64_e32 v[206:207], 0 + v_mov_b64_e32 v[208:209], 0 + v_mov_b64_e32 v[210:211], 0 + v_mov_b64_e32 v[212:213], 0 + v_mov_b64_e32 v[214:215], 0 + v_mov_b64_e32 v[216:217], 0 + v_mov_b64_e32 v[218:219], 0 + v_mov_b64_e32 v[220:221], 0 + v_mov_b64_e32 v[222:223], 0 + s_branch label_7DF0 +label_8EF4: + s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) + s_endpgm +label_8EFC: + s_waitcnt vmcnt(0) + s_waitcnt lgkmcnt(0) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_bfe_u32 v230, v230, v25, 8 + v_and_b32_e32 v231, v47, v231 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_bfe_u32 v234, v234, v25, 8 + v_and_b32_e32 v235, v47, v235 + s_and_b32 s77, s88, 3 + s_mul_i32 s75, s77, 0x4100 + v_add_u32_e32 v30, s75, v13 + ds_read_b128 a[56:59], v30 offset:8320 + ds_read_b128 a[60:63], v30 offset:8336 + ds_read_b128 a[88:91], v30 offset:10400 + ds_read_b128 a[92:95], v30 offset:10416 + v_add_u32_e32 v30, s75, v241 + ds_read_b128 a[64:67], v30 offset:8384 + ds_read_b128 a[68:71], v30 offset:8400 + ds_read_b128 a[96:99], v30 offset:10464 + ds_read_b128 a[100:103], v30 offset:10480 + v_add_u32_e32 v31, s75, v239 + ds_read_b128 a[40:43], v31 offset:8192 + ds_read_b128 a[44:47], v31 offset:8208 + ds_read_b128 a[72:75], v31 offset:10272 + ds_read_b128 a[76:79], v31 offset:10288 + v_add_u32_e32 v31, s75, v238 + v_add_u32_e32 v32, s75, v1 + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + ds_write_b128 v11, v[26:29] offset:14640 + s_mul_i32 s76, s77, 0x1040 + v_add_u32_e32 v31, s76, v16 + v_mov_b64_e32 v[26:27], 0 + v_mov_b64_e32 v[28:29], 0 + v_and_b32_e32 v30, 15, v0 + s_and_b32 s75, s70, 0xff + s_sub_i32 s76, s75, 0 + v_cmp_le_i32_e32 vcc, s76, v30 + s_nop 4 + s_mov_b64 exec, vcc + ds_write_b128 v11, v[26:29] + ds_write_b128 v11, v[26:29] offset:16 + ds_write_b128 v11, v[26:29] offset:32 + ds_write_b128 v11, v[26:29] offset:48 + ds_write_b128 v31, v[26:29] + ds_write_b128 v31, v[26:29] offset:16 + s_mov_b64 exec, -1 + s_sub_i32 s76, s75, 16 + v_cmp_le_i32_e32 vcc, s76, v30 + s_nop 4 + s_mov_b64 exec, vcc + ds_write_b128 v11, v[26:29] offset:14592 + ds_write_b128 v11, v[26:29] offset:14608 + ds_write_b128 v11, v[26:29] offset:14624 + ds_write_b128 v11, v[26:29] offset:14640 + ds_write_b128 v31, v[26:29] offset:2080 + ds_write_b128 v31, v[26:29] offset:2096 + s_mov_b64 exec, -1 + s_cmp_eq_i32 s7, 3 + s_cbranch_scc1 label_97C0 + s_waitcnt lgkmcnt(0) + s_waitcnt vmcnt(0) + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + s_waitcnt lgkmcnt(0) + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + v_mov_b32_e32 v31, 0xff800000 + s_and_b32 s75, s70, 0xff + v_mov_b32_e32 v30, s75 + v_lshrrev_b32_e32 v26, 4, v0 + v_mul_i32_i24_e32 v26, 4, v26 + v_add_u32_e32 v27, 1, v26 + v_add_u32_e32 v28, 2, v26 + v_add_u32_e32 v29, 3, v26 + v_cmp_lt_u32_e64 s[64:65], v26, v30 + v_add_u32_e32 v26, 16, v26 + s_nop 0 + v_cndmask_b32_e64 v48, v31, v48, s[64:65] + v_cmp_lt_u32_e64 s[64:65], v27, v30 + v_add_u32_e32 v27, 16, v27 + s_nop 0 + v_cndmask_b32_e64 v49, v31, v49, s[64:65] + v_cmp_lt_u32_e64 s[64:65], v28, v30 + v_add_u32_e32 v28, 16, v28 + s_nop 0 + v_cndmask_b32_e64 v50, v31, v50, s[64:65] + v_cmp_lt_u32_e64 s[64:65], v29, v30 + v_add_u32_e32 v29, 16, v29 + s_nop 0 + v_cndmask_b32_e64 v51, v31, v51, s[64:65] + v_cmp_lt_u32_e64 s[64:65], v26, v30 + v_add_u32_e32 v26, 16, v26 + s_nop 0 + v_cndmask_b32_e64 v52, v31, v52, s[64:65] + v_cmp_lt_u32_e64 s[64:65], v27, v30 + v_add_u32_e32 v27, 16, v27 + s_nop 0 + v_cndmask_b32_e64 v53, v31, v53, s[64:65] + v_cmp_lt_u32_e64 s[64:65], v28, v30 + v_add_u32_e32 v28, 16, v28 + s_nop 0 + v_cndmask_b32_e64 v54, v31, v54, s[64:65] + v_cmp_lt_u32_e64 s[64:65], v29, v30 + v_add_u32_e32 v29, 16, v29 + s_nop 0 + v_cndmask_b32_e64 v55, v31, v55, s[64:65] + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + v_exp_f32_e32 v50, v50 + v_exp_f32_e32 v51, v51 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_add_f32_e32 v26, v55, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + s_cmp_eq_u32 s72, 64 + s_cbranch_scc1 label_9604 + s_cmp_eq_u32 s72, 16 + s_cbranch_scc0 label_9590 + ds_write_b32 v17, v33 offset:4096 + ds_write_b64 v20, v[48:49] + ds_write_b64 v20, v[50:51] offset:2048 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b64 v[48:49], v18 + ds_read_b64 v[50:51], v19 + ds_read_b32 v34, v43 offset:4096 + s_waitcnt lgkmcnt(0) + s_barrier + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_cbranch_vccz label_958C + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_958C: + s_branch label_96AC +label_9590: + ds_write_b32 v17, v33 offset:4096 + ds_write_b64 v20, v[48:49] + ds_write_b64 v20, v[50:51] offset:2048 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b64 v[48:49], v18 + ds_read_b64 v[50:51], v19 + ds_read_b64 v[52:53], v18 offset:512 + ds_read_b64 v[54:55], v19 offset:512 + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + s_waitcnt lgkmcnt(0) + s_barrier + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_cbranch_vccz label_9600 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_9600: + s_branch label_96AC +label_9604: + ds_write_b32 v17, v33 offset:4096 + ds_write_b64 v20, v[48:49] + ds_write_b64 v20, v[50:51] offset:2048 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b64 v[48:49], v18 + ds_read_b64 v[50:51], v19 + ds_read_b64 v[52:53], v18 offset:512 + ds_read_b64 v[54:55], v19 offset:512 + ds_read_b64 v[56:57], v18 offset:1024 + ds_read_b64 v[58:59], v19 offset:1024 + ds_read_b64 v[60:61], v18 offset:1536 + ds_read_b64 v[62:63], v19 offset:1536 + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b32 v36, v43 offset:4608 + ds_read_b32 v37, v43 offset:4864 + s_waitcnt lgkmcnt(0) + s_barrier + v_min_f32_e32 v26, v34, v35 + v_min_f32_e32 v26, v26, v36 + v_min_f32_e32 v26, v26, v37 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_cbranch_vccz label_96AC + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9DB4 +label_96AC: + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + s_cmp_le_u32 s72, 16 + s_cbranch_scc1 label_97BC + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + s_cmp_le_u32 s72, 32 + s_cbranch_scc1 label_97BC + v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] + v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] + v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] + v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] + v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] + v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] + v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] + v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] + v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] + v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] + v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] + v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] + v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] + v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] + v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] + v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] +label_97BC: + s_branch label_7CF4 +label_97C0: + s_and_b32 s75, s88, 3 + s_mul_i32 s75, s75, 0x1040 + v_add_u32_e32 v23, s75, v23 + v_add_u32_e32 v24, s75, v24 + s_waitcnt lgkmcnt(0) + s_waitcnt vmcnt(0) + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 + ds_read_b64_tr_b16 a[138:139], v23 offset:128 + ds_read_b64_tr_b16 a[140:141], v24 + ds_read_b64_tr_b16 a[142:143], v24 offset:128 + ds_read_b64_tr_b16 a[144:145], v23 offset:512 + ds_read_b64_tr_b16 a[146:147], v23 offset:640 + ds_read_b64_tr_b16 a[148:149], v24 offset:512 + ds_read_b64_tr_b16 a[150:151], v24 offset:640 + s_waitcnt lgkmcnt(0) + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + v_mov_b32_e32 v31, 0xff800000 + s_and_b32 s75, s70, 0xff + v_mov_b32_e32 v30, s75 + v_lshrrev_b32_e32 v26, 4, v0 + v_mul_i32_i24_e32 v26, 4, v26 + v_add_u32_e32 v27, 1, v26 + v_add_u32_e32 v28, 2, v26 + v_add_u32_e32 v29, 3, v26 + v_cmp_lt_u32_e64 s[64:65], v26, v30 + v_add_u32_e32 v26, 16, v26 + s_nop 0 + v_cndmask_b32_e64 v48, v31, v48, s[64:65] + v_cmp_lt_u32_e64 s[64:65], v27, v30 + v_add_u32_e32 v27, 16, v27 + s_nop 0 + v_cndmask_b32_e64 v49, v31, v49, s[64:65] + v_cmp_lt_u32_e64 s[64:65], v28, v30 + v_add_u32_e32 v28, 16, v28 + s_nop 0 + v_cndmask_b32_e64 v50, v31, v50, s[64:65] + v_cmp_lt_u32_e64 s[64:65], v29, v30 + v_add_u32_e32 v29, 16, v29 + s_nop 0 + v_cndmask_b32_e64 v51, v31, v51, s[64:65] + v_cmp_lt_u32_e64 s[64:65], v26, v30 + v_add_u32_e32 v26, 16, v26 + s_nop 0 + v_cndmask_b32_e64 v52, v31, v52, s[64:65] + v_cmp_lt_u32_e64 s[64:65], v27, v30 + v_add_u32_e32 v27, 16, v27 + s_nop 0 + v_cndmask_b32_e64 v53, v31, v53, s[64:65] + v_cmp_lt_u32_e64 s[64:65], v28, v30 + v_add_u32_e32 v28, 16, v28 + s_nop 0 + v_cndmask_b32_e64 v54, v31, v54, s[64:65] + v_cmp_lt_u32_e64 s[64:65], v29, v30 + v_add_u32_e32 v29, 16, v29 + s_nop 0 + v_cndmask_b32_e64 v55, v31, v55, s[64:65] + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + v_exp_f32_e32 v50, v50 + v_exp_f32_e32 v51, v51 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_add_f32_e32 v26, v55, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + s_cmp_eq_u32 s72, 64 + s_cbranch_scc1 label_9BF8 + s_cmp_eq_u32 s72, 16 + s_cbranch_scc0 label_9B84 + ds_write_b32 v17, v33 offset:4096 + ds_write_b64 v20, v[48:49] + ds_write_b64 v20, v[50:51] offset:2048 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b64 v[48:49], v18 + ds_read_b64 v[50:51], v19 + ds_read_b32 v34, v43 offset:4096 + s_waitcnt lgkmcnt(0) + s_barrier + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_cbranch_vccz label_9B80 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_9B80: + s_branch label_9CA0 +label_9B84: + ds_write_b32 v17, v33 offset:4096 + ds_write_b64 v20, v[48:49] + ds_write_b64 v20, v[50:51] offset:2048 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b64 v[48:49], v18 + ds_read_b64 v[50:51], v19 + ds_read_b64 v[52:53], v18 offset:512 + ds_read_b64 v[54:55], v19 offset:512 + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + s_waitcnt lgkmcnt(0) + s_barrier + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_cbranch_vccz label_9BF4 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_9BF4: + s_branch label_9CA0 +label_9BF8: + ds_write_b32 v17, v33 offset:4096 + ds_write_b64 v20, v[48:49] + ds_write_b64 v20, v[50:51] offset:2048 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b64 v[48:49], v18 + ds_read_b64 v[50:51], v19 + ds_read_b64 v[52:53], v18 offset:512 + ds_read_b64 v[54:55], v19 offset:512 + ds_read_b64 v[56:57], v18 offset:1024 + ds_read_b64 v[58:59], v19 offset:1024 + ds_read_b64 v[60:61], v18 offset:1536 + ds_read_b64 v[62:63], v19 offset:1536 + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b32 v36, v43 offset:4608 + ds_read_b32 v37, v43 offset:4864 + s_waitcnt lgkmcnt(0) + s_barrier + v_min_f32_e32 v26, v34, v35 + v_min_f32_e32 v26, v26, v36 + v_min_f32_e32 v26, v26, v37 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_cbranch_vccz label_9CA0 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9DB4 +label_9CA0: + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + s_cmp_le_u32 s72, 16 + s_cbranch_scc1 label_9DB0 + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + s_cmp_le_u32 s72, 32 + s_cbranch_scc1 label_9DB0 + v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] + v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] + v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] + v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] + v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] + v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] + v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] + v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] + v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] + v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] + v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] + v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] + v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] + v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] + v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] + v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] +label_9DB0: + s_branch label_7CF4 +label_9DB4: + v_pk_mul_f32 v[192:193], v[36:37], v[192:193] op_sel:[1,0] + v_pk_mul_f32 v[194:195], v[36:37], v[194:195] op_sel:[1,0] + v_pk_mul_f32 v[196:197], v[36:37], v[196:197] op_sel:[1,0] + v_pk_mul_f32 v[198:199], v[36:37], v[198:199] op_sel:[1,0] + v_pk_mul_f32 v[200:201], v[36:37], v[200:201] op_sel:[1,0] + v_pk_mul_f32 v[202:203], v[36:37], v[202:203] op_sel:[1,0] + v_pk_mul_f32 v[204:205], v[36:37], v[204:205] op_sel:[1,0] + v_pk_mul_f32 v[206:207], v[36:37], v[206:207] op_sel:[1,0] + v_pk_mul_f32 v[208:209], v[36:37], v[208:209] op_sel:[1,0] + v_pk_mul_f32 v[210:211], v[36:37], v[210:211] op_sel:[1,0] + v_pk_mul_f32 v[212:213], v[36:37], v[212:213] op_sel:[1,0] + v_pk_mul_f32 v[214:215], v[36:37], v[214:215] op_sel:[1,0] + v_pk_mul_f32 v[216:217], v[36:37], v[216:217] op_sel:[1,0] + v_pk_mul_f32 v[218:219], v[36:37], v[218:219] op_sel:[1,0] + v_pk_mul_f32 v[220:221], v[36:37], v[220:221] op_sel:[1,0] + v_pk_mul_f32 v[222:223], v[36:37], v[222:223] op_sel:[1,0] + v_pk_mul_f32 v[160:161], v[36:37], v[160:161] op_sel_hi:[0,1] + v_pk_mul_f32 v[162:163], v[36:37], v[162:163] op_sel_hi:[0,1] + v_pk_mul_f32 v[164:165], v[36:37], v[164:165] op_sel_hi:[0,1] + v_pk_mul_f32 v[166:167], v[36:37], v[166:167] op_sel_hi:[0,1] + v_pk_mul_f32 v[168:169], v[36:37], v[168:169] op_sel_hi:[0,1] + v_pk_mul_f32 v[170:171], v[36:37], v[170:171] op_sel_hi:[0,1] + v_pk_mul_f32 v[172:173], v[36:37], v[172:173] op_sel_hi:[0,1] + v_pk_mul_f32 v[174:175], v[36:37], v[174:175] op_sel_hi:[0,1] + v_pk_mul_f32 v[176:177], v[36:37], v[176:177] op_sel_hi:[0,1] + v_pk_mul_f32 v[178:179], v[36:37], v[178:179] op_sel_hi:[0,1] + v_pk_mul_f32 v[180:181], v[36:37], v[180:181] op_sel_hi:[0,1] + v_pk_mul_f32 v[182:183], v[36:37], v[182:183] op_sel_hi:[0,1] + v_pk_mul_f32 v[184:185], v[36:37], v[184:185] op_sel_hi:[0,1] + v_pk_mul_f32 v[186:187], v[36:37], v[186:187] op_sel_hi:[0,1] + v_pk_mul_f32 v[188:189], v[36:37], v[188:189] op_sel_hi:[0,1] + v_pk_mul_f32 v[190:191], v[36:37], v[190:191] op_sel_hi:[0,1] +label_9EB4: + v_pk_mul_f32 v[128:129], v[34:35], v[128:129] op_sel:[1,0] + v_pk_mul_f32 v[130:131], v[34:35], v[130:131] op_sel:[1,0] + v_pk_mul_f32 v[132:133], v[34:35], v[132:133] op_sel:[1,0] + v_pk_mul_f32 v[134:135], v[34:35], v[134:135] op_sel:[1,0] + v_pk_mul_f32 v[136:137], v[34:35], v[136:137] op_sel:[1,0] + v_pk_mul_f32 v[138:139], v[34:35], v[138:139] op_sel:[1,0] + v_pk_mul_f32 v[140:141], v[34:35], v[140:141] op_sel:[1,0] + v_pk_mul_f32 v[142:143], v[34:35], v[142:143] op_sel:[1,0] + v_pk_mul_f32 v[144:145], v[34:35], v[144:145] op_sel:[1,0] + v_pk_mul_f32 v[146:147], v[34:35], v[146:147] op_sel:[1,0] + v_pk_mul_f32 v[148:149], v[34:35], v[148:149] op_sel:[1,0] + v_pk_mul_f32 v[150:151], v[34:35], v[150:151] op_sel:[1,0] + v_pk_mul_f32 v[152:153], v[34:35], v[152:153] op_sel:[1,0] + v_pk_mul_f32 v[154:155], v[34:35], v[154:155] op_sel:[1,0] + v_pk_mul_f32 v[156:157], v[34:35], v[156:157] op_sel:[1,0] + v_pk_mul_f32 v[158:159], v[34:35], v[158:159] op_sel:[1,0] +label_9F34: + v_pk_mul_f32 v[96:97], v[34:35], v[96:97] op_sel_hi:[0,1] + v_pk_mul_f32 v[98:99], v[34:35], v[98:99] op_sel_hi:[0,1] + v_pk_mul_f32 v[100:101], v[34:35], v[100:101] op_sel_hi:[0,1] + v_pk_mul_f32 v[102:103], v[34:35], v[102:103] op_sel_hi:[0,1] + v_pk_mul_f32 v[104:105], v[34:35], v[104:105] op_sel_hi:[0,1] + v_pk_mul_f32 v[106:107], v[34:35], v[106:107] op_sel_hi:[0,1] + v_pk_mul_f32 v[108:109], v[34:35], v[108:109] op_sel_hi:[0,1] + v_pk_mul_f32 v[110:111], v[34:35], v[110:111] op_sel_hi:[0,1] + v_pk_mul_f32 v[112:113], v[34:35], v[112:113] op_sel_hi:[0,1] + v_pk_mul_f32 v[114:115], v[34:35], v[114:115] op_sel_hi:[0,1] + v_pk_mul_f32 v[116:117], v[34:35], v[116:117] op_sel_hi:[0,1] + v_pk_mul_f32 v[118:119], v[34:35], v[118:119] op_sel_hi:[0,1] + v_pk_mul_f32 v[120:121], v[34:35], v[120:121] op_sel_hi:[0,1] + v_pk_mul_f32 v[122:123], v[34:35], v[122:123] op_sel_hi:[0,1] + v_pk_mul_f32 v[124:125], v[34:35], v[124:125] op_sel_hi:[0,1] + v_pk_mul_f32 v[126:127], v[34:35], v[126:127] op_sel_hi:[0,1] + s_setpc_b64 vcc +label_9FB8: + s_cmp_eq_u32 s88, 8 + s_cselect_b32 s69, 1, 0 + s_cmp_eq_u32 s70, 0 + s_cselect_b32 s69, s69, 0 + s_cmp_lt_u32 s7, 1 + s_cbranch_scc1 label_A02C + s_cmp_eq_u32 s70, 0 + s_cbranch_scc1 label_A018 + v_mov_b64_e32 v[48:49], 0 + v_mov_b64_e32 v[50:51], 0 + v_mov_b64_e32 v[52:53], 0 + v_mov_b64_e32 v[54:55], 0 + v_mov_b64_e32 v[56:57], 0 + v_mov_b64_e32 v[58:59], 0 + v_mov_b64_e32 v[60:61], 0 + v_mov_b64_e32 v[62:63], 0 + v_mov_b64_e32 v[64:65], 0 + v_mov_b64_e32 v[66:67], 0 + v_mov_b64_e32 v[68:69], 0 + v_mov_b64_e32 v[70:71], 0 + v_mov_b64_e32 v[72:73], 0 + v_mov_b64_e32 v[74:75], 0 + v_mov_b64_e32 v[76:77], 0 + v_mov_b64_e32 v[78:79], 0 +label_A018: + s_cmp_eq_u32 s7, 1 + s_cbranch_scc1 label_BBC0 + s_cmp_eq_u32 s7, 2 + s_cbranch_scc1 label_CC74 + s_branch label_DD28 +label_A02C: + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v39, 0, v39 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s56 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[50:51], v19 + v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] + buffer_load_dword v41, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] + buffer_load_dword v244, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] + s_cbranch_vccz label_A2A0 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_A2A0: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v64, v65, v64 + v_max3_f32 v30, v66, v67, v30 + v_max3_f32 v30, v68, v69, v30 + v_max3_f32 v30, v70, v71, v30 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v242, 0, v242 + v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s60 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_waitcnt vmcnt(12) + s_barrier + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + s_nop 1 + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:41472 + s_nop 1 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + s_nop 1 + ds_read_b128 a[48:51], v240 offset:41472 + ds_read_b128 a[52:55], v240 offset:41488 + ds_read_u8 v236, v238 offset:41472 + s_nop 1 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + s_nop 1 + ds_read_b128 a[80:83], v240 offset:43552 + ds_read_b128 a[84:87], v240 offset:43568 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:43552 + s_nop 1 + v_fma_f32 v64, v64, s5, -v32 + v_fma_f32 v65, v65, s5, -v32 + v_fma_f32 v66, v66, s5, -v32 + v_fma_f32 v67, v67, s5, -v32 + v_fma_f32 v68, v68, s5, -v32 + v_fma_f32 v69, v69, s5, -v32 + v_fma_f32 v70, v70, s5, -v32 + v_fma_f32 v71, v71, s5, -v32 + s_nop 1 + ds_read_b128 a[56:59], v13 offset:41600 + ds_read_b128 a[60:63], v13 offset:41616 + v_exp_f32_e32 v64, v64 + v_exp_f32_e32 v65, v65 + v_exp_f32_e32 v66, v66 + ds_read_u8 v237, v238 offset:43552 + s_nop 1 + v_exp_f32_e32 v67, v67 + v_exp_f32_e32 v68, v68 + v_exp_f32_e32 v69, v69 + v_exp_f32_e32 v70, v70 + v_exp_f32_e32 v71, v71 + v_mul_f32_e32 v46, v33, v46 + s_nop 1 + ds_read_b128 a[88:91], v13 offset:43680 + ds_read_b128 a[92:95], v13 offset:43696 + v_add_f32_e32 v26, v65, v64 + v_add_f32_e32 v26, v66, v26 + v_add_f32_e32 v26, v67, v26 + v_add_f32_e32 v26, v68, v26 + s_nop 1 + v_add_f32_e32 v26, v69, v26 + v_add_f32_e32 v26, v70, v26 + v_add_f32_e32 v26, v71, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v64, v64, v65 + v_cvt_pk_bf16_f32 v65, v66, v67 + v_cvt_pk_bf16_f32 v66, v68, v69 + v_cvt_pk_bf16_f32 v67, v70, v71 + s_nop 1 + ds_read_b128 a[64:67], v241 offset:41664 + ds_read_b128 a[68:71], v241 offset:41680 + ds_write_b32 v17, v33 offset:4096 + s_nop 1 + s_nop 1 + ds_read_b128 a[96:99], v241 offset:43744 + ds_read_b128 a[100:103], v241 offset:43760 + ds_write_b64 v20, v[64:65] + s_nop 1 + s_nop 1 + ds_read_b128 a[104:107], v16 offset:8320 + ds_read_b128 a[108:111], v16 offset:8336 + ds_write_b64 v20, v[66:67] offset:2048 + s_nop 1 + s_nop 1 + ds_read_b128 a[112:115], v16 offset:10400 + ds_read_b128 a[116:119], v16 offset:10416 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v40, 0, v40 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s57 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[66:67], v19 + v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + buffer_load_dword v42, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + buffer_load_dword v245, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + s_cbranch_vccz label_A970 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_A970: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v243, 0, v243 + v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s61 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_waitcnt vmcnt(11) + s_barrier + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + s_nop 1 + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:58112 + s_nop 1 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + s_nop 1 + ds_read_b128 a[48:51], v240 offset:58112 + ds_read_b128 a[52:55], v240 offset:58128 + ds_read_u8 v236, v238 offset:58112 + s_nop 1 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + s_nop 1 + ds_read_b128 a[80:83], v240 offset:60192 + ds_read_b128 a[84:87], v240 offset:60208 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:60192 + s_nop 1 + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + s_nop 1 + ds_read_b128 a[56:59], v13 offset:58240 + ds_read_b128 a[60:63], v13 offset:58256 + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + v_exp_f32_e32 v50, v50 + ds_read_u8 v237, v238 offset:60192 + s_nop 1 + v_exp_f32_e32 v51, v51 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + s_nop 1 + ds_read_b128 a[88:91], v13 offset:60320 + ds_read_b128 a[92:95], v13 offset:60336 + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + s_nop 1 + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_add_f32_e32 v26, v55, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + s_nop 1 + ds_read_b128 a[64:67], v241 offset:58304 + ds_read_b128 a[68:71], v241 offset:58320 + ds_write_b32 v17, v33 offset:4096 + s_nop 1 + s_nop 1 + ds_read_b128 a[96:99], v241 offset:60384 + ds_read_b128 a[100:103], v241 offset:60400 + ds_write_b64 v20, v[48:49] + s_nop 1 + s_nop 1 + ds_read_b128 a[104:107], v16 offset:12480 + ds_read_b128 a[108:111], v16 offset:12496 + ds_write_b64 v20, v[50:51] offset:2048 + s_nop 1 + s_nop 1 + ds_read_b128 a[112:115], v16 offset:14560 + ds_read_b128 a[116:119], v16 offset:14576 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v41, 0, v41 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s58 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[50:51], v19 + v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_AEF4 + buffer_load_dword v39, v38, s[24:27], 0 offen +label_AEF4: + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_AF1C + buffer_load_dword v242, v44, s[24:27], 0 offen +label_AF1C: + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_B040 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 +label_B040: + v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] + s_cbranch_vccz label_B058 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_B058: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v64, v65, v64 + v_max3_f32 v30, v66, v67, v30 + v_max3_f32 v30, v68, v69, v30 + v_max3_f32 v30, v70, v71, v30 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v244, 0, v244 + v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s62 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_cmp_eq_u32 s69, 0 + s_cbranch_scc1 label_B26C + s_waitcnt vmcnt(9) +label_B26C: + s_waitcnt vmcnt(12) + s_barrier + ds_read_b128 v[80:83], v10 offset:8192 + ds_read_b128 v[84:87], v10 offset:8208 + s_nop 1 + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:10272 + ds_read_b128 v[92:95], v10 offset:10288 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:8192 + s_nop 1 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + s_nop 1 + ds_read_b128 a[48:51], v240 offset:8192 + ds_read_b128 a[52:55], v240 offset:8208 + ds_read_u8 v236, v238 offset:8192 + s_nop 1 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + s_nop 1 + ds_read_b128 a[80:83], v240 offset:10272 + ds_read_b128 a[84:87], v240 offset:10288 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:10272 + s_nop 1 + v_fma_f32 v64, v64, s5, -v32 + v_fma_f32 v65, v65, s5, -v32 + v_fma_f32 v66, v66, s5, -v32 + v_fma_f32 v67, v67, s5, -v32 + v_fma_f32 v68, v68, s5, -v32 + v_fma_f32 v69, v69, s5, -v32 + v_fma_f32 v70, v70, s5, -v32 + v_fma_f32 v71, v71, s5, -v32 + s_nop 1 + ds_read_b128 a[56:59], v13 offset:8320 + ds_read_b128 a[60:63], v13 offset:8336 + v_exp_f32_e32 v64, v64 + v_exp_f32_e32 v65, v65 + v_exp_f32_e32 v66, v66 + ds_read_u8 v237, v238 offset:10272 + s_nop 1 + v_exp_f32_e32 v67, v67 + v_exp_f32_e32 v68, v68 + v_exp_f32_e32 v69, v69 + v_exp_f32_e32 v70, v70 + v_exp_f32_e32 v71, v71 + v_mul_f32_e32 v46, v33, v46 + s_nop 1 + ds_read_b128 a[88:91], v13 offset:10400 + ds_read_b128 a[92:95], v13 offset:10416 + v_add_f32_e32 v26, v65, v64 + v_add_f32_e32 v26, v66, v26 + v_add_f32_e32 v26, v67, v26 + v_add_f32_e32 v26, v68, v26 + s_nop 1 + v_add_f32_e32 v26, v69, v26 + v_add_f32_e32 v26, v70, v26 + v_add_f32_e32 v26, v71, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v64, v64, v65 + v_cvt_pk_bf16_f32 v65, v66, v67 + v_cvt_pk_bf16_f32 v66, v68, v69 + v_cvt_pk_bf16_f32 v67, v70, v71 + s_nop 1 + ds_read_b128 a[64:67], v241 offset:8384 + ds_read_b128 a[68:71], v241 offset:8400 + ds_write_b32 v17, v33 offset:4096 + s_nop 1 + s_nop 1 + ds_read_b128 a[96:99], v241 offset:10464 + ds_read_b128 a[100:103], v241 offset:10480 + ds_write_b64 v20, v[64:65] + s_nop 1 + s_nop 1 + ds_read_b128 a[104:107], v16 + ds_read_b128 a[108:111], v16 offset:16 + ds_write_b64 v20, v[66:67] offset:2048 + s_nop 1 + s_nop 1 + ds_read_b128 a[112:115], v16 offset:2080 + ds_read_b128 a[116:119], v16 offset:2096 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v42, 0, v42 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s59 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[66:67], v19 + v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_B5E8 + buffer_load_dword v40, v38, s[24:27], 0 offen +label_B5E8: + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_B610 + buffer_load_dword v243, v44, s[24:27], 0 offen +label_B610: + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_B734 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 +label_B734: + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + s_cbranch_vccz label_B74C + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_B74C: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v245, 0, v245 + v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s63 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_cmp_eq_u32 s69, 0 + s_cbranch_scc1 label_B960 + s_waitcnt vmcnt(10) +label_B960: + s_waitcnt vmcnt(12) + s_barrier + ds_read_b128 v[80:83], v10 offset:24832 + ds_read_b128 v[84:87], v10 offset:24848 + s_nop 1 + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:26912 + ds_read_b128 v[92:95], v10 offset:26928 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:24832 + s_nop 1 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + s_nop 1 + ds_read_b128 a[48:51], v240 offset:24832 + ds_read_b128 a[52:55], v240 offset:24848 + ds_read_u8 v236, v238 offset:24832 + s_nop 1 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + s_nop 1 + ds_read_b128 a[80:83], v240 offset:26912 + ds_read_b128 a[84:87], v240 offset:26928 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:26912 + s_nop 1 + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + s_nop 1 + ds_read_b128 a[56:59], v13 offset:24960 + ds_read_b128 a[60:63], v13 offset:24976 + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + v_exp_f32_e32 v50, v50 + ds_read_u8 v237, v238 offset:26912 + s_nop 1 + v_exp_f32_e32 v51, v51 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + s_nop 1 + ds_read_b128 a[88:91], v13 offset:27040 + ds_read_b128 a[92:95], v13 offset:27056 + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + s_nop 1 + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_add_f32_e32 v26, v55, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + s_nop 1 + ds_read_b128 a[64:67], v241 offset:25024 + ds_read_b128 a[68:71], v241 offset:25040 + ds_write_b32 v17, v33 offset:4096 + s_nop 1 + s_nop 1 + ds_read_b128 a[96:99], v241 offset:27104 + ds_read_b128 a[100:103], v241 offset:27120 + ds_write_b64 v20, v[48:49] + s_nop 1 + s_nop 1 + ds_read_b128 a[104:107], v16 offset:4160 + ds_read_b128 a[108:111], v16 offset:4176 + ds_write_b64 v20, v[50:51] offset:2048 + s_nop 1 + s_nop 1 + ds_read_b128 a[112:115], v16 offset:6240 + ds_read_b128 a[116:119], v16 offset:6256 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_13644 + s_branch label_A02C +label_BBC0: + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v39, 0, v39 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s56 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[50:51], v19 + v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + buffer_load_dword v41, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + buffer_load_dword v244, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + s_nop 0 + s_cbranch_vccz label_BD58 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_BD58: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v242, 0, v242 + v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s60 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_waitcnt vmcnt(12) + s_barrier + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:41472 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:43552 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v40, 0, v40 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s57 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[66:67], v19 + v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + buffer_load_dword v42, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + buffer_load_dword v245, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + s_nop 0 + s_cbranch_vccz label_C170 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_C170: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v243, 0, v243 + v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s61 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_waitcnt vmcnt(11) + s_barrier + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:58112 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:60192 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v41, 0, v41 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s58 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[50:51], v19 + v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_C4B0 + buffer_load_dword v39, v38, s[24:27], 0 offen +label_C4B0: + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_C4D0 + buffer_load_dword v242, v44, s[24:27], 0 offen +label_C4D0: + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_C58C + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 +label_C58C: + s_nop 0 + s_cbranch_vccz label_C5A0 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_C5A0: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v244, 0, v244 + v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s62 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_cmp_eq_u32 s69, 0 + s_cbranch_scc1 label_C794 + s_waitcnt vmcnt(9) +label_C794: + s_waitcnt vmcnt(12) + s_barrier + ds_read_b128 v[80:83], v10 offset:8192 + ds_read_b128 v[84:87], v10 offset:8208 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:10272 + ds_read_b128 v[92:95], v10 offset:10288 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:8192 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:10272 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v42, 0, v42 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s59 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[66:67], v19 + v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_C8EC + buffer_load_dword v40, v38, s[24:27], 0 offen +label_C8EC: + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_C90C + buffer_load_dword v243, v44, s[24:27], 0 offen +label_C90C: + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_C9C8 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 +label_C9C8: + s_nop 0 + s_cbranch_vccz label_C9DC + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_C9DC: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v245, 0, v245 + v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s63 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_cmp_eq_u32 s69, 0 + s_cbranch_scc1 label_CBD0 + s_waitcnt vmcnt(10) +label_CBD0: + s_waitcnt vmcnt(12) + s_barrier + ds_read_b128 v[80:83], v10 offset:24832 + ds_read_b128 v[84:87], v10 offset:24848 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:26912 + ds_read_b128 v[92:95], v10 offset:26928 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:24832 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:26912 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_14710 + s_branch label_BBC0 +label_CC74: + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v39, 0, v39 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s56 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + buffer_load_dword v41, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + buffer_load_dword v244, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_CE0C + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_CE0C: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v242, 0, v242 + v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s60 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_waitcnt vmcnt(12) + s_barrier + s_nop 1 + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:41472 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:43552 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v40, 0, v40 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s57 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + buffer_load_dword v42, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + buffer_load_dword v245, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_D224 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_D224: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v243, 0, v243 + v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s61 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_waitcnt vmcnt(11) + s_barrier + s_nop 1 + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:58112 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:60192 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v41, 0, v41 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s58 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_D564 + buffer_load_dword v39, v38, s[24:27], 0 offen +label_D564: + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_D584 + buffer_load_dword v242, v44, s[24:27], 0 offen +label_D584: + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_D630 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 +label_D630: + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_D654 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_D654: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v244, 0, v244 + v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s62 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_cmp_eq_u32 s69, 0 + s_cbranch_scc1 label_D848 + s_waitcnt vmcnt(9) +label_D848: + s_waitcnt vmcnt(12) + s_barrier + s_nop 1 + ds_read_b128 v[80:83], v10 offset:8192 + ds_read_b128 v[84:87], v10 offset:8208 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:10272 + ds_read_b128 v[92:95], v10 offset:10288 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:8192 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:10272 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v42, 0, v42 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s59 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_D9A0 + buffer_load_dword v40, v38, s[24:27], 0 offen +label_D9A0: + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_D9C0 + buffer_load_dword v243, v44, s[24:27], 0 offen +label_D9C0: + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_DA6C + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 +label_DA6C: + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_DA90 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_DA90: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v245, 0, v245 + v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s63 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_cmp_eq_u32 s69, 0 + s_cbranch_scc1 label_DC84 + s_waitcnt vmcnt(10) +label_DC84: + s_waitcnt vmcnt(12) + s_barrier + s_nop 1 + ds_read_b128 v[80:83], v10 offset:24832 + ds_read_b128 v[84:87], v10 offset:24848 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:26912 + ds_read_b128 v[92:95], v10 offset:26928 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:24832 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:26912 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_15038 + s_branch label_CC74 +label_DD28: + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v39, 0, v39 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s56 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + buffer_load_dword v41, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 + buffer_load_dword v244, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v23 offset:128 + ds_read_b64_tr_b16 a[140:141], v24 + ds_read_b64_tr_b16 a[142:143], v24 offset:128 + v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v23 offset:512 + ds_read_b64_tr_b16 a[146:147], v23 offset:640 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v24 offset:512 + ds_read_b64_tr_b16 a[150:151], v24 offset:640 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_DEC0 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_DEC0: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v242, 0, v242 + v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s60 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_waitcnt vmcnt(12) + s_barrier + s_nop 1 + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:41472 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:43552 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v40, 0, v40 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s57 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + buffer_load_dword v42, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 offset:4160 + buffer_load_dword v245, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v23 offset:4288 + ds_read_b64_tr_b16 a[140:141], v24 offset:4160 + ds_read_b64_tr_b16 a[142:143], v24 offset:4288 + v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v23 offset:4672 + ds_read_b64_tr_b16 a[146:147], v23 offset:4800 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v24 offset:4672 + ds_read_b64_tr_b16 a[150:151], v24 offset:4800 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_E2D8 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_E2D8: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v243, 0, v243 + v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s61 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_waitcnt vmcnt(11) + s_barrier + s_nop 1 + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:58112 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:60192 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v41, 0, v41 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s58 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_E618 + buffer_load_dword v39, v38, s[24:27], 0 offen +label_E618: + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 offset:8320 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_E638 + buffer_load_dword v242, v44, s[24:27], 0 offen +label_E638: + ds_read_b64_tr_b16 a[138:139], v23 offset:8448 + ds_read_b64_tr_b16 a[140:141], v24 offset:8320 + ds_read_b64_tr_b16 a[142:143], v24 offset:8448 + v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v23 offset:8832 + ds_read_b64_tr_b16 a[146:147], v23 offset:8960 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v24 offset:8832 + ds_read_b64_tr_b16 a[150:151], v24 offset:8960 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_E6E4 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 +label_E6E4: + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_E708 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_E708: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v244, 0, v244 + v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s62 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_cmp_eq_u32 s69, 0 + s_cbranch_scc1 label_E8FC + s_waitcnt vmcnt(9) +label_E8FC: + s_waitcnt vmcnt(12) + s_barrier + s_nop 1 + ds_read_b128 v[80:83], v10 offset:8192 + ds_read_b128 v[84:87], v10 offset:8208 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:10272 + ds_read_b128 v[92:95], v10 offset:10288 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:8192 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:10272 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v42, 0, v42 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s59 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_EA54 + buffer_load_dword v40, v38, s[24:27], 0 offen +label_EA54: + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 offset:12480 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_EA74 + buffer_load_dword v243, v44, s[24:27], 0 offen +label_EA74: + ds_read_b64_tr_b16 a[138:139], v23 offset:12608 + ds_read_b64_tr_b16 a[140:141], v24 offset:12480 + ds_read_b64_tr_b16 a[142:143], v24 offset:12608 + v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v23 offset:12992 + ds_read_b64_tr_b16 a[146:147], v23 offset:13120 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v24 offset:12992 + ds_read_b64_tr_b16 a[150:151], v24 offset:13120 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_EB20 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 +label_EB20: + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_EB44 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_EB44: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(17) + v_max_i32_e32 v245, 0, v245 + v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] + s_nop 1 + s_mov_b32 m0, s63 + global_load_lds_dwordx4 v[2:3], off + s_nop 1 + s_cmp_eq_u32 s69, 0 + s_cbranch_scc1 label_ED38 + s_waitcnt vmcnt(10) +label_ED38: + s_waitcnt vmcnt(12) + s_barrier + s_nop 1 + ds_read_b128 v[80:83], v10 offset:24832 + ds_read_b128 v[84:87], v10 offset:24848 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:26912 + ds_read_b128 v[92:95], v10 offset:26928 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:24832 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:26912 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_15960 + s_branch label_DD28 +label_EDDC: + s_cmp_eq_u32 s88, 8 + s_cselect_b32 s69, 1, 0 + s_cmp_eq_u32 s70, 0 + s_cselect_b32 s69, s69, 0 + s_cmp_lt_u32 s7, 2 + s_cbranch_scc1 label_EE48 + s_cmp_eq_u32 s70, 0 + s_cbranch_scc1 label_EE3C + v_mov_b64_e32 v[48:49], 0 + v_mov_b64_e32 v[50:51], 0 + v_mov_b64_e32 v[52:53], 0 + v_mov_b64_e32 v[54:55], 0 + v_mov_b64_e32 v[56:57], 0 + v_mov_b64_e32 v[58:59], 0 + v_mov_b64_e32 v[60:61], 0 + v_mov_b64_e32 v[62:63], 0 + v_mov_b64_e32 v[64:65], 0 + v_mov_b64_e32 v[66:67], 0 + v_mov_b64_e32 v[68:69], 0 + v_mov_b64_e32 v[70:71], 0 + v_mov_b64_e32 v[72:73], 0 + v_mov_b64_e32 v[74:75], 0 + v_mov_b64_e32 v[76:77], 0 + v_mov_b64_e32 v[78:79], 0 +label_EE3C: + s_cmp_eq_u32 s7, 2 + s_cbranch_scc1 label_10ACC + s_branch label_11C70 +label_EE48: + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v39, 0, v39 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s56 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[50:51], v19 + v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] + buffer_load_dword v41, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] + buffer_load_dword v244, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[52:53], v18 offset:512 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64 v[54:55], v19 offset:512 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] + s_cbranch_vccz label_F0D8 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_F0D8: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v64, v65, v64 + v_max3_f32 v30, v66, v67, v30 + v_max3_f32 v30, v68, v69, v30 + v_max3_f32 v30, v70, v71, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v242, 0, v242 + v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + s_mov_b32 m0, s60 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_waitcnt vmcnt(12) + s_barrier + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:41472 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + s_nop 1 + ds_read_b128 a[48:51], v240 offset:41472 + ds_read_b128 a[52:55], v240 offset:41488 + ds_read_u8 v236, v238 offset:41472 + s_nop 1 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + s_nop 1 + ds_read_b128 a[80:83], v240 offset:43552 + ds_read_b128 a[84:87], v240 offset:43568 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:43552 + s_nop 1 + v_fma_f32 v64, v64, s5, -v32 + v_fma_f32 v65, v65, s5, -v32 + v_fma_f32 v66, v66, s5, -v32 + v_fma_f32 v67, v67, s5, -v32 + v_fma_f32 v68, v68, s5, -v32 + v_fma_f32 v69, v69, s5, -v32 + v_fma_f32 v70, v70, s5, -v32 + v_fma_f32 v71, v71, s5, -v32 + s_nop 1 + ds_read_b128 a[56:59], v13 offset:41600 + ds_read_b128 a[60:63], v13 offset:41616 + v_exp_f32_e32 v64, v64 + v_exp_f32_e32 v65, v65 + v_exp_f32_e32 v66, v66 + ds_read_u8 v237, v238 offset:43552 + s_nop 1 + v_exp_f32_e32 v67, v67 + v_exp_f32_e32 v68, v68 + v_exp_f32_e32 v69, v69 + v_exp_f32_e32 v70, v70 + v_exp_f32_e32 v71, v71 + v_mul_f32_e32 v46, v33, v46 + s_nop 1 + ds_read_b128 a[88:91], v13 offset:43680 + ds_read_b128 a[92:95], v13 offset:43696 + v_add_f32_e32 v26, v65, v64 + v_add_f32_e32 v26, v66, v26 + v_add_f32_e32 v26, v67, v26 + v_add_f32_e32 v26, v68, v26 + s_nop 1 + v_add_f32_e32 v26, v69, v26 + v_add_f32_e32 v26, v70, v26 + v_add_f32_e32 v26, v71, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v64, v64, v65 + v_cvt_pk_bf16_f32 v65, v66, v67 + v_cvt_pk_bf16_f32 v66, v68, v69 + v_cvt_pk_bf16_f32 v67, v70, v71 + s_nop 1 + ds_read_b128 a[64:67], v241 offset:41664 + ds_read_b128 a[68:71], v241 offset:41680 + ds_write_b32 v17, v33 offset:4096 + s_nop 1 + s_nop 1 + ds_read_b128 a[96:99], v241 offset:43744 + ds_read_b128 a[100:103], v241 offset:43760 + ds_write_b64 v20, v[64:65] + s_nop 1 + s_nop 1 + ds_read_b128 a[104:107], v16 offset:8320 + ds_read_b128 a[108:111], v16 offset:8336 + ds_write_b64 v20, v[66:67] offset:2048 + s_nop 1 + s_nop 1 + ds_read_b128 a[112:115], v16 offset:10400 + ds_read_b128 a[116:119], v16 offset:10416 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v40, 0, v40 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s57 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[66:67], v19 + v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + buffer_load_dword v42, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + buffer_load_dword v245, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[68:69], v18 offset:512 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64 v[70:71], v19 offset:512 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + s_cbranch_vccz label_F7E4 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_F7E4: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v243, 0, v243 + v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] + s_mov_b32 m0, s61 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] + s_waitcnt vmcnt(11) + s_barrier + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:58112 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + s_nop 1 + ds_read_b128 a[48:51], v240 offset:58112 + ds_read_b128 a[52:55], v240 offset:58128 + ds_read_u8 v236, v238 offset:58112 + s_nop 1 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + s_nop 1 + ds_read_b128 a[80:83], v240 offset:60192 + ds_read_b128 a[84:87], v240 offset:60208 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:60192 + s_nop 1 + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + s_nop 1 + ds_read_b128 a[56:59], v13 offset:58240 + ds_read_b128 a[60:63], v13 offset:58256 + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + v_exp_f32_e32 v50, v50 + ds_read_u8 v237, v238 offset:60192 + s_nop 1 + v_exp_f32_e32 v51, v51 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + s_nop 1 + ds_read_b128 a[88:91], v13 offset:60320 + ds_read_b128 a[92:95], v13 offset:60336 + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + s_nop 1 + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_add_f32_e32 v26, v55, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + s_nop 1 + ds_read_b128 a[64:67], v241 offset:58304 + ds_read_b128 a[68:71], v241 offset:58320 + ds_write_b32 v17, v33 offset:4096 + s_nop 1 + s_nop 1 + ds_read_b128 a[96:99], v241 offset:60384 + ds_read_b128 a[100:103], v241 offset:60400 + ds_write_b64 v20, v[48:49] + s_nop 1 + s_nop 1 + ds_read_b128 a[104:107], v16 offset:12480 + ds_read_b128 a[108:111], v16 offset:12496 + ds_write_b64 v20, v[50:51] offset:2048 + s_nop 1 + s_nop 1 + ds_read_b128 a[112:115], v16 offset:14560 + ds_read_b128 a[116:119], v16 offset:14576 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v41, 0, v41 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s58 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[50:51], v19 + v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_FD90 + buffer_load_dword v39, v38, s[24:27], 0 offen +label_FD90: + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_FDB8 + buffer_load_dword v242, v44, s[24:27], 0 offen +label_FDB8: + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[52:53], v18 offset:512 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64 v[54:55], v19 offset:512 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_FEF0 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 +label_FEF0: + v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] + s_cbranch_vccz label_FF08 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_FF08: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v64, v65, v64 + v_max3_f32 v30, v66, v67, v30 + v_max3_f32 v30, v68, v69, v30 + v_max3_f32 v30, v70, v71, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v244, 0, v244 + v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + s_mov_b32 m0, s62 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_cmp_eq_u32 s69, 0 + s_cbranch_scc1 label_10130 + s_waitcnt vmcnt(9) +label_10130: + s_waitcnt vmcnt(12) + s_barrier + ds_read_b128 v[80:83], v10 offset:8192 + ds_read_b128 v[84:87], v10 offset:8208 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + ds_read_b128 v[88:91], v10 offset:10272 + ds_read_b128 v[92:95], v10 offset:10288 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:8192 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + s_nop 1 + ds_read_b128 a[48:51], v240 offset:8192 + ds_read_b128 a[52:55], v240 offset:8208 + ds_read_u8 v236, v238 offset:8192 + s_nop 1 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + s_nop 1 + ds_read_b128 a[80:83], v240 offset:10272 + ds_read_b128 a[84:87], v240 offset:10288 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:10272 + s_nop 1 + v_fma_f32 v64, v64, s5, -v32 + v_fma_f32 v65, v65, s5, -v32 + v_fma_f32 v66, v66, s5, -v32 + v_fma_f32 v67, v67, s5, -v32 + v_fma_f32 v68, v68, s5, -v32 + v_fma_f32 v69, v69, s5, -v32 + v_fma_f32 v70, v70, s5, -v32 + v_fma_f32 v71, v71, s5, -v32 + s_nop 1 + ds_read_b128 a[56:59], v13 offset:8320 + ds_read_b128 a[60:63], v13 offset:8336 + v_exp_f32_e32 v64, v64 + v_exp_f32_e32 v65, v65 + v_exp_f32_e32 v66, v66 + ds_read_u8 v237, v238 offset:10272 + s_nop 1 + v_exp_f32_e32 v67, v67 + v_exp_f32_e32 v68, v68 + v_exp_f32_e32 v69, v69 + v_exp_f32_e32 v70, v70 + v_exp_f32_e32 v71, v71 + v_mul_f32_e32 v46, v33, v46 + s_nop 1 + ds_read_b128 a[88:91], v13 offset:10400 + ds_read_b128 a[92:95], v13 offset:10416 + v_add_f32_e32 v26, v65, v64 + v_add_f32_e32 v26, v66, v26 + v_add_f32_e32 v26, v67, v26 + v_add_f32_e32 v26, v68, v26 + s_nop 1 + v_add_f32_e32 v26, v69, v26 + v_add_f32_e32 v26, v70, v26 + v_add_f32_e32 v26, v71, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v64, v64, v65 + v_cvt_pk_bf16_f32 v65, v66, v67 + v_cvt_pk_bf16_f32 v66, v68, v69 + v_cvt_pk_bf16_f32 v67, v70, v71 + s_nop 1 + ds_read_b128 a[64:67], v241 offset:8384 + ds_read_b128 a[68:71], v241 offset:8400 + ds_write_b32 v17, v33 offset:4096 + s_nop 1 + s_nop 1 + ds_read_b128 a[96:99], v241 offset:10464 + ds_read_b128 a[100:103], v241 offset:10480 + ds_write_b64 v20, v[64:65] + s_nop 1 + s_nop 1 + ds_read_b128 a[104:107], v16 + ds_read_b128 a[108:111], v16 offset:16 + ds_write_b64 v20, v[66:67] offset:2048 + s_nop 1 + s_nop 1 + ds_read_b128 a[112:115], v16 offset:2080 + ds_read_b128 a[116:119], v16 offset:2096 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v42, 0, v42 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + s_mov_b32 m0, s59 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[66:67], v19 + v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_104C0 + buffer_load_dword v40, v38, s[24:27], 0 offen +label_104C0: + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_104E8 + buffer_load_dword v243, v44, s[24:27], 0 offen +label_104E8: + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[68:69], v18 offset:512 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64 v[70:71], v19 offset:512 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_10620 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 +label_10620: + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + s_cbranch_vccz label_10638 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_10638: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v245, 0, v245 + v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] + s_mov_b32 m0, s63 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] + s_cmp_eq_u32 s69, 0 + s_cbranch_scc1 label_10860 + s_waitcnt vmcnt(10) +label_10860: + s_waitcnt vmcnt(12) + s_barrier + ds_read_b128 v[80:83], v10 offset:24832 + ds_read_b128 v[84:87], v10 offset:24848 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] + ds_read_b128 v[88:91], v10 offset:26912 + ds_read_b128 v[92:95], v10 offset:26928 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:24832 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + s_nop 1 + ds_read_b128 a[48:51], v240 offset:24832 + ds_read_b128 a[52:55], v240 offset:24848 + ds_read_u8 v236, v238 offset:24832 + s_nop 1 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + s_nop 1 + ds_read_b128 a[80:83], v240 offset:26912 + ds_read_b128 a[84:87], v240 offset:26928 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:26912 + s_nop 1 + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + s_nop 1 + ds_read_b128 a[56:59], v13 offset:24960 + ds_read_b128 a[60:63], v13 offset:24976 + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + v_exp_f32_e32 v50, v50 + ds_read_u8 v237, v238 offset:26912 + s_nop 1 + v_exp_f32_e32 v51, v51 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + s_nop 1 + ds_read_b128 a[88:91], v13 offset:27040 + ds_read_b128 a[92:95], v13 offset:27056 + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + s_nop 1 + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_add_f32_e32 v26, v55, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + s_nop 1 + ds_read_b128 a[64:67], v241 offset:25024 + ds_read_b128 a[68:71], v241 offset:25040 + ds_write_b32 v17, v33 offset:4096 + s_nop 1 + s_nop 1 + ds_read_b128 a[96:99], v241 offset:27104 + ds_read_b128 a[100:103], v241 offset:27120 + ds_write_b64 v20, v[48:49] + s_nop 1 + s_nop 1 + ds_read_b128 a[104:107], v16 offset:4160 + ds_read_b128 a[108:111], v16 offset:4176 + ds_write_b64 v20, v[50:51] offset:2048 + s_nop 1 + s_nop 1 + ds_read_b128 a[112:115], v16 offset:6240 + ds_read_b128 a[116:119], v16 offset:6256 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_16288 + s_branch label_EE48 +label_10ACC: + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v39, 0, v39 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s56 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + buffer_load_dword v41, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + buffer_load_dword v244, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[52:53], v18 offset:512 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64 v[54:55], v19 offset:512 + v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_10C80 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_10C80: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v242, 0, v242 + v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + s_mov_b32 m0, s60 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_waitcnt vmcnt(12) + s_barrier + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:41472 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:43552 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v40, 0, v40 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s57 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + buffer_load_dword v42, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + buffer_load_dword v245, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[68:69], v18 offset:512 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64 v[70:71], v19 offset:512 + v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_110D4 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_110D4: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v243, 0, v243 + v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] + s_mov_b32 m0, s61 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] + s_waitcnt vmcnt(11) + s_barrier + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:58112 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:60192 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v41, 0, v41 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s58 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_1143C + buffer_load_dword v39, v38, s[24:27], 0 offen +label_1143C: + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_1145C + buffer_load_dword v242, v44, s[24:27], 0 offen +label_1145C: + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[52:53], v18 offset:512 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64 v[54:55], v19 offset:512 + v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_1151C + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 +label_1151C: + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_11540 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_11540: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v244, 0, v244 + v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + s_mov_b32 m0, s62 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_cmp_eq_u32 s69, 0 + s_cbranch_scc1 label_11748 + s_waitcnt vmcnt(9) +label_11748: + s_waitcnt vmcnt(12) + s_barrier + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + ds_read_b128 v[80:83], v10 offset:8192 + ds_read_b128 v[84:87], v10 offset:8208 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + ds_read_b128 v[88:91], v10 offset:10272 + ds_read_b128 v[92:95], v10 offset:10288 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:8192 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:10272 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v42, 0, v42 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s59 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_118B4 + buffer_load_dword v40, v38, s[24:27], 0 offen +label_118B4: + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_118D4 + buffer_load_dword v243, v44, s[24:27], 0 offen +label_118D4: + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[68:69], v18 offset:512 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64 v[70:71], v19 offset:512 + v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_11994 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 +label_11994: + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_119B8 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_119B8: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v245, 0, v245 + v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] + s_mov_b32 m0, s63 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] + s_cmp_eq_u32 s69, 0 + s_cbranch_scc1 label_11BC0 + s_waitcnt vmcnt(10) +label_11BC0: + s_waitcnt vmcnt(12) + s_barrier + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] + ds_read_b128 v[80:83], v10 offset:24832 + ds_read_b128 v[84:87], v10 offset:24848 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] + ds_read_b128 v[88:91], v10 offset:26912 + ds_read_b128 v[92:95], v10 offset:26928 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:24832 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:26912 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_17408 + s_branch label_10ACC +label_11C70: + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v39, 0, v39 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s56 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + buffer_load_dword v41, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 + buffer_load_dword v244, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v23 offset:128 + ds_read_b64 v[52:53], v18 offset:512 + ds_read_b64_tr_b16 a[140:141], v24 + ds_read_b64_tr_b16 a[142:143], v24 offset:128 + ds_read_b64 v[54:55], v19 offset:512 + v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v23 offset:512 + ds_read_b64_tr_b16 a[146:147], v23 offset:640 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v24 offset:512 + ds_read_b64_tr_b16 a[150:151], v24 offset:640 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_11E24 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_11E24: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v242, 0, v242 + v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + s_mov_b32 m0, s60 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_waitcnt vmcnt(12) + s_barrier + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:41472 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:43552 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v40, 0, v40 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s57 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + buffer_load_dword v42, v38, s[24:27], 0 offen + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 offset:4160 + buffer_load_dword v245, v44, s[24:27], 0 offen + ds_read_b64_tr_b16 a[138:139], v23 offset:4288 + ds_read_b64 v[68:69], v18 offset:512 + ds_read_b64_tr_b16 a[140:141], v24 offset:4160 + ds_read_b64_tr_b16 a[142:143], v24 offset:4288 + ds_read_b64 v[70:71], v19 offset:512 + v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v23 offset:4672 + ds_read_b64_tr_b16 a[146:147], v23 offset:4800 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v24 offset:4672 + ds_read_b64_tr_b16 a[150:151], v24 offset:4800 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_12278 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_12278: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v243, 0, v243 + v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] + s_mov_b32 m0, s61 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] + s_waitcnt vmcnt(11) + s_barrier + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:58112 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:60192 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v41, 0, v41 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s58 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_125E0 + buffer_load_dword v39, v38, s[24:27], 0 offen +label_125E0: + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 offset:8320 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_12600 + buffer_load_dword v242, v44, s[24:27], 0 offen +label_12600: + ds_read_b64_tr_b16 a[138:139], v23 offset:8448 + ds_read_b64 v[52:53], v18 offset:512 + ds_read_b64_tr_b16 a[140:141], v24 offset:8320 + ds_read_b64_tr_b16 a[142:143], v24 offset:8448 + ds_read_b64 v[54:55], v19 offset:512 + v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v23 offset:8832 + ds_read_b64_tr_b16 a[146:147], v23 offset:8960 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v24 offset:8832 + ds_read_b64_tr_b16 a[150:151], v24 offset:8960 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_126C0 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 +label_126C0: + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_126E4 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_126E4: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v244, 0, v244 + v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + s_mov_b32 m0, s62 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_cmp_eq_u32 s69, 0 + s_cbranch_scc1 label_128EC + s_waitcnt vmcnt(9) +label_128EC: + s_waitcnt vmcnt(12) + s_barrier + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + ds_read_b128 v[80:83], v10 offset:8192 + ds_read_b128 v[84:87], v10 offset:8208 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + ds_read_b128 v[88:91], v10 offset:10272 + ds_read_b128 v[92:95], v10 offset:10288 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:8192 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:10272 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + v_max_i32_e32 v42, 0, v42 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + s_mov_b32 m0, s59 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_12A58 + buffer_load_dword v40, v38, s[24:27], 0 offen +label_12A58: + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 offset:12480 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_12A78 + buffer_load_dword v243, v44, s[24:27], 0 offen +label_12A78: + ds_read_b64_tr_b16 a[138:139], v23 offset:12608 + ds_read_b64 v[68:69], v18 offset:512 + ds_read_b64_tr_b16 a[140:141], v24 offset:12480 + ds_read_b64_tr_b16 a[142:143], v24 offset:12608 + ds_read_b64 v[70:71], v19 offset:512 + v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + ds_read_b64_tr_b16 a[144:145], v23 offset:12992 + ds_read_b64_tr_b16 a[146:147], v23 offset:13120 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + ds_read_b64_tr_b16 a[148:149], v24 offset:12992 + ds_read_b64_tr_b16 a[150:151], v24 offset:13120 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_12B38 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 +label_12B38: + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + s_nop 0 + s_cbranch_vccz label_12B5C + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_12B5C: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] + s_waitcnt vmcnt(17) + v_max_i32_e32 v245, 0, v245 + v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] + s_mov_b32 m0, s63 + global_load_lds_dwordx4 v[2:3], off + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] + s_cmp_eq_u32 s69, 0 + s_cbranch_scc1 label_12D64 + s_waitcnt vmcnt(10) +label_12D64: + s_waitcnt vmcnt(12) + s_barrier + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] + ds_read_b128 v[80:83], v10 offset:24832 + ds_read_b128 v[84:87], v10 offset:24848 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] + ds_read_b128 v[88:91], v10 offset:26912 + ds_read_b128 v[92:95], v10 offset:26928 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:24832 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:26912 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_cmp_eq_u32 s69, 1 + s_cbranch_scc1 label_17DE4 + s_branch label_11C70 +label_12E14: + s_cmp_ge_u32 s7, 2 + s_cbranch_scc1 label_13230 + s_waitcnt lgkmcnt(0) + s_barrier + v_mov_b32_e32 v33, 1.0 + v_mov_b64_e32 v[48:49], 0 + v_mov_b64_e32 v[50:51], 0 + v_mov_b64_e32 v[52:53], 0 + v_mov_b64_e32 v[54:55], 0 + v_mov_b64_e32 v[56:57], 0 + v_mov_b64_e32 v[58:59], 0 + v_mov_b64_e32 v[60:61], 0 + v_mov_b64_e32 v[62:63], 0 + v_mov_b64_e32 v[64:65], 0 + v_mov_b64_e32 v[66:67], 0 + v_mov_b64_e32 v[68:69], 0 + v_mov_b64_e32 v[70:71], 0 + v_mov_b64_e32 v[72:73], 0 + v_mov_b64_e32 v[74:75], 0 + v_mov_b64_e32 v[76:77], 0 + v_mov_b64_e32 v[78:79], 0 + s_nop 1 + buffer_load_dword v39, v38, s[24:27], 0 offen + buffer_load_dword v242, v44, s[24:27], 0 offen + v_max_i32_e32 v41, 0, v41 + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + s_nop 1 + v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + s_nop 1 + s_mov_b32 m0, s58 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + s_nop 1 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_cvt_scalef32_pk_bf16_fp8 v34, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v81, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[34:37] + v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cvt_scalef32_pk_bf16_fp8 v34, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v83, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[34:37] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v34, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v85, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[34:37] offset:32 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_cvt_scalef32_pk_bf16_fp8 v34, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v87, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[34:37] offset:48 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + s_nop 1 + v_cvt_scalef32_pk_bf16_fp8 v34, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v89, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[34:37] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v34, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v91, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[34:37] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v34, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[34:37] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v34, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v95, v33 op_sel:[1,0,0] + ds_write_b128 v11, v[34:37] offset:14640 + v_max_i32_e32 v244, 0, v244 + v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] + s_mov_b32 m0, s62 + global_load_lds_dwordx4 v[2:3], off + s_waitcnt vmcnt(5) + s_barrier + ds_read_u8 v236, v238 offset:24832 + ds_read_u8 v237, v238 offset:26912 + ds_read_b128 v[80:83], v10 offset:24832 + ds_read_b128 v[84:87], v10 offset:24848 + ds_read_b128 v[88:91], v10 offset:26912 + ds_read_b128 v[92:95], v10 offset:26928 + v_max_i32_e32 v42, 0, v42 + s_nop 1 + v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + buffer_load_dword v40, v38, s[24:27], 0 offen + buffer_load_dword v243, v44, s[24:27], 0 offen + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + s_mov_b32 m0, s59 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_max_i32_e32 v245, 0, v245 + v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] + s_mov_b32 m0, s63 + global_load_lds_dwordx4 v[2:3], off + s_branch label_1F1C +label_13230: + s_waitcnt lgkmcnt(0) + s_barrier + v_mov_b32_e32 v33, 1.0 + v_mov_b64_e32 v[48:49], 0 + v_mov_b64_e32 v[50:51], 0 + v_mov_b64_e32 v[52:53], 0 + v_mov_b64_e32 v[54:55], 0 + v_mov_b64_e32 v[56:57], 0 + v_mov_b64_e32 v[58:59], 0 + v_mov_b64_e32 v[60:61], 0 + v_mov_b64_e32 v[62:63], 0 + v_mov_b64_e32 v[64:65], 0 + v_mov_b64_e32 v[66:67], 0 + v_mov_b64_e32 v[68:69], 0 + v_mov_b64_e32 v[70:71], 0 + v_mov_b64_e32 v[72:73], 0 + v_mov_b64_e32 v[74:75], 0 + v_mov_b64_e32 v[76:77], 0 + v_mov_b64_e32 v[78:79], 0 + s_nop 1 + buffer_load_dword v39, v38, s[24:27], 0 offen + buffer_load_dword v242, v44, s[24:27], 0 offen + v_max_i32_e32 v41, 0, v41 + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + s_nop 1 + v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + s_nop 1 + s_mov_b32 m0, s58 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + s_nop 1 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_cvt_scalef32_pk_bf16_fp8 v34, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v81, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[34:37] + v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + v_cvt_scalef32_pk_bf16_fp8 v34, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v83, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[34:37] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v34, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v85, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[34:37] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v34, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v87, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[34:37] offset:48 + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + s_nop 1 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + v_cvt_scalef32_pk_bf16_fp8 v34, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v89, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[34:37] offset:14592 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_cvt_scalef32_pk_bf16_fp8 v34, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v91, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[34:37] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v34, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[34:37] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v34, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v35, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v36, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v37, v95, v33 op_sel:[1,0,0] + ds_write_b128 v11, v[34:37] offset:14640 + v_max_i32_e32 v244, 0, v244 + v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] + s_mov_b32 m0, s62 + global_load_lds_dwordx4 v[2:3], off + s_waitcnt vmcnt(5) + s_barrier + ds_read_u8 v236, v238 offset:24832 + ds_read_u8 v237, v238 offset:26912 + ds_read_b128 v[80:83], v10 offset:24832 + ds_read_b128 v[84:87], v10 offset:24848 + ds_read_b128 v[88:91], v10 offset:26912 + ds_read_b128 v[92:95], v10 offset:26928 + v_max_i32_e32 v42, 0, v42 + s_nop 1 + v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] + v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf + v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf + v_cndmask_b32_e64 v26, v26, v27, s[92:93] + v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] + buffer_load_dword v40, v38, s[24:27], 0 offen + buffer_load_dword v243, v44, s[24:27], 0 offen + v_add_u32_e32 v38, s94, v38 + v_add_u32_e32 v44, s94, v44 + s_mov_b32 m0, s59 + global_load_lds_dwordx4 v[2:3], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[4:5], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[6:7], off + s_add_u32 m0, m0, 0x410 + global_load_lds_dwordx4 v[8:9], off + s_add_u32 m0, m0, 0x410 + v_max_i32_e32 v245, 0, v245 + v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] + s_mov_b32 m0, s63 + global_load_lds_dwordx4 v[2:3], off + s_branch label_1F1C +label_13644: + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[50:51], v19 + v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_cmp_neq_f32_e32 vcc, 1.0, v34 + v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] + v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] + s_nop 2 + s_cbranch_vccz label_137DC + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_137DC: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v64, v65, v64 + v_max3_f32 v30, v66, v67, v30 + v_max3_f32 v30, v68, v69, v30 + v_max3_f32 v30, v70, v71, v30 + s_nop 1 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(5) + s_barrier + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + s_nop 1 + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:41472 + s_nop 1 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + s_nop 1 + ds_read_b128 a[48:51], v240 offset:41472 + ds_read_b128 a[52:55], v240 offset:41488 + ds_read_u8 v236, v238 offset:41472 + s_nop 1 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + s_nop 1 + ds_read_b128 a[80:83], v240 offset:43552 + ds_read_b128 a[84:87], v240 offset:43568 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:43552 + s_nop 1 + v_fma_f32 v64, v64, s5, -v32 + v_fma_f32 v65, v65, s5, -v32 + v_fma_f32 v66, v66, s5, -v32 + v_fma_f32 v67, v67, s5, -v32 + v_fma_f32 v68, v68, s5, -v32 + v_fma_f32 v69, v69, s5, -v32 + v_fma_f32 v70, v70, s5, -v32 + v_fma_f32 v71, v71, s5, -v32 + s_nop 1 + ds_read_b128 a[56:59], v13 offset:41600 + ds_read_b128 a[60:63], v13 offset:41616 + v_exp_f32_e32 v64, v64 + v_exp_f32_e32 v65, v65 + v_exp_f32_e32 v66, v66 + ds_read_u8 v237, v238 offset:43552 + s_nop 1 + v_exp_f32_e32 v67, v67 + v_exp_f32_e32 v68, v68 + v_exp_f32_e32 v69, v69 + v_exp_f32_e32 v70, v70 + v_exp_f32_e32 v71, v71 + v_mul_f32_e32 v46, v33, v46 + s_nop 1 + ds_read_b128 a[88:91], v13 offset:43680 + ds_read_b128 a[92:95], v13 offset:43696 + v_add_f32_e32 v26, v65, v64 + v_add_f32_e32 v26, v66, v26 + v_add_f32_e32 v26, v67, v26 + v_add_f32_e32 v26, v68, v26 + s_nop 1 + v_add_f32_e32 v26, v69, v26 + v_add_f32_e32 v26, v70, v26 + v_add_f32_e32 v26, v71, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v64, v64, v65 + v_cvt_pk_bf16_f32 v65, v66, v67 + v_cvt_pk_bf16_f32 v66, v68, v69 + v_cvt_pk_bf16_f32 v67, v70, v71 + s_nop 1 + ds_read_b128 a[64:67], v241 offset:41664 + ds_read_b128 a[68:71], v241 offset:41680 + ds_write_b32 v17, v33 offset:4096 + s_nop 1 + s_nop 1 + ds_read_b128 a[96:99], v241 offset:43744 + ds_read_b128 a[100:103], v241 offset:43760 + ds_write_b64 v20, v[64:65] + s_nop 1 + s_nop 1 + ds_read_b128 a[104:107], v16 offset:8320 + ds_read_b128 a[108:111], v16 offset:8336 + ds_write_b64 v20, v[66:67] offset:2048 + s_nop 1 + s_nop 1 + ds_read_b128 a[112:115], v16 offset:10400 + ds_read_b128 a[116:119], v16 offset:10416 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[66:67], v19 + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_cmp_neq_f32_e32 vcc, 1.0, v34 + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + s_nop 2 + s_cbranch_vccz label_13DB4 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_13DB4: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + s_nop 1 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(0) + s_barrier + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + s_nop 1 + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:58112 + s_nop 1 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + s_nop 1 + ds_read_b128 a[48:51], v240 offset:58112 + ds_read_b128 a[52:55], v240 offset:58128 + ds_read_u8 v236, v238 offset:58112 + s_nop 1 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + s_nop 1 + ds_read_b128 a[80:83], v240 offset:60192 + ds_read_b128 a[84:87], v240 offset:60208 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:60192 + s_nop 1 + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + s_nop 1 + ds_read_b128 a[56:59], v13 offset:58240 + ds_read_b128 a[60:63], v13 offset:58256 + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + v_exp_f32_e32 v50, v50 + ds_read_u8 v237, v238 offset:60192 + s_nop 1 + v_exp_f32_e32 v51, v51 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + s_nop 1 + ds_read_b128 a[88:91], v13 offset:60320 + ds_read_b128 a[92:95], v13 offset:60336 + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + s_nop 1 + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_add_f32_e32 v26, v55, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + s_nop 1 + ds_read_b128 a[64:67], v241 offset:58304 + ds_read_b128 a[68:71], v241 offset:58320 + ds_write_b32 v17, v33 offset:4096 + s_nop 1 + s_nop 1 + ds_read_b128 a[96:99], v241 offset:60384 + ds_read_b128 a[100:103], v241 offset:60400 + ds_write_b64 v20, v[48:49] + s_nop 1 + s_nop 1 + ds_read_b128 a[104:107], v16 offset:12480 + ds_read_b128 a[108:111], v16 offset:12496 + ds_write_b64 v20, v[50:51] offset:2048 + s_nop 1 + s_nop 1 + ds_read_b128 a[112:115], v16 offset:14560 + ds_read_b128 a[116:119], v16 offset:14576 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[50:51], v19 + v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_cmp_neq_f32_e32 vcc, 1.0, v34 + v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] + v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] + s_nop 2 + s_cbranch_vccz label_1438C + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_1438C: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v64, v65, v64 + v_max3_f32 v30, v66, v67, v30 + v_max3_f32 v30, v68, v69, v30 + v_max3_f32 v30, v70, v71, v30 + s_nop 1 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(0) + s_barrier + s_nop 1 + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + s_nop 1 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + s_nop 1 + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + s_nop 1 + s_nop 1 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + s_nop 1 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + s_nop 1 + v_fma_f32 v64, v64, s5, -v32 + v_fma_f32 v65, v65, s5, -v32 + v_fma_f32 v66, v66, s5, -v32 + v_fma_f32 v67, v67, s5, -v32 + v_fma_f32 v68, v68, s5, -v32 + v_fma_f32 v69, v69, s5, -v32 + v_fma_f32 v70, v70, s5, -v32 + v_fma_f32 v71, v71, s5, -v32 + s_nop 1 + v_exp_f32_e32 v64, v64 + v_exp_f32_e32 v65, v65 + v_exp_f32_e32 v66, v66 + s_nop 1 + v_exp_f32_e32 v67, v67 + v_exp_f32_e32 v68, v68 + v_exp_f32_e32 v69, v69 + v_exp_f32_e32 v70, v70 + v_exp_f32_e32 v71, v71 + v_mul_f32_e32 v46, v33, v46 + s_nop 1 + v_add_f32_e32 v26, v65, v64 + v_add_f32_e32 v26, v66, v26 + v_add_f32_e32 v26, v67, v26 + v_add_f32_e32 v26, v68, v26 + s_nop 1 + v_add_f32_e32 v26, v69, v26 + v_add_f32_e32 v26, v70, v26 + v_add_f32_e32 v26, v71, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v64, v64, v65 + v_cvt_pk_bf16_f32 v65, v66, v67 + v_cvt_pk_bf16_f32 v66, v68, v69 + v_cvt_pk_bf16_f32 v67, v70, v71 + s_nop 1 + ds_write_b32 v17, v33 offset:4096 + s_nop 1 + s_nop 1 + ds_write_b64 v20, v[64:65] + s_nop 1 + s_nop 1 + ds_write_b64 v20, v[66:67] offset:2048 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_branch label_3D68 +label_14710: + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_nop 4 + s_cbranch_vccz label_147C8 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_147C8: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(5) + s_barrier + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:41472 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:43552 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[66:67], v19 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_nop 4 + s_cbranch_vccz label_14AE4 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_14AE4: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(0) + s_barrier + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:58112 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:60192 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_nop 4 + s_cbranch_vccz label_14E00 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_14E00: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(0) + s_barrier + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_branch label_3D68 +label_15038: + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_nop 4 + s_cbranch_vccz label_150F0 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_150F0: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(5) + s_barrier + s_nop 1 + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:41472 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:43552 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_nop 4 + s_cbranch_vccz label_1540C + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_1540C: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(0) + s_barrier + s_nop 1 + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:58112 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:60192 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_nop 4 + s_cbranch_vccz label_15728 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_15728: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(0) + s_barrier + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_branch label_3D68 +label_15960: + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 + ds_read_b64_tr_b16 a[138:139], v23 offset:128 + ds_read_b64_tr_b16 a[140:141], v24 + ds_read_b64_tr_b16 a[142:143], v24 offset:128 + ds_read_b64_tr_b16 a[144:145], v23 offset:512 + ds_read_b64_tr_b16 a[146:147], v23 offset:640 + ds_read_b64_tr_b16 a[148:149], v24 offset:512 + ds_read_b64_tr_b16 a[150:151], v24 offset:640 + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_nop 4 + s_cbranch_vccz label_15A18 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_15A18: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(5) + s_barrier + s_nop 1 + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:41472 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:43552 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 offset:4160 + ds_read_b64_tr_b16 a[138:139], v23 offset:4288 + ds_read_b64_tr_b16 a[140:141], v24 offset:4160 + ds_read_b64_tr_b16 a[142:143], v24 offset:4288 + ds_read_b64_tr_b16 a[144:145], v23 offset:4672 + ds_read_b64_tr_b16 a[146:147], v23 offset:4800 + ds_read_b64_tr_b16 a[148:149], v24 offset:4672 + ds_read_b64_tr_b16 a[150:151], v24 offset:4800 + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_nop 4 + s_cbranch_vccz label_15D34 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_15D34: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(0) + s_barrier + s_nop 1 + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + s_nop 1 + s_nop 1 + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:58112 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:60192 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 offset:8320 + ds_read_b64_tr_b16 a[138:139], v23 offset:8448 + ds_read_b64_tr_b16 a[140:141], v24 offset:8320 + ds_read_b64_tr_b16 a[142:143], v24 offset:8448 + ds_read_b64_tr_b16 a[144:145], v23 offset:8832 + ds_read_b64_tr_b16 a[146:147], v23 offset:8960 + ds_read_b64_tr_b16 a[148:149], v24 offset:8832 + ds_read_b64_tr_b16 a[150:151], v24 offset:8960 + v_cmp_neq_f32_e32 vcc, 1.0, v34 + s_nop 4 + s_cbranch_vccz label_16050 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9F34 +label_16050: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + s_nop 1 + ds_write_b128 v11, v[26:29] offset:14640 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_waitcnt vmcnt(0) + s_barrier + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_branch label_3D68 +label_16288: + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[50:51], v19 + v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[52:53], v18 offset:512 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64 v[54:55], v19 offset:512 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] + v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] + s_nop 2 + s_cbranch_vccz label_1643C + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_1643C: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v64, v65, v64 + v_max3_f32 v30, v66, v67, v30 + v_max3_f32 v30, v68, v69, v30 + v_max3_f32 v30, v70, v71, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_waitcnt vmcnt(5) + s_barrier + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:41472 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + s_nop 1 + ds_read_b128 a[48:51], v240 offset:41472 + ds_read_b128 a[52:55], v240 offset:41488 + ds_read_u8 v236, v238 offset:41472 + s_nop 1 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + s_nop 1 + ds_read_b128 a[80:83], v240 offset:43552 + ds_read_b128 a[84:87], v240 offset:43568 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:43552 + s_nop 1 + v_fma_f32 v64, v64, s5, -v32 + v_fma_f32 v65, v65, s5, -v32 + v_fma_f32 v66, v66, s5, -v32 + v_fma_f32 v67, v67, s5, -v32 + v_fma_f32 v68, v68, s5, -v32 + v_fma_f32 v69, v69, s5, -v32 + v_fma_f32 v70, v70, s5, -v32 + v_fma_f32 v71, v71, s5, -v32 + s_nop 1 + ds_read_b128 a[56:59], v13 offset:41600 + ds_read_b128 a[60:63], v13 offset:41616 + v_exp_f32_e32 v64, v64 + v_exp_f32_e32 v65, v65 + v_exp_f32_e32 v66, v66 + ds_read_u8 v237, v238 offset:43552 + s_nop 1 + v_exp_f32_e32 v67, v67 + v_exp_f32_e32 v68, v68 + v_exp_f32_e32 v69, v69 + v_exp_f32_e32 v70, v70 + v_exp_f32_e32 v71, v71 + v_mul_f32_e32 v46, v33, v46 + s_nop 1 + ds_read_b128 a[88:91], v13 offset:43680 + ds_read_b128 a[92:95], v13 offset:43696 + v_add_f32_e32 v26, v65, v64 + v_add_f32_e32 v26, v66, v26 + v_add_f32_e32 v26, v67, v26 + v_add_f32_e32 v26, v68, v26 + s_nop 1 + v_add_f32_e32 v26, v69, v26 + v_add_f32_e32 v26, v70, v26 + v_add_f32_e32 v26, v71, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v64, v64, v65 + v_cvt_pk_bf16_f32 v65, v66, v67 + v_cvt_pk_bf16_f32 v66, v68, v69 + v_cvt_pk_bf16_f32 v67, v70, v71 + s_nop 1 + ds_read_b128 a[64:67], v241 offset:41664 + ds_read_b128 a[68:71], v241 offset:41680 + ds_write_b32 v17, v33 offset:4096 + s_nop 1 + s_nop 1 + ds_read_b128 a[96:99], v241 offset:43744 + ds_read_b128 a[100:103], v241 offset:43760 + ds_write_b64 v20, v[64:65] + s_nop 1 + s_nop 1 + ds_read_b128 a[104:107], v16 offset:8320 + ds_read_b128 a[108:111], v16 offset:8336 + ds_write_b64 v20, v[66:67] offset:2048 + s_nop 1 + s_nop 1 + ds_read_b128 a[112:115], v16 offset:10400 + ds_read_b128 a[116:119], v16 offset:10416 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[66:67], v19 + v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[68:69], v18 offset:512 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64 v[70:71], v19 offset:512 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] + v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] + s_nop 2 + s_cbranch_vccz label_16A50 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_16A50: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v48, v49, v48 + v_max3_f32 v30, v50, v51, v30 + v_max3_f32 v30, v52, v53, v30 + v_max3_f32 v30, v54, v55, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] + s_waitcnt vmcnt(0) + s_barrier + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + ds_read_b128 v[228:231], v1 offset:58112 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + s_nop 1 + ds_read_b128 a[48:51], v240 offset:58112 + ds_read_b128 a[52:55], v240 offset:58128 + ds_read_u8 v236, v238 offset:58112 + s_nop 1 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + s_nop 1 + ds_read_b128 a[80:83], v240 offset:60192 + ds_read_b128 a[84:87], v240 offset:60208 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + ds_read_b128 v[232:235], v1 offset:60192 + s_nop 1 + v_fma_f32 v48, v48, s5, -v32 + v_fma_f32 v49, v49, s5, -v32 + v_fma_f32 v50, v50, s5, -v32 + v_fma_f32 v51, v51, s5, -v32 + v_fma_f32 v52, v52, s5, -v32 + v_fma_f32 v53, v53, s5, -v32 + v_fma_f32 v54, v54, s5, -v32 + v_fma_f32 v55, v55, s5, -v32 + s_nop 1 + ds_read_b128 a[56:59], v13 offset:58240 + ds_read_b128 a[60:63], v13 offset:58256 + v_exp_f32_e32 v48, v48 + v_exp_f32_e32 v49, v49 + v_exp_f32_e32 v50, v50 + ds_read_u8 v237, v238 offset:60192 + s_nop 1 + v_exp_f32_e32 v51, v51 + v_exp_f32_e32 v52, v52 + v_exp_f32_e32 v53, v53 + v_exp_f32_e32 v54, v54 + v_exp_f32_e32 v55, v55 + v_mul_f32_e32 v46, v33, v46 + s_nop 1 + ds_read_b128 a[88:91], v13 offset:60320 + ds_read_b128 a[92:95], v13 offset:60336 + v_add_f32_e32 v26, v49, v48 + v_add_f32_e32 v26, v50, v26 + v_add_f32_e32 v26, v51, v26 + v_add_f32_e32 v26, v52, v26 + s_nop 1 + v_add_f32_e32 v26, v53, v26 + v_add_f32_e32 v26, v54, v26 + v_add_f32_e32 v26, v55, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v48, v48, v49 + v_cvt_pk_bf16_f32 v49, v50, v51 + v_cvt_pk_bf16_f32 v50, v52, v53 + v_cvt_pk_bf16_f32 v51, v54, v55 + s_nop 1 + ds_read_b128 a[64:67], v241 offset:58304 + ds_read_b128 a[68:71], v241 offset:58320 + ds_write_b32 v17, v33 offset:4096 + s_nop 1 + s_nop 1 + ds_read_b128 a[96:99], v241 offset:60384 + ds_read_b128 a[100:103], v241 offset:60400 + ds_write_b64 v20, v[48:49] + s_nop 1 + s_nop 1 + ds_read_b128 a[104:107], v16 offset:12480 + ds_read_b128 a[108:111], v16 offset:12496 + ds_write_b64 v20, v[50:51] offset:2048 + s_nop 1 + s_nop 1 + ds_read_b128 a[112:115], v16 offset:14560 + ds_read_b128 a[116:119], v16 offset:14576 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(2) + s_barrier + v_perm_b32 v32, v228, v229, v12 + v_perm_b32 v228, v229, v228, v12 + v_mov_b32_e32 v229, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + v_bfe_u32 v230, v230, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + v_and_b32_e32 v231, v47, v231 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64 v[50:51], v19 + v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[52:53], v18 offset:512 + v_perm_b32 v32, v232, v233, v12 + v_perm_b32 v232, v233, v232, v12 + v_mov_b32_e32 v233, v32 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + v_bfe_u32 v234, v234, v25, 8 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64 v[54:55], v19 offset:512 + v_and_b32_e32 v235, v47, v235 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] + v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] + s_nop 2 + s_cbranch_vccz label_17064 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_17064: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + ds_write_b128 v11, v[26:29] offset:14640 + v_max3_f32 v30, v64, v65, v64 + v_max3_f32 v30, v66, v67, v30 + v_max3_f32 v30, v68, v69, v30 + v_max3_f32 v30, v70, v71, v30 + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_waitcnt vmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + v_mov_b32_e32 v26, v30 + v_mov_b32_e32 v27, v30 + s_nop 1 + v_permlane16_swap_b32_e32 v26, v27 + v_mov_b32_e32 v29, v26 + v_mov_b32_e32 v28, v27 + s_nop 1 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_permlane32_swap_b32_e32 v26, v27 + v_permlane32_swap_b32_e32 v28, v29 + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + v_max3_f32 v30, v26, v27, v30 + v_max3_f32 v30, v28, v29, v30 + v_cmp_eq_u32_e64 s[64:65], s74, v45 + v_max_f32_e32 v31, v30, v45 + s_nop 1 + s_nop 1 + v_sub_f32_e32 v27, v31, v45 + v_mul_f32_e32 v27, s5, v27 + v_cmp_lt_f32_e32 vcc, 0x41000000, v27 + v_sub_f32_e32 v33, v45, v31 + v_cndmask_b32_e64 v33, v33, 0, s[64:65] + v_cndmask_b32_e32 v33, 0, v33, vcc + v_cndmask_b32_e32 v45, v45, v31, vcc + s_nop 1 + v_mul_f32_e32 v32, s5, v45 + v_mul_f32_e32 v33, s5, v33 + v_exp_f32_e32 v33, v33 + s_nop 1 + v_fma_f32 v64, v64, s5, -v32 + v_fma_f32 v65, v65, s5, -v32 + v_fma_f32 v66, v66, s5, -v32 + v_fma_f32 v67, v67, s5, -v32 + v_fma_f32 v68, v68, s5, -v32 + v_fma_f32 v69, v69, s5, -v32 + v_fma_f32 v70, v70, s5, -v32 + v_fma_f32 v71, v71, s5, -v32 + s_nop 1 + v_exp_f32_e32 v64, v64 + v_exp_f32_e32 v65, v65 + v_exp_f32_e32 v66, v66 + s_nop 1 + v_exp_f32_e32 v67, v67 + v_exp_f32_e32 v68, v68 + v_exp_f32_e32 v69, v69 + v_exp_f32_e32 v70, v70 + v_exp_f32_e32 v71, v71 + v_mul_f32_e32 v46, v33, v46 + s_nop 1 + v_add_f32_e32 v26, v65, v64 + v_add_f32_e32 v26, v66, v26 + v_add_f32_e32 v26, v67, v26 + v_add_f32_e32 v26, v68, v26 + s_nop 1 + v_add_f32_e32 v26, v69, v26 + v_add_f32_e32 v26, v70, v26 + v_add_f32_e32 v26, v71, v26 + v_add_f32_e32 v46, v26, v46 + v_cvt_pk_bf16_f32 v64, v64, v65 + v_cvt_pk_bf16_f32 v65, v66, v67 + v_cvt_pk_bf16_f32 v66, v68, v69 + v_cvt_pk_bf16_f32 v67, v70, v71 + s_nop 1 + ds_write_b32 v17, v33 offset:4096 + s_nop 1 + s_nop 1 + ds_write_b64 v20, v[64:65] + s_nop 1 + s_nop 1 + ds_write_b64 v20, v[66:67] offset:2048 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_branch label_3D68 +label_17408: + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[52:53], v18 offset:512 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64 v[54:55], v19 offset:512 + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_nop 4 + s_cbranch_vccz label_174DC + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_174DC: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_waitcnt vmcnt(5) + s_barrier + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:41472 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:43552 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[68:69], v18 offset:512 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64 v[70:71], v19 offset:512 + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_nop 4 + s_cbranch_vccz label_17834 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_17834: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] + s_waitcnt vmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:58112 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:60192 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v21 offset:128 + ds_read_b64_tr_b16 a[138:139], v21 offset:3776 + ds_read_b64 v[52:53], v18 offset:512 + ds_read_b64_tr_b16 a[140:141], v22 offset:128 + ds_read_b64_tr_b16 a[142:143], v22 offset:3776 + ds_read_b64 v[54:55], v19 offset:512 + ds_read_b64_tr_b16 a[144:145], v21 offset:192 + ds_read_b64_tr_b16 a[146:147], v21 offset:3840 + ds_read_b64_tr_b16 a[148:149], v22 offset:192 + ds_read_b64_tr_b16 a[150:151], v22 offset:3840 + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_nop 4 + s_cbranch_vccz label_17B8C + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_17B8C: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_waitcnt vmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_branch label_3D68 +label_17DE4: + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 + ds_read_b64_tr_b16 a[138:139], v23 offset:128 + ds_read_b64 v[52:53], v18 offset:512 + ds_read_b64_tr_b16 a[140:141], v24 + ds_read_b64_tr_b16 a[142:143], v24 offset:128 + ds_read_b64 v[54:55], v19 offset:512 + ds_read_b64_tr_b16 a[144:145], v23 offset:512 + ds_read_b64_tr_b16 a[146:147], v23 offset:640 + ds_read_b64_tr_b16 a[148:149], v24 offset:512 + ds_read_b64_tr_b16 a[150:151], v24 offset:640 + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_nop 4 + s_cbranch_vccz label_17EB8 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_17EB8: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_waitcnt vmcnt(5) + s_barrier + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + ds_read_b128 v[80:83], v10 offset:41472 + ds_read_b128 v[84:87], v10 offset:41488 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + ds_read_b128 v[88:91], v10 offset:43552 + ds_read_b128 v[92:95], v10 offset:43568 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:41472 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:43552 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[64:65], v18 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[66:67], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 offset:4160 + ds_read_b64_tr_b16 a[138:139], v23 offset:4288 + ds_read_b64 v[68:69], v18 offset:512 + ds_read_b64_tr_b16 a[140:141], v24 offset:4160 + ds_read_b64_tr_b16 a[142:143], v24 offset:4288 + ds_read_b64 v[70:71], v19 offset:512 + ds_read_b64_tr_b16 a[144:145], v23 offset:4672 + ds_read_b64_tr_b16 a[146:147], v23 offset:4800 + ds_read_b64_tr_b16 a[148:149], v24 offset:4672 + ds_read_b64_tr_b16 a[150:151], v24 offset:4800 + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_nop 4 + s_cbranch_vccz label_18210 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_18210: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] + s_waitcnt vmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] + ds_read_b128 v[80:83], v10 offset:58112 + ds_read_b128 v[84:87], v10 offset:58128 + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] + ds_read_b128 v[88:91], v10 offset:60192 + ds_read_b128 v[92:95], v10 offset:60208 + s_nop 1 + s_nop 1 + ds_read_u8 v236, v238 offset:58112 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + ds_read_u8 v237, v238 offset:60192 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + ds_read_b32 v34, v43 offset:4096 + ds_read_b32 v35, v43 offset:4352 + ds_read_b64_tr_b16 a[120:121], v21 + ds_read_b64_tr_b16 a[122:123], v21 offset:3648 + ds_read_b64_tr_b16 a[124:125], v22 + ds_read_b64 v[48:49], v18 + ds_read_b64_tr_b16 a[126:127], v22 offset:3648 + ds_read_b64_tr_b16 a[128:129], v21 offset:64 + ds_read_b64 v[50:51], v19 + ds_read_b64_tr_b16 a[130:131], v21 offset:3712 + ds_read_b64_tr_b16 a[132:133], v22 offset:64 + ds_read_b64_tr_b16 a[134:135], v22 offset:3712 + ds_read_b64_tr_b16 a[136:137], v23 offset:8320 + ds_read_b64_tr_b16 a[138:139], v23 offset:8448 + ds_read_b64 v[52:53], v18 offset:512 + ds_read_b64_tr_b16 a[140:141], v24 offset:8320 + ds_read_b64_tr_b16 a[142:143], v24 offset:8448 + ds_read_b64 v[54:55], v19 offset:512 + ds_read_b64_tr_b16 a[144:145], v23 offset:8832 + ds_read_b64_tr_b16 a[146:147], v23 offset:8960 + ds_read_b64_tr_b16 a[148:149], v24 offset:8832 + ds_read_b64_tr_b16 a[150:151], v24 offset:8960 + v_min_f32_e32 v26, v34, v35 + v_cmp_neq_f32_e32 vcc, 1.0, v26 + s_nop 4 + s_cbranch_vccz label_18568 + s_getpc_b64 vcc + s_add_i32 vcc_lo, vcc_lo, 8 + s_branch label_9EB4 +label_18568: + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_waitcnt lgkmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] + v_lshlrev_b32_e32 v33, 23, v236 + v_cmp_eq_u32_e64 vcc, v236, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] + ds_write_b128 v11, v[26:29] + v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] + ds_write_b128 v11, v[26:29] offset:16 + v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] + ds_write_b128 v11, v[26:29] offset:32 + v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] + ds_write_b128 v11, v[26:29] offset:48 + v_lshlrev_b32_e32 v33, 23, v237 + v_cmp_eq_u32_e64 vcc, v237, 0 + v_cndmask_b32_e64 v33, v33, 0, vcc + v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] + ds_write_b128 v11, v[26:29] offset:14592 + v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] + ds_write_b128 v11, v[26:29] offset:14608 + v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] + ds_write_b128 v11, v[26:29] offset:14624 + v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 + v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] + v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 + v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] + v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] + ds_write_b128 v11, v[26:29] offset:14640 + v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] + v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] + v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] + v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] + s_waitcnt vmcnt(0) + s_barrier + v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] + v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] + v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_nop 1 + s_addk_i32 s87, 0x1 + s_cmp_lt_i32 s87, s88 + s_cbranch_scc0 label_3D68 + v_permlane32_swap_b32_e32 v26, v26 + v_permlane32_swap_b32_e32 v26, v26 + s_branch label_3D68 +.rodata +.p2align 6 +.globl _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE.kd +.type _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE.kd,@object +.size _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE.kd, 64 +_ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE.kd: +.byte 0, 128, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +.quad _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE - _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE.kd +.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 63, 3, 12, 0, 132, 3, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0 +.amdgpu_metadata +--- +amdhsa.kernels: + - .args: + - .actual_access: read_write + .address_space: global + .name: O + .offset: 0 + .size: 8 + .value_kind: global_buffer + - .name: pad + .offset: 8 + .size: 8 + .value_kind: by_value + .value_type: i32 + - .actual_access: read_write + .address_space: global + .name: Q + .offset: 16 + .size: 8 + .value_kind: global_buffer + - .name: pad + .offset: 24 + .size: 8 + .value_kind: by_value + .value_type: i32 + - .actual_access: read_only + .address_space: global + .name: K + .offset: 32 + .size: 8 + .value_kind: global_buffer + - .name: pad + .offset: 40 + .size: 8 + .value_kind: by_value + .value_type: i32 + - .actual_access: read_only + .address_space: global + .name: V + .offset: 48 + .size: 8 + .value_kind: global_buffer + - .name: pad + .offset: 56 + .size: 8 + .value_kind: by_value + .value_type: i32 + - .actual_access: read_only + .address_space: global + .name: BT + .offset: 64 + .size: 8 + .value_kind: global_buffer + - .name: pad + .offset: 72 + .size: 8 + .value_kind: by_value + .value_type: i32 + - .actual_access: read_only + .address_space: global + .name: CL + .offset: 80 + .size: 8 + .value_kind: global_buffer + - .name: pad + .offset: 88 + .size: 8 + .value_kind: by_value + .value_type: i32 + - .actual_access: read_only + .address_space: global + .name: KQ + .offset: 96 + .size: 8 + .value_kind: global_buffer + - .name: pad + .offset: 104 + .size: 8 + .value_kind: by_value + .value_type: i32 + - .name: sclg + .offset: 112 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 116 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 120 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 124 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: mblk + .offset: 128 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 132 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 136 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 140 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: batch + .offset: 144 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 148 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 152 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 156 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: Qs + .offset: 160 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 164 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 168 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 172 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: Bs + .offset: 176 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 180 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 184 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 188 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: Cs + .offset: 192 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 196 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 200 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 204 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .actual_access: read_only + .address_space: global + .name: QT + .offset: 208 + .size: 8 + .value_kind: global_buffer + - .name: pad + .offset: 216 + .size: 8 + .value_kind: by_value + .value_type: i32 + - .actual_access: read_only + .address_space: global + .name: ST + .offset: 224 + .size: 8 + .value_kind: global_buffer + - .name: pad + .offset: 232 + .size: 8 + .value_kind: by_value + .value_type: i32 + - .actual_access: read_write + .address_space: global + .name: RP + .offset: 240 + .size: 8 + .value_kind: global_buffer + - .name: pad + .offset: 248 + .size: 8 + .value_kind: by_value + .value_type: i32 + - .actual_access: read_only + .address_space: global + .name: DQ + .offset: 256 + .size: 8 + .value_kind: global_buffer + - .name: pad + .offset: 264 + .size: 8 + .value_kind: by_value + .value_type: i32 + - .actual_access: read_only + .address_space: global + .name: DK + .offset: 272 + .size: 8 + .value_kind: global_buffer + - .name: pad + .offset: 280 + .size: 8 + .value_kind: by_value + .value_type: i32 + - .name: out16 + .offset: 288 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 292 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 296 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 300 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .actual_access: read_write + .address_space: global + .name: VALID_SPLIT + .offset: 304 + .size: 8 + .value_kind: global_buffer + - .name: pad + .offset: 312 + .size: 8 + .value_kind: by_value + .value_type: i32 + - .name: use_valid_split + .offset: 320 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 324 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 328 + .size: 4 + .value_kind: by_value + .value_type: i32 + - .name: pad + .offset: 332 + .size: 4 + .value_kind: by_value + .value_type: i32 + .group_segment_fixed_size: 163840 + .kernarg_segment_align: 4 + .kernarg_segment_size: 336 + .max_flat_workgroup_size: 256 + .name: _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE + .private_segment_fixed_size: 0 + .reqd_workgroup_size: + - 256 + - 1 + - 1 + .sgpr_count: 96 + .symbol: _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE.kd + .vgpr_count: 512 + .wavefront_size: 64 +amdhsa.version: + - 1 + - 2 +... +.end_amdgpu_metadata diff --git a/hsa/gfx950/mla_v4/nm_ps_src/gen.py b/hsa/gfx950/mla_v4/nm_ps_src/gen.py new file mode 100644 index 00000000000..2fb09666a8e --- /dev/null +++ b/hsa/gfx950/mla_v4/nm_ps_src/gen.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""Regenerate hsa/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co from sources (python3 stdlib only). + + python3 gen.py --out [--check ] [--keep

] [--verify-rt ] + +Steps: asm/rt.s --(asm/patch.py: persistent walk + splice/pre.hip planner + splice/fin.hip fused combine, both +compiled with hipcc -S)--> patched .s --(kernel symbol renamed to the _ps name)--> clang -x assembler (gfx950) +--> ld.lld -shared --> .co. Tools come from $ROCM_PATH (default /opt/rocm).""" + +import argparse +import hashlib +import os +import re +import shutil +import subprocess +import sys +import tempfile + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "asm")) +import patch as PATCH # noqa: E402 + +RT_SHA256 = "7fc7516ef916ba83770744ed5d55faafb57684908fd31b76727adbc6ae9bd20b" +OLD_NAME = "mla_a8w8_qh64_qseqlen1_gqaratio64_nm" +NEW_NAME = "mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps" + + +def mangle(name): + """aiter:: (Itanium mangling of a namespace-scope name, as used by the AITER ASM kernels).""" + return f"_ZN5aiter{len(name)}{name}E" + + +def tools(): + rocm = os.environ.get("ROCM_PATH") or "/opt/rocm" + t = dict( + clang=f"{rocm}/llvm/bin/clang", + lld=f"{rocm}/llvm/bin/ld.lld", + readelf=f"{rocm}/llvm/bin/llvm-readelf", + objcopy=f"{rocm}/llvm/bin/llvm-objcopy", + hipcc=f"{rocm}/bin/hipcc", + ) + for k, p in t.items(): + if not os.access(p, os.X_OK): + sys.exit(f"gen.py: {k} not found at {p} (set ROCM_PATH)") + return t + + +def sha256(path): + return hashlib.sha256(open(path, "rb").read()).hexdigest() + + +def rename(text, old, new): + """Rename the kernel symbol everywhere (symbol, .kd, metadata .name / .symbol); returns (text, occurrences).""" + mo, mn = mangle(old), mangle(new) + assert mn not in text, mn + pat = re.compile( + re.escape(mo) + r"(?![A-Za-z0-9_])" + ) # also matches the ".kd" occurrences + text, n = pat.subn(mn, text) + assert n > 0 and mo not in text, (mo, n) + return text, n + + +def assemble(t, s_path, co_path): + o_path = os.path.splitext(co_path)[0] + ".o" + subprocess.run( + [ + t["clang"], + "-x", + "assembler", + "-target", + "amdgcn-amd-amdhsa", + "-mcpu=gfx950", + "-c", + s_path, + "-o", + o_path, + ], + check=True, + ) + subprocess.run([t["lld"], "-shared", o_path, "-o", co_path], check=True) + + +def section(t, co, name, d): + out = os.path.join(d, os.path.basename(co) + name) + subprocess.run( + [t["objcopy"], "-O", "binary", f"--only-section={name}", co, out], check=True + ) + return open(out, "rb").read() + + +def main(): + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("--out", required=True, help="output code object (.co)") + ap.add_argument( + "--check", + help="expected .co: fail unless the output is byte-identical (sha256)", + ) + ap.add_argument( + "--keep", + help="directory to keep the intermediates in (hipcc .s, patched .s, .o)", + ) + ap.add_argument( + "--verify-rt", + help="the #5195 .co (mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co): check that asm/rt.s " + "reassembles to its .text and kernel descriptor", + ) + a = ap.parse_args() + t = tools() + rt = os.path.join(HERE, "asm", "rt.s") + got = sha256(rt) + if got != RT_SHA256: + sys.exit(f"gen.py: asm/rt.s sha256 {got} != {RT_SHA256}") + tmp = tempfile.TemporaryDirectory() + work = a.keep or tmp.name + os.makedirs(work, exist_ok=True) + if a.verify_rt: + rco = os.path.join(work, "rt.co") + assemble(t, rt, rco) + for sec in (".text", ".rodata"): + same = section(t, rco, sec, work) == section(t, a.verify_rt, sec, work) + print(f"verify-rt {sec}: {'identical' if same else 'DIFFERENT'}") + if not same: + sys.exit(1) + text, marks, pads = PATCH.patch( + open(rt).read(), t["clang"], t["readelf"], t["hipcc"], work + ) + print("patched", marks, "pads", pads) + text, n = rename(text, OLD_NAME, NEW_NAME) + print(f"renamed {mangle(OLD_NAME)} -> {mangle(NEW_NAME)} ({n} occurrences)") + s_path = os.path.join(work, NEW_NAME + ".s") + open(s_path, "w").write(text) + co_tmp = os.path.join(work, NEW_NAME + ".co") + assemble(t, s_path, co_tmp) + out = os.path.abspath(a.out) + os.makedirs(os.path.dirname(out), exist_ok=True) + shutil.copyfile(co_tmp, out) + print(f"wrote {out} sha256 {sha256(out)}") + if a.check: + exp = sha256(a.check) + ok = exp == sha256(out) + print(f"check {a.check} sha256 {exp}: {'IDENTICAL' if ok else 'MISMATCH'}") + if not ok: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/hsa/gfx950/mla_v4/nm_ps_src/splice/fin.hip b/hsa/gfx950/mla_v4/nm_ps_src/splice/fin.hip new file mode 100644 index 00000000000..0df8a63fdc8 --- /dev/null +++ b/hsa/gfx950/mla_v4/nm_ps_src/splice/fin.hip @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: MIT +// Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. +// +// Splice source: fused last-arriver combine, spliced by asm/patch.py at the end of every SPLIT piece +// (after the ASM epilogue stored the piece's partial o_acc[slot] / lse_acc[slot], before the walk's barrier). +// Only compiled to assembly (hipcc -S). ABI of the splice: kernarg pointer s[0:1], thread id v0, exec = all. +// Inputs (uniform, AGPRs written by the wrapper): a230 row r, a231 slot, a232 partition p, a233 hg, a234 tb, +// a235 te (unclamped), a236 kv_indptr[r], a237 kv_indptr[r+1]; from pre.hip: a248 per, a249 Pa, a250 org. +// Kernarg: 0x00 o_acc, 0x08 out, 0x10 lse_acc, 0x18 lse_out (0 = none), 0x98 cnt (uint32, zero at rest: +// [0, 2*65536) row counters, [2*65536, +16*512) reserved (unused), [CNTG, +4*1024) group counters), 0xb0 planner cfg, 0x120 sink. +// +// Protocol (per row r and head group hg). The pieces of r live in partitions pf..pl (pf = owner(r, 0), pl = owner(r, T-1) +// on the planner's cost line); a partition q in between holds a piece iff its tile range [ftg(q*per), ftg((q+1)*per)) is +// non-empty (partitions may be empty when per < 32); slot(q) = 2q, except the first piece: 2pf + (r is not the begin row +// of pf), whose parity travels through the counters (bit 30, added by the piece with tb == 0). +// * nq = pl - pf + 1 <= FIN_TREE_MIN ("narrow" row): every piece, after its partial is complete (s_waitcnt vmcnt(0) in +// every wave + barrier), does ONE agent-scope atomic add on cnt[2r+hg] of ntiles (+ parity). The piece that brings the +// tile sum to T(r) is the last arriver: it zeroes the counter (next launch starts from 0; ordered by the kernel +// boundary) and merges all pieces (+ sink) into out (bf16) and lse_out. +// * wider rows (two levels, fan-in FIN_G): partitions are grouped by (q - pf) / FIN_G; group counter +// cntg[4*q0 + 2*(q0 == pf) + hg] (q0 = first partition of the group; unique per (row, group)); the group's last arriver +// merges its pieces (no sink) into an fp32 partial + log2 lse stored IN PLACE in the slot of the group's first piece, +// then adds the group's tiles (+ parity for group 0) to the row counter; the row's last arriver merges the group +// partials (+ sink). The two levels spread the merge of wide rows over several CUs. +// Coherence: sc1 instruction scope instead of cache maintenance. The ASM's partial stores are built with sc1 +// (patch.py sc1: agent-scope write-through = LLVM's "store atomic monotonic agent" encoding on gfx942/950), group +// partials are stored with sc1 buffer stores, and every merge read is an sc1 load ("load atomic monotonic agent"): the +// release is s_waitcnt vmcnt(0) + barrier before the counter atomic, the acquire needs no buffer_inv; no buffer_wbl2. +// Merge math (FlashMLA combine, log2 domain): Mx = max(sk, lse_z); S = exp2(sk-Mx) + sum exp2(lse_z-Mx); +// out = sum exp2(lse_z-Mx) o_z / S; lse_out = (Mx + log2 S) ln2. Wave w merges heads hg*64 + 16w .. +15. +#include "hca_common.hip" + +typedef unsigned v4u __attribute__((ext_vector_type(4))); +struct KA { char b[336]; }; +// kernarg reads through the constant-address-space kernarg pointer -> s_load (the by-value struct compiled to vector loads) +#define KARG(T, off) (*(T const __attribute__((address_space(4)))*)((const __attribute__((address_space(4))) char*)__builtin_amdgcn_kernarg_segment_ptr() + (off))) +template __device__ __forceinline__ int rda() { + int v; + asm volatile("v_accvgpr_read_b32 %0, a%c1" : "=v"(v) : "i"(A)); + return __builtin_amdgcn_readfirstlane(v); +} +template __device__ __forceinline__ T* uni(T* p) { // make a pointer provably uniform (SGPR pair) + const unsigned long long u = (unsigned long long)p; + return (T*)(((unsigned long long)__builtin_amdgcn_readfirstlane((unsigned)(u >> 32)) << 32) | + (unsigned)__builtin_amdgcn_readfirstlane((unsigned)u)); +} +#define FIN_MAXJ 256 +#define FIN_NPF 100000 // rows spanning more than FIN_NPF partitions would be left to an external combine: none +#define FIN_TREE_MIN 8 // rows spanning more partitions than this use the two-level merge +#define FIN_G 4 // fan-in of the first level +#define FIN_LR 4 // lse values per lane held in registers (pieces 0..15); more pieces are re-read from memory +#define CNTG (2 * 65536 + 16 * 512) +#define CPOL_SC1 16 +#define OCP " sc1" + +extern "C" __global__ void __launch_bounds__(256) t8fin(KA ka) { + __shared__ unsigned s_dec[2]; + __shared__ int s_slot[4][FIN_MAXJ]; + __shared__ float s_wt[4][FIN_MAXJ + 1][16]; + const int r = rda<230>(), slot = rda<231>(), p = rda<232>(), hg = rda<233>(), tb = rda<234>(), te = rda<235>(); + const int i0 = rda<236>(), i1 = rda<237>(); + const int per = rda<248>(), Pa = rda<249>(), org = rda<250>(); + float* o_acc = uni(KARG(float*, 0x00)); + __bf16* out = uni(KARG(__bf16*, 0x08)); + float* lse_acc = uni(KARG(float*, 0x10)); + float* lse_out = uni(KARG(float*, 0x18)); + unsigned* cnt = uni(KARG(unsigned*, 0x98)); + const int F = __builtin_amdgcn_readfirstlane(KARG(int, 0xb0)); + const float* sink = uni(KARG(const float*, 0x120)); + const int T = (i1 - i0 + TILE - 1) >> 5; + const int nt = min(te, T) - tb; + const int C0 = cfg_C0(F); + const int b = i0 - org + r * C0; + const int pf = min(Pa - 1, (b + C0) / per), pl = min(Pa - 1, (b + C0 + TILE * (T - 1)) / per); + auto ftg = [&](int pos) { const int d = pos - b - C0; const int i = d <= 0 ? 0 : (d + TILE - 1) >> 5; return min(i, T); }; + auto t0of = [&](int q) { return q == pf ? 0 : ftg(q * per); }; + auto t1of = [&](int q) { return q == pl ? T : ftg((q + 1) * per); }; + const int nq = pl - pf + 1; + if (nq > FIN_NPF) return; // never taken (FIN_NPF exceeds any partition count); pre.hip mirrors it for desc.comb + const bool tree = nq > FIN_TREE_MIN; + const int gi = tree ? (p - pf) / FIN_G : 0; + const int q0 = tree ? pf + gi * FIN_G : pf, q1 = tree ? min(pl, q0 + FIN_G - 1) : pl; + unsigned* c1 = tree ? cnt + CNTG + 4 * q0 + 2 * (q0 == pf) + hg : cnt + 2 * r + hg; + const int w = threadIdx.x >> 6, l = threadIdx.x & 63; + const int target1 = tree ? t1of(q1) - t0of(q0) : T; + const unsigned par = tb == 0 ? (unsigned)(slot & 1) << 30 : 0u; + + // one counter step: all waves' stores complete -> barrier -> thread 0 atomic (+reset if last) -> barrier -> decision + auto arrive = [&](unsigned* c, unsigned add, int target) -> unsigned { + asm volatile("s_waitcnt vmcnt(0)" ::: "memory"); + __syncthreads(); + if (threadIdx.x == 0) { + const unsigned nw = __hip_atomic_fetch_add(c, add, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT) + add; + const bool last = (int)(nw & 0x3fffffffu) == target; + if (last) __hip_atomic_store(c, 0u, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); + s_dec[0] = last ? 1u | ((nw >> 30) & 1u) << 1 : 0u; + } + __syncthreads(); + return s_dec[0]; + }; + unsigned dec = arrive(c1, (unsigned)nt | par, target1); + if (!(dec & 1u)) return; + + const int hw0 = hg * 64 + 16 * w; // first head of this wave + const int hl = l & 15, jl = l >> 4; + const __amdgpu_buffer_rsrc_t rl = __builtin_amdgcn_make_buffer_rsrc(lse_acc, 0, 0x7fffffff, 0x00020000); + const __amdgpu_buffer_rsrc_t ro = __builtin_amdgcn_make_buffer_rsrc(o_acc, 0, 0x7fffffff, 0x00020000); + auto ldl = [&](int j) { + return __builtin_bit_cast(float, __builtin_amdgcn_raw_buffer_load_b32(rl, (s_slot[w][j] * NH + hw0 + hl) * 4, 0, CPOL_SC1)); + }; + // slots of the pieces in partitions [qa, qb] (lanes over partitions) -> s_slot[w][0..np), one private copy per wave + auto slots_pieces = [&](int qa, int qb, unsigned pr) -> int { + int np = 0; + for (int j0 = qa; j0 <= qb; j0 += 64) { + const int q = j0 + l; + const bool has = q <= qb && t0of(q) < t1of(q); + const unsigned long long m = __ballot(has); + if (has) { + const int k = np + __popcll(m & ((1ull << l) - 1)); + if (k < FIN_MAXJ) s_slot[w][k] = 2 * q + (q == pf ? (int)pr : 0); + } + np += __popcll(m); + } + __builtin_amdgcn_s_waitcnt(0xc07f); // lgkmcnt(0) + __builtin_amdgcn_wave_barrier(); + return __builtin_amdgcn_readfirstlane(min(np, FIN_MAXJ)); + }; + + // ---- merge s_slot[w][0..np) for heads hw0..hw0+15. final: + sink -> out bf16 / lse_out (natural log); + // !final: no sink -> fp32 partial + log2 lse written in place into slot s_slot[w][0] (sc1 stores) ---- + auto merge = [&](int np, bool final) { + // weights: lane l = (piece j = jl + 4i, head hl); lse of pieces 0..15 in registers, one round trip + const float sk = final ? sink[hw0 + hl] * LOG2E : -__builtin_inff(); + float lz[FIN_LR]; +#pragma unroll + for (int i = 0; i < FIN_LR; ++i) lz[i] = ldl(min(jl + 4 * i, np - 1)); + // O pipeline: 2 head groups (8 heads) x np pieces, batch b = g*npp + j (npp = np rounded up to even), 16 x 1 KB + // sc1 loads per wave per batch into one of 2 register buffers; inline asm with explicit s_waitcnt vmcnt(16) before + // each consume (vmcnt completes in order; stores in between only make the wait conservative). The compiler's + // own waitcnt pass would drain vmcnt(0) at the loop head, serialising every batch. + constexpr int HB = 8; + const int npp = (np + 1) & ~1, nbt = 2 * npp; + v4f buf0[HB][2], buf1[HB][2]; + auto issue = [&](int bb, v4f (&bf)[HB][2]) { + bb = min(bb, nbt - 1); + const int g = bb >= npp ? 1 : 0, jj = min(bb - g * npp, np - 1); + const int sl = __builtin_amdgcn_readfirstlane(s_slot[w][jj]); + const float* base = uni(o_acc + ((size_t)sl * NH + hw0 + g * HB) * DV); + const int vo = 16 * l; + // the hazard recognizer does not look into inline asm: VALU (v_readfirstlane) SGPR write -> VMEM saddr read + // needs 5 wait states on gfx9 + asm volatile("s_nop 4" :: "s"(base)); +#pragma unroll + for (int k = 0; k < HB; ++k) + asm volatile("global_load_dwordx4 %0, %2, %3" OCP "\n\tglobal_load_dwordx4 %1, %2, %3 offset:1024" OCP + : "=&v"(bf[k][0]), "=&v"(bf[k][1]) : "v"(vo + k * DV * 4), "s"(base) : "memory"); + }; +#define FIN_TIE16(bf, CNT) \ + asm volatile("s_waitcnt vmcnt(" #CNT ")" : "+v"(bf[0][0]), "+v"(bf[0][1]), "+v"(bf[1][0]), "+v"(bf[1][1]), \ + "+v"(bf[2][0]), "+v"(bf[2][1]), "+v"(bf[3][0]), "+v"(bf[3][1]), "+v"(bf[4][0]), "+v"(bf[4][1]), \ + "+v"(bf[5][0]), "+v"(bf[5][1]), "+v"(bf[6][0]), "+v"(bf[6][1]), "+v"(bf[7][0]), "+v"(bf[7][1]) :: "memory") + issue(0, buf0); + float Mx = sk; +#pragma unroll + for (int i = 0; i < FIN_LR; ++i) if (jl + 4 * i < np) Mx = fmaxf(Mx, lz[i]); + for (int j = jl + 4 * FIN_LR; j < np; j += 4) Mx = fmaxf(Mx, ldl(j)); + Mx = fmaxf(Mx, __shfl_xor(Mx, 16)); Mx = fmaxf(Mx, __shfl_xor(Mx, 32)); + const bool dead = Mx == -__builtin_inff(); + float S = 0.f; +#pragma unroll + for (int i = 0; i < FIN_LR; ++i) { + if (jl + 4 * i < np) { + const float wt = dead ? 0.f : __builtin_amdgcn_exp2f(lz[i] - Mx); + s_wt[w][jl + 4 * i][hl] = wt; + S += wt; + } + } + for (int j = jl + 4 * FIN_LR; j < np; j += 4) { + const float wt = dead ? 0.f : __builtin_amdgcn_exp2f(ldl(j) - Mx); + s_wt[w][j][hl] = wt; + S += wt; + } + S += __shfl_xor(S, 16); S += __shfl_xor(S, 32); + if (final) S += dead ? 0.f : __builtin_amdgcn_exp2f(sk - Mx); + const float invS = S > 0.f ? 1.f / S : 0.f; + if (jl == 0) { + if (final) { + if (lse_out) lse_out[(size_t)r * NH + hw0 + hl] = (__builtin_amdgcn_logf(S) + Mx) * LN2; + } else { // group partial: log2-domain lse without sink (dead -> -inf); read later only by the row merge + const float lg = dead ? -__builtin_inff() : Mx + __builtin_amdgcn_logf(S); + __builtin_amdgcn_raw_buffer_store_b32(__builtin_bit_cast(unsigned, lg), rl, (s_slot[w][0] * NH + hw0 + hl) * 4, 0, + CPOL_SC1); + } + } + __builtin_amdgcn_s_waitcnt(0xc07f); + __builtin_amdgcn_wave_barrier(); + const int dslot = s_slot[w][0]; + // ---- accumulate: lane l owns dims 4l..4l+3 and 256+4l..+3 ---- + auto consume = [&](v4f (&acc)[HB][2], v4f (&bf)[HB][2], int j, int g) { +#pragma unroll + for (int k = 0; k < HB; ++k) { + const float wz = s_wt[w][j][g * HB + k]; + acc[k][0] += wz * bf[k][0]; acc[k][1] += wz * bf[k][1]; + } + }; +#pragma unroll + for (int g = 0; g < 2; ++g) { + v4f acc[HB][2]; +#pragma unroll + for (int k = 0; k < HB; ++k) acc[k][0] = acc[k][1] = (v4f){0, 0, 0, 0}; +#pragma unroll 1 + for (int j = 0; j + 1 < np; j += 2) { + issue(g * npp + j + 1, buf1); FIN_TIE16(buf0, 16); consume(acc, buf0, j, g); + issue(g * npp + j + 2, buf0); FIN_TIE16(buf1, 16); consume(acc, buf1, j + 1, g); + } + if (np & 1) { // odd np: batch np-1 is in buf0; the next group's first batch goes to buf1, moved to buf0 after it landed + issue(g * npp + npp, buf1); FIN_TIE16(buf0, 16); consume(acc, buf0, np - 1, g); + FIN_TIE16(buf1, 0); +#pragma unroll + for (int k = 0; k < HB; ++k) { buf0[k][0] = buf1[k][0]; buf0[k][1] = buf1[k][1]; } + } +#pragma unroll + for (int k = 0; k < HB; ++k) { + const float inv = __builtin_bit_cast(float, __builtin_amdgcn_readlane(__builtin_bit_cast(int, invS), g * HB + k)); + if (final) { + __bf16* dst = out + ((size_t)r * NH + hw0 + g * HB + k) * DV + 4 * l; + *(v4bf*)dst = __builtin_convertvector(acc[k][0] * inv, v4bf); + *(v4bf*)(dst + 256) = __builtin_convertvector(acc[k][1] * inv, v4bf); + } else { // in place into the group's first slot (its own reads of these addresses are complete: batch order) + const int off = ((dslot * NH + hw0 + g * HB + k) * DV + 4 * l) * 4; + __builtin_amdgcn_raw_buffer_store_b128(__builtin_bit_cast(v4u, acc[k][0] * inv), ro, off, 0, CPOL_SC1); + __builtin_amdgcn_raw_buffer_store_b128(__builtin_bit_cast(v4u, acc[k][1] * inv), ro, off + 1024, 0, CPOL_SC1); + } + } + } + // the last (clamped, unused) prefetch: all 16 destination registers stay allocated until it has landed + FIN_TIE16(buf0, 0); + }; + + if (!tree) { + merge(slots_pieces(pf, pl, (dec >> 1) & 1u), true); + } else { + // level 1: this group's pieces -> fp32 partial in the slot of the group's first piece (a 1-piece group is already there) + const int n1 = slots_pieces(q0, q1, (dec >> 1) & 1u); + if (n1 > 1) merge(n1, false); + // level 2: the group's tiles (+ the first-piece parity for group 0) go to the row counter + const unsigned par2 = gi == 0 ? (dec & 2u) << 29 : 0u; + const unsigned dec2 = arrive(cnt + 2 * r + hg, (unsigned)target1 | par2, T); + if (!(dec2 & 1u)) return; + // group partial slots: first piece of each group (lanes over groups) + const int ng = (nq + FIN_G - 1) / FIN_G; + const unsigned pr = (dec2 >> 1) & 1u; + int n2 = 0; + for (int g0 = 0; g0 < ng; g0 += 64) { + const int g = g0 + l; + int sl = -1; + if (g < ng) { + const int qa = pf + g * FIN_G, qb = min(pl, qa + FIN_G - 1); + for (int q = qa; q <= qb; ++q) + if (t0of(q) < t1of(q)) { sl = 2 * q + (q == pf ? (int)pr : 0); break; } + } + const unsigned long long m = __ballot(sl >= 0); + if (sl >= 0) { + const int k = n2 + __popcll(m & ((1ull << l) - 1)); + if (k < FIN_MAXJ) s_slot[w][k] = sl; + } + n2 += __popcll(m); + } + __builtin_amdgcn_s_waitcnt(0xc07f); + __builtin_amdgcn_wave_barrier(); + merge(__builtin_amdgcn_readfirstlane(min(n2, FIN_MAXJ)), true); + } +} diff --git a/hsa/gfx950/mla_v4/nm_ps_src/splice/hca_common.hip b/hsa/gfx950/mla_v4/nm_ps_src/splice/hca_common.hip new file mode 100644 index 00000000000..93ef7662531 --- /dev/null +++ b/hsa/gfx950/mla_v4/nm_ps_src/splice/hca_common.hip @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: MIT +// Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. +// +// Common definitions of the splice sources (pre.hip, fin.hip): the closed-form partition planner's cost line, +// the per-partition descriptor, the wave-cooperative 64-way boundary search and the WG -> (partition, head group) +// map. Only compiled as part of pre.hip / fin.hip (hipcc -S); nothing here is a kernel. +#include +#include + +typedef int v4i __attribute__((ext_vector_type(4))); +typedef float v4f __attribute__((ext_vector_type(4))); +typedef __bf16 v4bf __attribute__((ext_vector_type(4))); + +#define TILE 32 +#define NH 128 // heads +#define DV 512 +#define LOG2E 1.4426950408889634f +#define LN2 0.6931471805599453f + +// ---------------- closed-form partition planner: cost line ---------------- +// Planner config word Fcfg = F | (MT << 8): +// C0 = 32*F + 16 (fixed cost per row piece, token units, charged once per row on the cost line); +// MT = minimum partition size in tiles beyond one C0 (0 = off): the number of ACTIVE partitions is +// Pa = clamp(total / (C0 + 32*MT), 1, P), per = ceil(total / Pa); partitions p >= Pa are empty. +// A cut inside a row adds at most one extra C0 per partition (the fixed cost is the same for every +// partition, so balance is unaffected); Pa bounds how many pieces tiny shapes are cut into. +__device__ __forceinline__ int cfg_C0(int Fcfg) { return 32 * (Fcfg & 0xff) + 16; } +__device__ __forceinline__ int active_parts(int total, int P, int Fcfg) { + const int MT = Fcfg >> 8; + if (MT <= 0) return P; + return max(1, min(P, total / (cfg_C0(Fcfg) + 32 * MT))); +} + +struct __attribute__((aligned(32))) Desc { + // comb = 1 iff begin_row is split and its LAST piece is in this partition (begin_tile > 0 and the row ends here): + // this partition's combine WG merges that row (only used by an external combine; the fused combine clears it). + int begin_row, end_row, begin_tile, end_tile, flags, kv_begin_tok, comb, pad1; +}; + +// ---------------- wave-cooperative planner search ---------------- +// The two partition-boundary binary searches of partition p (positions p*per and (p+1)*per on the cost +// line), 64-way: each round every lane samples one row of the current interval, a ballot counts the samples +// with base() <= pos, and the interval shrinks 64x. N <= 64: 1 round, N <= 4096: 2 rounds (+ the +// ind[N], ind[N-1] loads, issued in parallel with round 0). Each lane loads ind[j-1..j+2] so the final +// step-1 round also yields every neighbour value the descriptor needs (no extra dependent round). +// Must be called by all 64 lanes of a wave with uniform arguments. +struct WRes { int r, im1, i0, i1, i2; }; // row, ind[r-1], ind[r], ind[r+1], ind[r+2] (clamped) + +__device__ __forceinline__ int rdl(int v, int lane) { return __builtin_amdgcn_readlane(v, lane); } + +__device__ __forceinline__ void wsearch2(const int* __restrict__ ind, int N, int Fcfg, int tN, int org, int P, int p, + int& total, int& per, int& posA, int& posB, WRes& A, WRes& B) { + const int C0 = cfg_C0(Fcfg); // positions are relative to org = ind[0] (tN is ind[N], absolute) + const int lane = threadIdx.x & 63; + int loA = 0, lenA = N, stA = (N + 63) >> 6; + int loB = 0, lenB = N, stB = stA; + bool doneA = false, doneB = false, first = true; + total = 0; per = 1; posA = posB = 0; + while (!(doneA && doneB)) { + const bool vA = !doneA && lane * stA < lenA, vB = !doneB && lane * stB < lenB; + const int jA = vA ? loA + lane * stA : loA, jB = vB ? loB + lane * stB : loB; + const int a_m1 = ind[max(jA - 1, 0)], a0 = ind[jA], a1 = ind[jA + 1], a2 = ind[min(jA + 2, N)]; + const int b_m1 = ind[max(jB - 1, 0)], b0 = ind[jB], b1 = ind[jB + 1], b2 = ind[min(jB + 2, N)]; + if (first) { + total = tN - org + N * C0; + const int Pa = active_parts(total, P, Fcfg); + per = max(1, (total + Pa - 1) / Pa); + posA = min(p, Pa) * per; // p >= Pa: posA = Pa*per >= total -> empty (no int overflow) + posB = (p >= Pa - 1) ? posA : (p + 1) * per; + first = false; + } + const int cA = __popcll(__ballot(vA && a0 - org + jA * C0 <= posA)); + const int cB = __popcll(__ballot(vB && b0 - org + jB * C0 <= posB)); + if (!doneA) { + if (stA == 1) { + A.r = loA + cA - 1; A.im1 = rdl(a_m1, cA - 1); A.i0 = rdl(a0, cA - 1); A.i1 = rdl(a1, cA - 1); A.i2 = rdl(a2, cA - 1); + doneA = true; + } else { + loA += (cA - 1) * stA; lenA = min(stA, lenA - (cA - 1) * stA); stA = (lenA + 63) >> 6; + } + } + if (!doneB) { + if (stB == 1) { + B.r = loB + cB - 1; B.im1 = rdl(b_m1, cB - 1); B.i0 = rdl(b0, cB - 1); B.i1 = rdl(b1, cB - 1); B.i2 = rdl(b2, cB - 1); + doneB = true; + } else { + loB += (cB - 1) * stB; lenB = min(stB, lenB - (cB - 1) * stB); stB = (lenB + 63) >> 6; + } + } + } +} + +__device__ __forceinline__ void wg_map(int wg, int P, int& p, int& hg) { + if ((P & 7) == 0) { // keep the hg pair and neighbouring partitions on one XCD (round-robin dispatch, inferred) + int xcd = wg & 7, loc = wg >> 3; + p = xcd * (P >> 3) + (loc >> 1); hg = loc & 1; + return; + } + p = wg >> 1; hg = wg & 1; +} diff --git a/hsa/gfx950/mla_v4/nm_ps_src/splice/pre.hip b/hsa/gfx950/mla_v4/nm_ps_src/splice/pre.hip new file mode 100644 index 00000000000..05695bcca49 --- /dev/null +++ b/hsa/gfx950/mla_v4/nm_ps_src/splice/pre.hip @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: MIT +// Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. +// +// Splice source: the in-kernel split planner, compiled for gfx950 and spliced into the prologue of the patched ASM by +// asm/patch.py. This file is only compiled to assembly (hipcc -S); the kernel is never launched. +// ABI assumed by the splice (checked by splice.py): kernarg pointer in s[0:1], workgroup id x in s2, thread id in v0. +// Outputs (uniform) in AGPRs: a243 begin_row, a244 end_row, a245 begin_tile, a246 end_tile, a247 flags, +// a248 per, a249 Pa (active partitions), a250 org = kv_indptr[0] -- the planner's cost-line constants, used by the +// fused combine (fin.hip). +// The descriptor is also published to desc[p] (hg == 0 WG, thread 0). +#include "hca_common.hip" +#define FIN_NPF 100000 // split rows spanning <= FIN_NPF partitions are merged by the fused combine (fin.hip): all rows + +struct KA { char b[336]; }; +extern "C" __device__ int __ockl_wfred_max_i32(int); +// LDS-only barrier: __syncthreads() also waits for every outstanding global load (fence), which would make the first +// barrier wait for a kv_indptr round trip +#define NS_BAR() do { __builtin_amdgcn_s_waitcnt(0xc07f); __builtin_amdgcn_s_barrier(); } while (0) +// Split / no-split choice: no-split iff Minf - (per + C0) < NS_WANY (token units; 384 = 12 tiles; an empirically +// chosen threshold). Minf = max partition load of the no-split plan, exact, per row in closed form: row r (K > 0) +// goes to partition q(r) = min(Pa-1, m_r / per), m_r = b + C0 + 32*(ceil(T/2)-1), load += C0 + 32T. +// WG-cooperative (all 256 threads; every WG computes the same result): thread t walks rows [t*c, t*c+c), c = ceil(N/256), +// adding each row's load into an LDS per-partition array (atomics); the max by an LDS atomic. +// Code size matters (straight-line instructions executed at kernel start cost instruction fetch): no unrolling, one +// division per row (float reciprocal estimate + exact correction). +#define NS_WANY 384 +#define NS_MAXP 1024 +// Split descriptor (closed-form planner, cut at p*per / (p+1)*per) + the no-split descriptor (every cut snapped to the +// nearer end of the row it falls in: tile i -> 0 if 2i < T, else T) from the same wsearch2 (one search) +__device__ __forceinline__ void plan_both(const int* __restrict__ ind, int N, int P, int Fcfg, int p, Desc& ds, Desc& dn, + int& total, int& per) { + const int C0 = cfg_C0(Fcfg); + const int tN = ind[N], tNm1 = ind[max(N - 1, 0)], org = ind[0]; + int posA, posB; + WRes A, B; + wsearch2(ind, N, Fcfg, tN, org, P, p, total, per, posA, posB, A, B); + const int Pa = active_parts(total, P, Fcfg); + auto Tof = [](int a, int b) { return (b - a + TILE - 1) >> 5; }; + auto ftg = [&](int r, int ir, int T_, int pos) { + int d = pos - (ir - org + r * C0) - C0; + int i = d <= 0 ? 0 : (d + TILE - 1) >> 5; + return min(i, T_); + }; + // ---- split ---- + int br, bt, Tb; + const int T0 = Tof(A.i0, A.i1); + const int iA = posA >= total ? 0 : ftg(A.r, A.i0, T0, posA); + if (posA >= total) { br = N; bt = 0; Tb = 0; } + else if (iA < T0) { br = A.r; bt = iA; Tb = T0; } + else { br = A.r + 1; bt = 0; Tb = (br < N) ? Tof(A.i1, A.i2) : 0; } + int er, et, Tm1, T0e; + Tm1 = Tof(B.im1, B.i0); T0e = Tof(B.i0, B.i1); + const int TN1 = Tof(tNm1, tN); + int Te_minus = 0; + const bool endN = p >= Pa - 1 || posB >= total; + const int iB = endN ? 0 : ftg(B.r, B.i0, T0e, posB); + if (endN) { er = N; et = 0; Te_minus = TN1; } + else if (iB < T0e) { er = B.r; et = iB; Te_minus = Tm1; } + else { er = B.r + 1; et = 0; Te_minus = T0e; } + ds.comb = ds.pad1 = 0; + if (!((br < er) || (br == er && bt < et))) { + ds.begin_row = N; ds.end_row = -1; ds.begin_tile = ds.end_tile = ds.flags = ds.kv_begin_tok = 0; + } else { + int Te; + if (et == 0) { er -= 1; et = Te_minus; Te = Te_minus; } + else Te = (er == B.r) ? T0e : 0; + const int f0 = (bt > 0) || (er == br && et < Tb); + const int f1 = (et < Te) || (er == br && bt > 0); + ds.begin_row = br; ds.end_row = er; ds.begin_tile = bt; ds.end_tile = et; + ds.flags = f0 | (f1 << 1); ds.kv_begin_tok = (br == A.r) ? A.i0 : A.i1; + ds.comb = bt > 0 && (er > br || et == Tb); + } + // ---- no-split ---- + const int nb = posA >= total ? N : ((2 * iA < T0 || T0 == 0) ? A.r : A.r + 1); + const int ne = endN ? N : ((2 * iB < T0e || T0e == 0) ? B.r : B.r + 1); + dn.comb = dn.pad1 = 0; dn.begin_tile = 0; dn.flags = 0; + if (nb >= ne) { + dn.begin_row = N; dn.end_row = -1; dn.end_tile = dn.kv_begin_tok = 0; + } else { + dn.begin_row = nb; dn.end_row = ne - 1; + dn.end_tile = ne == N ? TN1 : (ne == B.r ? Tm1 : T0e); + dn.kv_begin_tok = nb == A.r ? A.i0 : A.i1; + } +} +// kernarg reads through the constant-address-space kernarg pointer -> s_load (the by-value struct compiled to vector loads) +#define KARG(T, off) (*(T const __attribute__((address_space(4)))*)((const __attribute__((address_space(4))) char*)__builtin_amdgcn_kernarg_segment_ptr() + (off))) + +extern "C" __global__ void __launch_bounds__(256) t8pre(KA ka) { + const int* ind = KARG(const int*, 0x40); + Desc* desc = KARG(Desc*, 0x60); + const int P = KARG(int, 0xa0), N = KARG(int, 0xa4), F = KARG(int, 0xb0); + int p, hg; + wg_map(blockIdx.x, P, p, hg); + int total, per; + // Wave 0 runs the planner search (plan_both: split and no-split descriptors from one search) while waves + // 1..3 compute the no-split load profile (ns rows, LDS atomics); the descriptors travel through LDS, so the decision + // overlaps the search. Every wave executes the same 4 barriers. + Desc d0; + if (N > 0) { + __shared__ int s_load[NS_MAXP]; + __shared__ int s_max; + __shared__ Desc s_dd[2]; + __shared__ int s_tp[3]; // total, per, Pa (wave 0's search) + const int C0 = cfg_C0(F), tid = threadIdx.x, wv = tid >> 6; + // LDS init before any memory dependence (the first barrier must not wait for a kv_indptr round trip) +#pragma unroll + for (int i = 0; i < NS_MAXP / 256; ++i) s_load[tid + 256 * i] = 0; + if (tid == 0) s_max = 0; + int r0 = 0, r1 = 0; + if (wv) { + const int cs = (N + 191) / 192; + r0 = min(N, (tid - 64) * cs); r1 = min(N, r0 + cs); + } + NS_BAR(); + if (wv == 0) { + Desc ds, dn; + int tt, pp; + plan_both(ind, N, P, F, p, ds, dn, tt, pp); + if (tid == 0) { s_dd[0] = ds; s_dd[1] = dn; s_tp[0] = tt; s_tp[1] = pp; s_tp[2] = active_parts(tt, P, F); } + } else { + int i0 = ind[r0], i1 = ind[min(r0 + 1, N)]; + const int org = ind[0], tot0 = ind[N] - org + N * C0; // == wsearch2's cost-line constants + const int Pa0 = active_parts(tot0, P, F), per0 = max(1, (tot0 + Pa0 - 1) / Pa0); + const float rf = __builtin_amdgcn_rcpf((float)per0); +#pragma unroll 1 + for (int r = r0; r < r1; ++r) { + if (r != r0) i1 = ind[r + 1]; + const int T = (i1 - i0 + TILE - 1) >> 5; + if (T > 0) { + const int m = i0 - org + r * C0 + C0 + TILE * (((T + 1) >> 1) - 1); + // q <= total/per <= Pa <= 1024: the float estimate has |error| < 1 (relative error ~2^-22), one correction + int q = (int)((float)m * rf); + const int rm = m - q * per0; + q += rm < 0 ? -1 : (rm >= per0 ? 1 : 0); + atomicAdd(&s_load[min(Pa0 - 1, q)], C0 + TILE * T); + } + i0 = i1; + } + } + NS_BAR(); + const int tot0 = s_tp[0], per0 = s_tp[1], Pa0 = s_tp[2]; + int mx = 0; + for (int i = tid; i < Pa0; i += 256) mx = max(mx, s_load[i]); + // wave max first: 256 lanes atomicMax-ing one LDS word would serialise + mx = __ockl_wfred_max_i32(mx); // DPP wave reduction + if ((tid & 63) == 0 && mx) atomicMax(&s_max, mx); + NS_BAR(); + d0 = s_dd[(s_max - per0 - C0 < NS_WANY) ? 1 : 0]; + NS_BAR(); // every wave has read LDS before any wave enters the ASM (which reuses LDS) + total = tot0; per = per0; + } else { // N = 0: all partitions empty (without instantiating a second search) + d0.begin_row = N; d0.end_row = -1; d0.begin_tile = d0.end_tile = d0.flags = d0.kv_begin_tok = d0.comb = d0.pad1 = 0; + total = 0; per = 1; + } + const int Pa = active_parts(total, P, F); + Desc d = d0; + // fused combine (fin.hip): a split row spanning <= FIN_NPF partitions (every row) is merged in the attention kernel + // by its last-arriving piece; desc.comb is cleared for it. Same closed form as fin.hip. + if (d.comb) { + const int br = d.begin_row, org = ind[0], C0 = cfg_C0(F); + const int b = ind[br] - org + br * C0, T = (ind[br + 1] - ind[br] + TILE - 1) >> 5; + const int pf = min(Pa - 1, (b + C0) / per), pl = min(Pa - 1, (b + C0 + TILE * (T - 1)) / per); + if (pl - pf + 1 <= FIN_NPF) d.comb = 0; + } + if (hg == 0 && threadIdx.x == 0) desc[p] = d; + asm volatile("v_accvgpr_write_b32 a243, %0\n\tv_accvgpr_write_b32 a244, %1\n\tv_accvgpr_write_b32 a245, %2\n\t" + "v_accvgpr_write_b32 a246, %3\n\tv_accvgpr_write_b32 a247, %4\n\tv_accvgpr_write_b32 a248, %5\n\t" + "v_accvgpr_write_b32 a249, %6\n\tv_accvgpr_write_b32 a250, %7" + :: "v"(d.begin_row), "v"(d.end_row), "v"(d.begin_tile), "v"(d.end_tile), "v"(d.flags), "v"(per), "v"(Pa), + "v"(N > 0 ? ind[0] : 0) + : "memory"); +} diff --git a/hsa/gfx950/mla_v4/nm_ps_src/splice/splice.py b/hsa/gfx950/mla_v4/nm_ps_src/splice/splice.py new file mode 100644 index 00000000000..86048e5a09c --- /dev/null +++ b/hsa/gfx950/mla_v4/nm_ps_src/splice/splice.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""Compile a HIP splice source (splice/*.hip) to gfx950 assembly with `hipcc -S` and extract the body of one kernel +as a text block that asm/patch.py inlines into the patched ASM. + +Extraction rules (checked here): + - the kernel ABI must be: user SGPRs = kernarg pointer only (s[0:1]), workgroup id x only (s2), workitem id x (v0), + no kernarg preload, no scratch, no LDS (group segment 0) unless allow_lds; + - register budget: next_free_vgpr <= vmax, next_free_sgpr <= smax (the wrapper keeps its state above these); + - labels .LBBk_n -> _n; every s_endpgm -> s_branch _RET (the caller places _RET after the block); + - assembler directives and comments are dropped. +""" + +import os +import re +import subprocess + +HERE = os.path.dirname(os.path.abspath(__file__)) +HIP_FLAGS = [ + "--offload-arch=gfx950", + "-O3", + "--offload-device-only", + "-std=c++20", + "-w", +] + + +def compile_s(hipcc, src, out, extra=()): + subprocess.run( + [hipcc, *HIP_FLAGS, *extra, "-S", src, "-o", out], + check=True, + cwd=os.path.dirname(src), + ) + return out + + +def body( + hipcc, src, kernel, prefix, workdir, vmax=240, smax=100, allow_lds=False, extra=() +): + """-> (block text, kernel descriptor fields {name: int})""" + out_s = os.path.join(workdir, os.path.basename(src)[:-4] + ".s") + s = open(compile_s(hipcc, src, out_s, extra)).read().split("\n") + i0 = next(i for i, ln in enumerate(s) if ln.split(";")[0].strip() == f"{kernel}:") + i1 = next(i for i in range(i0, len(s)) if s[i].startswith(".Lfunc_end")) + # kernel descriptor block + k0 = next( + i for i in range(i0, len(s)) if s[i].split() == [".amdhsa_kernel", kernel] + ) + k1 = next(i for i in range(k0, len(s)) if s[i].strip() == ".end_amdhsa_kernel") + kd = {} + for ln in s[k0 + 1 : k1]: + t = ln.split() + if len(t) == 2 and t[0].startswith(".amdhsa_"): + kd[t[0][8:]] = int(t[1]) + req = dict( + user_sgpr_count=2, + user_sgpr_kernarg_segment_ptr=1, + user_sgpr_dispatch_ptr=0, + user_sgpr_queue_ptr=0, + user_sgpr_kernarg_preload_length=0, + system_sgpr_workgroup_id_y=0, + system_sgpr_workgroup_id_z=0, + system_vgpr_workitem_id=0, + private_segment_fixed_size=0, + enable_private_segment=0, + ) + for k, v in req.items(): + assert kd.get(k, v) == v, (kernel, k, kd.get(k)) + if not allow_lds: + assert kd["group_segment_fixed_size"] == 0, kd["group_segment_fixed_size"] + assert kd["next_free_vgpr"] <= vmax and kd["next_free_sgpr"] <= smax, ( + kernel, + kd["next_free_vgpr"], + kd["next_free_sgpr"], + ) + out = [] + for ln in s[i0 + 1 : min(i1, k0)]: + t = ln.split(";")[0].rstrip() + if not t.strip() or t.strip().startswith("."): + m = re.match(r"^\.LBB\d+_(\d+):", t) + if m: + out.append(f"{prefix}_{m.group(1)}:") + continue + t = re.sub(r"\.LBB\d+_(\d+)", lambda m: f"{prefix}_{m.group(1)}", t) + if t.strip() == "s_endpgm": + t = f"\ts_branch {prefix}_RET" + out.append(t) + # every register named must stay inside the budget (inline-asm AGPR outputs are allowed) + for ln in out: + for m in re.finditer(r"\bv\[?(\d+)(?::(\d+))?", ln): + assert int(m.group(2) or m.group(1)) < vmax, ln + for m in re.finditer(r"\bs\[?(\d+)(?::(\d+))?", ln): + if re.match(r"\s*s_", ln) or "," in ln: + assert int(m.group(2) or m.group(1)) < smax, ln + return "\n".join(out), kd From c7cf2b0f245f1e355451809aa26126b4dd0800ca Mon Sep 17 00:00:00 2001 From: yihonglie Date: Tue, 29 Sep 2026 11:45:58 -0500 Subject: [PATCH 06/13] [MLA v4 nm] Drop the unused span buffer from the persistent decode API The persistent code object never reads kernarg slot 0xb8, yet the span workspace [P, 2] was a field of MlaV4NmPsWorkspace, a required argument of mla_decode_v4_ps_asm and size-checked in C++. Remove it from the workspace, the ctypes signature and the C++ entry point, and pass nullptr at 0xb8. Document the three regions of the cnt counter buffer next to its size. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie --- aiter/mla.py | 4 ++-- aiter/ops/attention.py | 2 -- csrc/py_itfs_cu/asm_mla_v4.cu | 12 +++++------- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/aiter/mla.py b/aiter/mla.py index 63277fcec17..c2742c30c69 100644 --- a/aiter/mla.py +++ b/aiter/mla.py @@ -455,6 +455,8 @@ def get_mla_v4_nm_split_plan( # Workspace of the persistent v4 nm decode kernel (mla_decode_v4_ps_asm). +# cnt: [0, 2*65536) row counters, [2*65536, +16*512) reserved, +# [.., +4*1024) group counters. _V4_NM_PS_CNT_INTS = 2 * 65536 + 16 * 512 + 4 * 1024 _V4_NM_PS_ARANGE = 65537 _V4_NM_PS_MAX_PARTITIONS = 1024 @@ -467,7 +469,6 @@ class MlaV4NmPsWorkspace(NamedTuple): o_acc: torch.Tensor # [2P, 128, 512] fp32 split partials lse_acc: torch.Tensor # [2P, 128] fp32 split partials desc: torch.Tensor # [P, 8] int32 in-kernel plan - span: torch.Tensor # [P, 2] int32 in-kernel plan cnt: torch.Tensor # int32 merge counters, zero at rest arange: torch.Tensor # int32 arange, read-only @@ -497,7 +498,6 @@ def get_mla_v4_nm_ps_workspace(device="cuda", num_partitions=128) -> MlaV4NmPsWo o_acc=torch.empty(2 * P, 128, 512, dtype=dtypes.fp32, device=device), lse_acc=torch.empty(2 * P, 128, dtype=dtypes.fp32, device=device), desc=torch.empty(P, 8, **i32), - span=torch.zeros(P, 2, **i32), cnt=torch.zeros(_V4_NM_PS_CNT_INTS, **i32), arange=torch.arange(_V4_NM_PS_ARANGE, **i32), ) diff --git a/aiter/ops/attention.py b/aiter/ops/attention.py index 880168df0c9..b0adf2baf9d 100644 --- a/aiter/ops/attention.py +++ b/aiter/ops/attention.py @@ -907,8 +907,6 @@ def mla_decode_v4_ps_asm( lse_acc: torch.Tensor, # [P, 8] int32 desc: torch.Tensor, - # [P, 2] int32 - span: torch.Tensor, # int32 counters, zero at rest cnt: torch.Tensor, # int32 arange diff --git a/csrc/py_itfs_cu/asm_mla_v4.cu b/csrc/py_itfs_cu/asm_mla_v4.cu index 76ed4eb0703..a716f6e3d5c 100644 --- a/csrc/py_itfs_cu/asm_mla_v4.cu +++ b/csrc/py_itfs_cu/asm_mla_v4.cu @@ -587,7 +587,7 @@ struct __attribute__((packed)) MlaV4PsKernelArgs p2 _p_nq; // 0xa8: pad unsigned int s_plan; // 0xb0: repurposed: planner config F | (MT << 8) unsigned int _p_pl; // 0xb4: pad - void* ptr_span; // 0xb8: workspace span [P, 2] int32 (rw) + void* ptr_unused; // 0xb8: not read by this kernel, nullptr unsigned int s_zero; p3 _p_z; // 0xc0: 0 void* ptr_QTP; @@ -610,12 +610,14 @@ struct __attribute__((packed)) MlaV4PsKernelArgs static_assert(sizeof(MlaV4PsKernelArgs) == 21 * 16, "persistent v4 nm kernarg is 0x150 bytes"); static_assert(offsetof(MlaV4PsKernelArgs, ptr_cnt) == 0x98 && offsetof(MlaV4PsKernelArgs, s_num_q) == 0xa4 && - offsetof(MlaV4PsKernelArgs, ptr_span) == 0xb8 && + offsetof(MlaV4PsKernelArgs, ptr_unused) == 0xb8 && offsetof(MlaV4PsKernelArgs, ptr_sink) == 0x120, "persistent v4 nm kernarg offsets"); // Workspace sizes the kernel indexes into (aiter/mla.py // get_mla_v4_nm_ps_workspace). +// cnt holds [0, 2*65536) row counters, [2*65536, +16*512) reserved (unused) +// and [.., +4*1024) group counters. static constexpr int kV4PsHeads = 128; static constexpr int kV4PsDim = kV4DimNope + kV4DimRope; static constexpr int kV4PsMaxParts = 1024; @@ -636,7 +638,6 @@ AITER_CTYPES_DEFINE_ENTRYPOINT_VOID( aiter_tensor_t* o_acc, // workspace [2P, 128, 512] FP32 aiter_tensor_t* lse_acc, // workspace [2P, 128] FP32 aiter_tensor_t* desc, // workspace [P, 8] int32 - aiter_tensor_t* span, // workspace [P, 2] int32 aiter_tensor_t* cnt, // workspace [kV4PsCntInts] int32, zero at rest aiter_tensor_t* arange, // workspace arange(kV4PsArange) int32 aiter_tensor_t* output, // [N, 128, 512] BF16 @@ -652,7 +653,6 @@ AITER_CTYPES_DEFINE_ENTRYPOINT_VOID( o_acc, lse_acc, desc, - span, cnt, arange, output, @@ -692,7 +692,6 @@ AITER_CTYPES_DEFINE_ENTRYPOINT_VOID( check_buf(o_acc, AITER_DTYPE_fp32, "o_acc"); check_buf(lse_acc, AITER_DTYPE_fp32, "lse_acc"); check_buf(desc, AITER_DTYPE_i32, "desc"); - check_buf(span, AITER_DTYPE_i32, "span"); check_buf(cnt, AITER_DTYPE_i32, "cnt"); check_buf(arange, AITER_DTYPE_i32, "arange"); check_buf(output, AITER_DTYPE_bf16, "output"); @@ -740,7 +739,6 @@ AITER_CTYPES_DEFINE_ENTRYPOINT_VOID( AITER_CHECK(lse_acc->numel() == static_cast(2 * num_p * kV4PsHeads), __func__, ": lse_acc must be [2P, 128]"); - AITER_CHECK(span->numel() == static_cast(2 * num_p), __func__, ": span must be [P, 2]"); AITER_CHECK(cnt->numel() >= static_cast(kV4PsCntInts), __func__, ": cnt needs ", @@ -774,7 +772,7 @@ AITER_CTYPES_DEFINE_ENTRYPOINT_VOID( args.s_num_p = static_cast(num_p); args.s_num_q = static_cast(num_q); args.s_plan = kV4PsPlanCfg; - args.ptr_span = span->data_ptr(); + args.ptr_unused = nullptr; args.ptr_QTP = arange->data_ptr(); args.ptr_STP = arange->data_ptr(); args.out_16_nosplit = 1; From fe8dd3813d75b096cae0d1d58f0136ace3b2a28b Mon Sep 17 00:00:00 2001 From: yihonglie Date: Tue, 29 Sep 2026 11:45:58 -0500 Subject: [PATCH 07/13] [MLA v4 nm] Add a guard-page out-of-bounds test for the persistent kernel Run mla_decode_fwd_v4_nm_ps in a subprocess at large N (up to the 32768 row limit) with every input and output placed at the very end of its own 2 MiB-rounded allocation under the non-caching allocator, so an access one element past q, qrope, the KV pools, kv_indptr (exactly N+1 entries), kv_page_indices, sink, out or lse faults the GPU. Covers the lse and no-lse paths and checks the merge counters return to zero. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie --- op_tests/test_mla_v4_nm.py | 61 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/op_tests/test_mla_v4_nm.py b/op_tests/test_mla_v4_nm.py index cc0e819ae48..7498083c645 100644 --- a/op_tests/test_mla_v4_nm.py +++ b/op_tests/test_mla_v4_nm.py @@ -2619,6 +2619,7 @@ def test_v4_nm_kv_tail_not_tile_multiple(): # Persistent kernel: mla_decode_fwd_v4_nm_ps (gqa=128, qlen=1, one launch). # --------------------------------------------------------------------------- _PS_HEADS = 128 +_PS_MAX_ROWS = 32768 # mla_decode_fwd_v4_nm_ps row limit _PS_SENTINEL = -3.0 # rows with K=0 must keep it (the kernel leaves them unwritten) @@ -2922,6 +2923,62 @@ def test_v4_nm_ps_rejects_bad_args(ps_workspace): aiter.mla.get_mla_v4_nm_ps_workspace(num_partitions=1025) +def _ps_oob_worker_main(): + """Subprocess body of test_v4_nm_ps_oob_guardpage. Every buffer the + persistent kernel reads or writes (q, qrope, KV pools, kv_indptr with + exactly N+1 entries, kv_page_indices, sink, out, lse) is placed at the + very end of its own allocation, rounded to 2 MiB, under the non-caching + allocator, so an access even one element past any of them crosses into an + unmapped page and faults the GPU. (A one-entry over-read of + kv_page_indices faults with this layout; a plain clone of the same tensor + leaves up to 2 MiB of slack and does not.)""" + granule = 2 << 20 + + def tail(t): + nbytes = t.numel() * t.element_size() + total = -(-nbytes // granule) * granule + buf = torch.empty(total, dtype=torch.uint8, device=t.device) + v = buf[total - nbytes :].view(t.dtype).view(t.shape) + v.copy_(t) + return v + + for n, hi, zero_every, return_lse in ( + (8192, 700, 13, True), + (_PS_MAX_ROWS, 64, 0, True), # largest N: out ends at the 4 GiB offset limit + (3000, 4000, 0, False), + ): + inp = _ps_inputs(_ps_ragged(n, hi, n, zero_every=zero_every), seed=n) + inp = {k: tail(v) for k, v in inp.items()} + out = tail( + torch.empty(n, _PS_HEADS, V_HEAD_DIM, dtype=dtypes.bf16, device="cuda") + ) + lse = tail(torch.empty(n, _PS_HEADS, device="cuda")) if return_lse else None + ws = aiter.mla.get_mla_v4_nm_ps_workspace() + _ps_call(inp, ws, out=out, lse=lse, return_lse=return_lse) + torch.cuda.synchronize() + assert int(ws.cnt.abs().sum()) == 0 + del inp, out, lse, ws + + +@needs_gfx950 +def test_v4_nm_ps_oob_guardpage(): + """Guard-page out-of-bounds check for mla_decode_fwd_v4_nm_ps. + + The persistent kernel runs the head 64..127 path of the ASM (the one the + gqa=128 Q_rope over-read regression lived on) and adds its own reads: + the kv_indptr scans of the in-kernel planner, the split partials and the + merge. An over-read there can be numerically silent, so this runs large + batches in a subprocess with every input and output in a tight + allocation and asserts a clean exit, like + test_v4_nm_gqa128_qrope_oob_guardpage. + """ + _run_oob_guardpage_worker( + ["ps"], + "v4 nm ps OOB detected: the persistent kernel accessed memory past " + "one of its input / output buffers.", + ) + + if __name__ == "__main__": import argparse import itertools @@ -2940,6 +2997,10 @@ def test_v4_nm_ps_rejects_bad_args(ps_workspace): # non-caching allocator so any tg_idx=1 Q_rope over-read crosses into an # unmapped page and faults the GPU (killing THIS process). A clean exit 0 # means no OOB. See the test's docstring for the full rationale. + if sys.argv[1:3] == ["--oob-worker", "ps"]: + _ps_oob_worker_main() + print("[v4 nm][oob-worker] COMPLETED no fault") + sys.exit(0) if len(sys.argv) >= 2 and sys.argv[1] == "--oob-worker": _gqa = int(sys.argv[2]) if len(sys.argv) >= 3 else 128 _msq = int(sys.argv[3]) if len(sys.argv) >= 4 else 1 From 4363a56377d79cd6f55447f29c4f031e51e2a22c Mon Sep 17 00:00:00 2001 From: yihonglie Date: Tue, 29 Sep 2026 11:45:59 -0500 Subject: [PATCH 08/13] [MLA v4 nm] Rename research identifiers in the persistent kernel generator Rename the spliced planner / combine kernels and their labels (t8pre -> ps_plan, t8fin -> ps_merge, T8_* -> PS_*) and regenerate the code object. .text, .rodata (kernel descriptor) and .note (metadata) are byte-identical to the previous build; only local symbol names in .symtab change. Update the expected sha256. README fixes: the row limit is N <= 32768 (out / q byte offsets wrap above it; the cnt / arange sizes come from a 65536-row layout), cnt is int32 with its three regions spelled out, and slot 0xb8 is passed as 0. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie --- ...mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co | Bin 153984 -> 154424 bytes hsa/gfx950/mla_v4/nm_ps_src/README.md | 14 ++--- hsa/gfx950/mla_v4/nm_ps_src/asm/patch.py | 50 +++++++++--------- hsa/gfx950/mla_v4/nm_ps_src/splice/fin.hip | 2 +- hsa/gfx950/mla_v4/nm_ps_src/splice/pre.hip | 2 +- 5 files changed, 35 insertions(+), 33 deletions(-) diff --git a/hsa/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co b/hsa/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co index dd75375eb3e6879b78465021a9de52b7b8084630..f162a7fa95c21bddf96474cd1c5718be5e03dee5 100755 GIT binary patch delta 10324 zcmZ|UeQ;FO83yp1g|ZMM1RDh`>IxB82#~v9*;NL!nm~{cB4h+9nt-8YG$JT!EKF}S zDKa&KiA8UqsSQp>urd-bl9WbWaZsvO1EQuC5yli_P1~R;Vkd>(bI-GT&il@oKj7x~ zyze>Zp8K&O)Vw^ZbN{FYzo&D_qf>u1HSLwDp3{@kGVHOFJ>w=^W{5AZQ8dLpNd4in z<078sg4Z~AT`F-8BKA5F#}N;*xY*ty;w6X&{Iq+eeMH3fB5se+xY|z3cH&yZH9;D0 zu}u+gLtK5cjd}A^YlUJmV~H zn#jFI)(qNx3A~E=74TN(es~S@Ap98f+0yNfYdm8o%tO%-JB2d)l(WVO<%qZD(ly>~ zkDcPg4VW1~eq1 z)aSsPnBNNTVZIz*#KvC@?{@tnHOYhat$t^}EaZKxZ;Sn?h@U}RbG_;mN58Z8iueV@ z&F|8<-99Da*AbV!L*sYsvDZ2AM~FKMX?)U-i}-8Ahv(4vti8^Ot+Zi+y~f75G|qrG zF&_tSXFdrYDx%{|hp#TC9+hqxO)R(>4MPiPLkT>ig!=99te;Y^OnQ%dv!=qe@Rl+` z+NIoJH%@i-YZLOJDD7*s-w<&#;sQ3^PWz0A_aQ!R)9zR8tEV~fVZ^KL)Oy^55A9n; zd;;;rJ8Art{iuisliiQd_=3IHiLLbEg1ttzeCelZE`{$}pCb3b2M=weJ`LVhOFaV5 zd4hUex@GKoiU!4K@NcKS1m3!X`bxO(1?u<0o7<>YC+qhM*DhtPeI&=(uPw+Euh712 zcG`3&ZbjVYwyTmowkhHRh}(5OFu+t`5~Hjy|;;MSL3ZU?+{w*l#$oHI$71 zNMrAh#P?pqI6(apcp396;H$f6y&qo9JSg2VGT)>Pv(b>jd|q+_=8NE^?4@)Eyo&j1 zc(40dY1ir@`_v3)ziN?Zy-(M*#U4A;iMJsxW^t<>7jYZn1dIP@LIk7@mV@Fut3QsD*!`K;k_cq^OWH}D!(|9f~FtA7!m(CfwL zLkEJjpQy!%qc`Cp<{!Wf=AYVoP3MAtnT&hraNpXeoY=a6xa%_-U*tTO;)1vSmHIe% z!3pY<;e#is&xALep?-sO%UJ(+8q7sQw4eH7_|~tem&0?up?(j%Wq|s_@G94}3to#L zn+^0N+{ez~xA3A2`p(=1?;E5Ox5Jxt7xO0&bpD4nyazwWya(REycZrlOUD_2H?#U7 z$DR4TMm1{~IZ_PZHFmLvEO;lY_rWW_qZ8-A{mi5Ab7#f;+E)RB3f52xKgs+Kc+NR( z5a;mjhG8=HJ7!GU+%YMu`*j8dlK+8vGc+RyL3i-oku3yp8$w@ZD_Uxp2Rn zS6%-y1iRRRR=}&7uYu2EJG22_FaH&sx&b!B^NBn2KaHT`O1cxz!JC=yfk&Bl!F!nh z8D7BrW8s!_4YIDH6Mun*Qr0k#bXNaA_yF_tba{>2nU90lIj;8Ksm(^v&l*g4jgKyL zcCw!Ne0a_jTE7&YH7VtC{#6LtS;PJC%Ii`M?)9|bLnifl_{r(ix54WI)Spjx=l2>p zVH&)Qh9>q8qXaw=r}gi`hv!rO7~XIr^;7VAiv|4%cC&%LgB#2}?oXFqql+!*V)(g2 zI`MdT9XxgYCnKm~4Kv^^bLc=Z_#mq`xf; z3-cNcJE>m^@7$TD?Y~p%Lol$5HcW$$e1Uodp8F#8o8XC;sF%R|QjYnTA@H@)hM&Qs z%Tp3nR!+$ws5 zHk?4vevtY<;K!Jsg>QY6)_catYv0LyG(2(0tykCoas(rcHS?#0i=Y3aAJK+q(XgEP zAK;ar(E9!G-fQX39fao|r}gi{kDp>c|Njj^`jK0Q<3 z6UMoe%j)1BXIl%~_* zPBhFKP5o|oX%_W|;Fa=!`BD=<20zYj>Mih)HHi+i9YMYPZ^6_+&%-O^B|x|lkTm|A5_t5wr2Dl8K_ zzaU;JZnt?Q_@;DnVS#%^<`v#tSiZO{zQ`>rSXfdhcz($;=dX^PMV>o6p3=qki20uR z;*Kp@SiHE)`sayN_^fl>s%ERUXtQa4(%~2xz3)1tn6EMlO*8=&JEU?^*AJt}vX{Q<0PC-5@ za)zijvwZ!*wbPWZLg_Kxwe5<9EOk%Ev_Z{WZ53v&wi#xwb|aa&+5*j7eE?G*z|`h6 zO>GxUOMX(wWtyfwLP&3t?`@Q6_iwow(Du#@>gx&U%MR$vmLCXWG^?=O)E_G|sLvYI z_RS2E#px#@U)NZie#@Hr{b=epswv;=xc;VmpJRluzV5I-Ygj)C{hl`Ud)JKUdoMp9 zFtq$sNPgp-s(Sy(&jUpA-Jh(}cUXQvB+K;m$j^XeqHm-A#F_fjw5YIS<3i8D5gqxr Td3?isv&D}c+86zC&M^KDNT;tT delta 9857 zcmZ|Ue{dA#83*vq1-XdfmrVs6q!9xK8sYYGm)udRq+axp5DthIk(!{ObWClck%@}k zAgM5#sX^nssf|u>XpNP&K_jF#>QO2~DWgF{jh&PRC)Vgh8)($nrk(El?)QD~o@b`_ z2esUgziHZHoSacbH2~IY3Je}FV1*nvG4EmGp2}{SNLYlx!95~zZGAo z4$ME7iV_)Lp}=pAFEWWbC=j(W4kI38@zr9VjIT#LnoqkE;*^YUM%-0I(*>B{IGi@eqs4M6HZ(L|hf5-M5N;GQJb>4t8g2#3>oy zkGRd-US^1gMaGqG{21cikV(`*i?H3;c?NMl>;46NjJ5p+-pcB`;aiv=fhX9mJ>j`y z6-4Q}-bF()LVW<<%KSrkKl9JvaW?)qyjS@}sYxynYxCXfHQgu4Zw<4)d19xG7b0$0 zmgvpkL#&;v0duu9^NB4`JGH%q}_tW@sanOyOrxEAUsnm&Vn4Qs(!;=e|She*`~piu!JN z1@mUl9jl!MKSjf&({!Nc;5~iRUxX)FeHYyNGp&CO-m2=IRCo-*3fAy8d_S9@AKt*~ z&%!%d{U`8bdcFMl@J|Fa-p4+(q6v72`2~|@iQlrAPZI}i_lD2XalaW!9ps9$ZtN^V z-1Ao&2jTnQr~WN?>Nk1rSal!Lpb`y{5$fCEE#9YF6V}2Dyid2r zAAq-y8d3+l;ML00Zg>-dJT}l%@LYBV&%)!`^qJWKA0DFlG`Fp7o_*6Tg;KSR*L7Bmj8XYR|Ax5#fTXICg2-gFr~u}k19h`aOWBB+{2 z8y3Ucn1|pI=2ye}naANV=If+8?iu8GzdL(7&i!{28WODG4(+VI20qIC0eBblMtGy^ zsq61f{1XHttf3X&kV~)7bGn}S0eHbeT7L+hGvBy3e-c3#Yd8VlvdA>3tJDXd@cyh| zuG|^;;8nDK7~UA9em+ah@3#uVG?+j`EBnQ0@`Z9Pzm+VZ^)umC|9=4$M@bOYQ@vZPixVit|N6^3;>fr6y(SaQJ7^{C2-g!N(e+pi3 z19A5QAA;F5uwH~Gw~_|y5WH?X^&{~7-PBLQ+smlGr`&!1)~u(&2WS}EK>btr+?%P7 z!&hvio;k(aD;t&^9dYv&*suA~Fy!Dm!aUkVRxroIeb#=I0>b$hB_&7VLJ z8%zaq?VI2myg#r>Qm;GVHFwead*GdSQ-28F%Y4rib^d;<>4&u8Ni?)Ge-_?SOY2{N zkK9AO3!Z&1^*7+h?{#Q!0>J=lI1TT;kJb;uXPl!x0?)3e{v|wb2VCEOpZYD){RqVt zG!@>+d^SALKqt5YUcr0`JpM4PccKW|chMjQPcmNvuWqFE8{y5&x567Aq4hPMJ67ah zG}wWLi$dkPgH{IR&+Yls4U|AjTA?981k$2JuvaBS0Lm%~% z@OtL`(&gv>$Z6UzgoaJbN8wxEqxBQ;p)2W?%bMoxiGcS%L-Gst%7G7@WuO0Z5p;b- zCn$h7eM&t7KhA#o#o$9@v_1j1#*KUTe-naH_OoCcJo~?+d*MrS)gNrlW-?bNN@!n(8#raa-Zl@xWSXWwU zM=bB45br^}+ffa+7JU#5XbI^9-E9{^qI#UDj*7JeRhQ#V9@N!Ay^3Hl`U~|Og}SX! zPaiThVO^}(5i-Skz>pp%q}LJh#&Jq3L%Kht&o!*)29wVm>nlpusj}Fb@>0pG$~U@ihi9$yeb48sh=}d0 ze5>3IR9U{JY<;O!z4^AS)s>N0NyHppE3e;mE6Xd_uUeVFp{>qAqC8*uu8ZU<<=3`6 z6L}z4t3aN$Ixr==r^wMqYOBa1wJ%k*ybwyvfxIG0^xQ!`VxbvFUTaxXsJqnWQKH+` z=224Ypjz!YMS9h0t4RqN(UXPrWNH)3ny@~_u%14wi`6DnHG2B69Ro?q=gRiwvO-}*}QB2HFopVb?ZVW=W?AZ~qk7F|u+1&E%`ULj zLrK-Chmm&_)Ysn*o1w$zBAI<+t7nsHF&o8Jk2259ITe{jt7n%=)uYO*GuJ~s$CL$% x&2;K<<+Ye$+#Tv|HCu1P!-Z>oZ%%&l>YT}kC%)^Oo9SzuE`D~i&;9G|{{Vf)4X*$I diff --git a/hsa/gfx950/mla_v4/nm_ps_src/README.md b/hsa/gfx950/mla_v4/nm_ps_src/README.md index b6c25bd1406..7a063c7cde6 100644 --- a/hsa/gfx950/mla_v4/nm_ps_src/README.md +++ b/hsa/gfx950/mla_v4/nm_ps_src/README.md @@ -45,8 +45,8 @@ DeepSeek-V4 HCA decode kernel (fp8 KV "nm" layout, 128 query heads, qseqlen 1) f | `asm/patch.py` | rt.s -> persistent kernel (library, called by gen.py) | | `asm/align.py` | alignment pads (assembles trial passes, reads label offsets with llvm-readelf) | | `splice/splice.py` | `hipcc -S` + extraction of one kernel body as a splice block | -| `splice/pre.hip` | in-kernel split planner (`t8pre`) | -| `splice/fin.hip` | fused last-arriver combine (`t8fin`) | +| `splice/pre.hip` | in-kernel split planner (`ps_plan`) | +| `splice/fin.hip` | fused last-arriver combine (`ps_merge`) | | `splice/hca_common.hip` | shared definitions: cost line, `Desc`, wave-cooperative search, WG map | ## Build and verify @@ -95,12 +95,14 @@ AMD LLD 22.0.0 (... f58b06dce1f9c15707c5f808fd002e18c2accf7e) (compatible with G ``` Expected sha256 of the output: -`b1a1944b57d006aba1fa91fff4d1ddc87029bef94d8bea066e59a4f7c5ab342c`. +`889d8c653c3278f7c0df138476b1ca0d491b7fced25617f816d6818a68392af8`. ## Launch and kernarg ABI * Grid `(2P, 1, 1)` and block `256`, with no dynamic LDS. `P` is the number of partitions (`P <= 1024`, and - `P % 8 == 0` selects the XCD-aware WG map). Rows: `N <= 65536`. + `P % 8 == 0` selects the XCD-aware WG map). Rows: `N <= 32768`; the byte offsets + of `out` / `q` wrap above that (inherited from the #5195 ASM). The `cnt` / `arange` sizes below come from a + 65536-row counter layout and are larger than that limit needs. * The kernarg block is 336 B: 21 slots of 16 B. Offsets are in bytes. Slots the kernel does not read are passed as the values shown. @@ -119,11 +121,11 @@ Expected sha256 of the output: | 0x70 | f32 | softmax scale `1/sqrt(512)` | | 0x80 | u32 | 128 (gqa ratio) | | 0x90 | u32 | 1 (num_kv_splits) | -| 0x98 | ptr | `cnt` uint32 `[2*65536 + 16*512 + 4*1024]`, zero at rest (row / group counters of the fused combine) | +| 0x98 | ptr | `cnt` int32 `[2*65536 + 16*512 + 4*1024]`, zero at rest: `[0, 2*65536)` row counters, `[2*65536, +16*512)` reserved (unused, but allocated), then `4*1024` group counters of the fused combine | | 0xa0 | u32 | `P` | | 0xa4 | u32 | `N` (rows) | | 0xb0 | u32 | planner config `F | MT << 8` (default `6 | 1 << 8`) | -| 0xb8 | ptr | `span` int32 `[P][2]` (passed by the host wrapper; not read by this build) | +| 0xb8 | ptr | 0 (unused) | | 0xc0 | u32 | 0 | | 0xd0 | ptr | `qo_indptr` = int32 `arange(>= N+1)` | | 0xe0 | ptr | `split_indptr` = int32 `arange(>= N+1)` (same buffer) | diff --git a/hsa/gfx950/mla_v4/nm_ps_src/asm/patch.py b/hsa/gfx950/mla_v4/nm_ps_src/asm/patch.py index 44b003ee190..8925eea2a14 100644 --- a/hsa/gfx950/mla_v4/nm_ps_src/asm/patch.py +++ b/hsa/gfx950/mla_v4/nm_ps_src/asm/patch.py @@ -67,8 +67,8 @@ def patch(src_text, clang, readelf, hipcc, workdir): _pre, _kd = SP.body( hipcc, os.path.join(SP.HERE, "pre.hip"), - "t8pre", - "T8PRE", + "ps_plan", + "PS_PLAN", workdir, vmax=240, smax=100, @@ -78,8 +78,8 @@ def patch(src_text, clang, readelf, hipcc, workdir): _fin, _kf = SP.body( hipcc, os.path.join(SP.HERE, "fin.hip"), - "t8fin", - "T8FIN", + "ps_merge", + "PS_MERGE", workdir, vmax=240, smax=100, @@ -93,7 +93,7 @@ def patch(src_text, clang, readelf, hipcc, workdir): "\tv_accvgpr_write_b32 a241, s1", "\tv_accvgpr_write_b32 a242, s2", _pre, - "T8PRE_RET:", + "PS_PLAN_RET:", "\ts_waitcnt vmcnt(0) lgkmcnt(0)", "\ts_nop 4", "\tv_accvgpr_read_b32 v250, a240", @@ -111,7 +111,7 @@ def patch(src_text, clang, readelf, hipcc, workdir): "\ts_waitcnt lgkmcnt(0)", "\ts_and_b32 s43, s42, 7", "\ts_cmp_eq_u32 s43, 0", - "\ts_cbranch_scc0 T8_MAP_PLAIN", + "\ts_cbranch_scc0 PS_MAP_PLAIN", "\ts_and_b32 s84, s2, 7", "\ts_lshr_b32 s85, s2, 3", "\ts_lshr_b32 s43, s42, 3", @@ -119,17 +119,17 @@ def patch(src_text, clang, readelf, hipcc, workdir): "\ts_lshr_b32 s43, s85, 1", "\ts_add_u32 s101, s84, s43", "\ts_and_b32 s2, s85, 1", - "\ts_branch T8_MAPPED", - "T8_MAP_PLAIN:", + "\ts_branch PS_MAPPED", + "PS_MAP_PLAIN:", "\ts_lshr_b32 s101, s2, 1", "\ts_and_b32 s2, s2, 1", - "T8_MAPPED:", + "PS_MAPPED:", "\tv_accvgpr_read_b32 v250, a243", "\ts_nop 4", "\tv_readfirstlane_b32 s30, v250", # r = begin_row ] PIECE = [ - "T8_PIECE_START:", + "PS_PIECE_START:", "\ts_and_b32 s1, s1, 0xffff", "\ts_load_dwordx2 s[42:43], s[0:1], 0x60", "\ts_load_dwordx2 s[76:77], s[0:1], 0x40", @@ -154,13 +154,13 @@ def patch(src_text, clang, readelf, hipcc, workdir): "\ts_waitcnt lgkmcnt(0)", # s64 begin_row s65 end_row s66 begin_tile s67 end_tile s68 flags "\ts_cmp_gt_i32 s30, s65", - "\ts_cbranch_scc1 T8_DONE", + "\ts_cbranch_scc1 PS_DONE", # K = 0 row inside the walk (e.g. bucket padding): skip without entering the ASM (no LDS use, no barrier) "\ts_cmp_eq_u32 s78, s79", - "\ts_cbranch_scc0 T8_KNZ", + "\ts_cbranch_scc0 PS_KNZ", "\ts_add_u32 s30, s30, 1", - "\ts_branch T8_PIECE_START", - "T8_KNZ:", + "\ts_branch PS_PIECE_START", + "PS_KNZ:", "\tv_accvgpr_write_b32 a236, s78", "\tv_accvgpr_write_b32 a237, s79", "\ts_and_b32 s69, s68, 1", @@ -184,9 +184,9 @@ def patch(src_text, clang, readelf, hipcc, workdir): "\ts_mov_b32 s88, 0", "\ts_mov_b32 s70, 0", "\tv_mov_b32_e32 v0, v248", - "\ts_branch T8_PADA_END", + "\ts_branch PS_PADA_END", "@@PADA@@", - "T8_PADA_END:", # alignment pad (skipped), see align.py + "PS_PADA_END:", # alignment pad (skipped), see align.py ] EPI_PART = [ # label_8068: o_acc / lse_acc slot bases "\ts_mul_i32 s75, s31, 0x40000", @@ -204,7 +204,7 @@ def patch(src_text, clang, readelf, hipcc, workdir): "\ts_and_b32 s9, s9, 0xffff", "\ts_or_b32 s9, s9, 0x40000", "\ts_cmp_eq_u64 s[42:43], 0", - "\ts_cbranch_scc1 T8_NOLSE", + "\ts_cbranch_scc1 PS_NOLSE", "\ts_lshl_b32 s84, s89, 9", "\ts_lshl_b32 s85, s2, 8", "\ts_add_u32 s84, s84, s85", @@ -215,13 +215,13 @@ def patch(src_text, clang, readelf, hipcc, workdir): "\tv_and_b32_e32 v249, 15, v0", "\tv_lshlrev_b32_e32 v249, 2, v249", "\tglobal_store_dword v249, v33, s[42:43]", - "T8_NOLSE:", + "PS_NOLSE:", ] END = [ "\ts_mov_b64 exec, -1", # fused combine (split pieces only) "\ts_cmp_eq_u32 s31, -1", - "\ts_cbranch_scc1 T8FIN_SKIP", + "\ts_cbranch_scc1 PS_MERGE_SKIP", "\tv_accvgpr_write_b32 a230, s30", "\tv_accvgpr_write_b32 a231, s31", "\tv_accvgpr_write_b32 a232, s101", @@ -231,7 +231,7 @@ def patch(src_text, clang, readelf, hipcc, workdir): "\tv_mov_b32_e32 v0, v248", "\ts_nop 4", _fin, - "T8FIN_RET:", + "PS_MERGE_RET:", "\ts_waitcnt vmcnt(0) lgkmcnt(0)", "\ts_mov_b64 exec, -1", "\ts_nop 4", @@ -246,12 +246,12 @@ def patch(src_text, clang, readelf, hipcc, workdir): "\tv_readfirstlane_b32 s2, v252", "\tv_readfirstlane_b32 s30, v253", "\tv_readfirstlane_b32 s101, v254", - "T8FIN_SKIP:", + "PS_MERGE_SKIP:", "\ts_barrier", "\ts_add_u32 s30, s30, 1", - "\ts_branch T8_PIECE_START", + "\ts_branch PS_PIECE_START", "@@PADB@@", - "T8_DONE:", + "PS_DONE:", "\ts_endpgm", ] @@ -327,9 +327,9 @@ def mark(k): out += [ ln, "\ts_cmp_eq_u32 s82, 1", - "\ts_cbranch_scc1 T8_LSE_NAT", + "\ts_cbranch_scc1 PS_LSE_NAT", "\tv_fma_f32 v33, v45, s5, v27", - "T8_LSE_NAT:", + "PS_LSE_NAT:", ] mark("P4") continue diff --git a/hsa/gfx950/mla_v4/nm_ps_src/splice/fin.hip b/hsa/gfx950/mla_v4/nm_ps_src/splice/fin.hip index 0df8a63fdc8..2f015baa888 100644 --- a/hsa/gfx950/mla_v4/nm_ps_src/splice/fin.hip +++ b/hsa/gfx950/mla_v4/nm_ps_src/splice/fin.hip @@ -53,7 +53,7 @@ template __device__ __forceinline__ T* uni(T* p) { // make a pointer #define CPOL_SC1 16 #define OCP " sc1" -extern "C" __global__ void __launch_bounds__(256) t8fin(KA ka) { +extern "C" __global__ void __launch_bounds__(256) ps_merge(KA ka) { __shared__ unsigned s_dec[2]; __shared__ int s_slot[4][FIN_MAXJ]; __shared__ float s_wt[4][FIN_MAXJ + 1][16]; diff --git a/hsa/gfx950/mla_v4/nm_ps_src/splice/pre.hip b/hsa/gfx950/mla_v4/nm_ps_src/splice/pre.hip index 05695bcca49..9547b878fdd 100644 --- a/hsa/gfx950/mla_v4/nm_ps_src/splice/pre.hip +++ b/hsa/gfx950/mla_v4/nm_ps_src/splice/pre.hip @@ -85,7 +85,7 @@ __device__ __forceinline__ void plan_both(const int* __restrict__ ind, int N, in // kernarg reads through the constant-address-space kernarg pointer -> s_load (the by-value struct compiled to vector loads) #define KARG(T, off) (*(T const __attribute__((address_space(4)))*)((const __attribute__((address_space(4))) char*)__builtin_amdgcn_kernarg_segment_ptr() + (off))) -extern "C" __global__ void __launch_bounds__(256) t8pre(KA ka) { +extern "C" __global__ void __launch_bounds__(256) ps_plan(KA ka) { const int* ind = KARG(const int*, 0x40); Desc* desc = KARG(Desc*, 0x60); const int P = KARG(int, 0xa0), N = KARG(int, 0xa4), F = KARG(int, 0xb0); From 1b6ca77bd2c2951ed5ddaa4bb24d4ce46bb0ff90 Mon Sep 17 00:00:00 2001 From: yihonglie Date: Tue, 29 Sep 2026 12:07:40 -0500 Subject: [PATCH 09/13] [MLA v4 nm] Mark kernarg slot 0x68 of the persistent kernel as unused Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie --- csrc/py_itfs_cu/asm_mla_v4.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csrc/py_itfs_cu/asm_mla_v4.cu b/csrc/py_itfs_cu/asm_mla_v4.cu index a716f6e3d5c..de0a345b1ea 100644 --- a/csrc/py_itfs_cu/asm_mla_v4.cu +++ b/csrc/py_itfs_cu/asm_mla_v4.cu @@ -574,7 +574,7 @@ struct __attribute__((packed)) MlaV4PsKernelArgs void* ptr_LTD; p2 _p_ltd; // 0x50: kv_page_indices void* ptr_desc; // 0x60: repurposed: workspace desc [P, 8] int32 (rw) - void* ptr_dbg; // 0x68: debug probe, nullptr + void* ptr_dbg; // 0x68: unused, nullptr float scalar_f; p3 _p_sc; // 0x70: 1.0f/sqrtf(kV4DimNope+kV4DimRope) unsigned int s_gqa_ratio; From 5e6d258a17419811d198a04ac62ad8b0b1c79abb Mon Sep 17 00:00:00 2001 From: yihonglie Date: Sat, 3 Oct 2026 10:46:23 -0500 Subject: [PATCH 10/13] [MLA v4 nm] Ship the persistent decode kernel as a code object only Drop hsa/gfx950/mla_v4/nm_ps_src (assembly, patch scripts and HIP splice sources). Nothing at build or run time reads it; the kernel ships as mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co, unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie --- hsa/gfx950/mla_v4/nm_ps_src/README.md | 144 - hsa/gfx950/mla_v4/nm_ps_src/asm/align.py | 65 - hsa/gfx950/mla_v4/nm_ps_src/asm/patch.py | 437 - hsa/gfx950/mla_v4/nm_ps_src/asm/rt.s | 15413 ---------------- hsa/gfx950/mla_v4/nm_ps_src/gen.py | 154 - hsa/gfx950/mla_v4/nm_ps_src/splice/fin.hip | 273 - .../mla_v4/nm_ps_src/splice/hca_common.hip | 100 - hsa/gfx950/mla_v4/nm_ps_src/splice/pre.hip | 172 - hsa/gfx950/mla_v4/nm_ps_src/splice/splice.py | 95 - 9 files changed, 16853 deletions(-) delete mode 100644 hsa/gfx950/mla_v4/nm_ps_src/README.md delete mode 100644 hsa/gfx950/mla_v4/nm_ps_src/asm/align.py delete mode 100644 hsa/gfx950/mla_v4/nm_ps_src/asm/patch.py delete mode 100644 hsa/gfx950/mla_v4/nm_ps_src/asm/rt.s delete mode 100644 hsa/gfx950/mla_v4/nm_ps_src/gen.py delete mode 100644 hsa/gfx950/mla_v4/nm_ps_src/splice/fin.hip delete mode 100644 hsa/gfx950/mla_v4/nm_ps_src/splice/hca_common.hip delete mode 100644 hsa/gfx950/mla_v4/nm_ps_src/splice/pre.hip delete mode 100644 hsa/gfx950/mla_v4/nm_ps_src/splice/splice.py diff --git a/hsa/gfx950/mla_v4/nm_ps_src/README.md b/hsa/gfx950/mla_v4/nm_ps_src/README.md deleted file mode 100644 index 7a063c7cde6..00000000000 --- a/hsa/gfx950/mla_v4/nm_ps_src/README.md +++ /dev/null @@ -1,144 +0,0 @@ - -# mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps: generator sources - -This directory regenerates the AITER code object -`hsa/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co` (kernel symbol -`_ZN5aiter39mla_a8w8_qh64_qseqlen1_gqaratio64_nm_psE`) from sources. It is the persistent -DeepSeek-V4 HCA decode kernel (fp8 KV "nm" layout, 128 query heads, qseqlen 1) for gfx950. - -## Provenance - -* **Base.** The kernel is derived from the AITER #5195 ASM kernel - `hsa/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co` - (`_ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE`). `asm/rt.s` is a disassembly of that code object, made - re-assemblable (llvm-objdump output plus the original kernel descriptor bytes and metadata). Assembled, it gives - the same `.text` and kernel descriptor (`.rodata`) as the #5195 `.co` (`gen.py --verify-rt` checks this). `rt.s` - is kept as generated: `asm/patch.py` finds some of its patch sites by line number, so the file carries no license - header, and `gen.py` checks its sha256 before it uses the file. -* **Persistent descriptor loop.** `asm/patch.py` rewrites the disassembly with scripts only; nobody edits the - output by hand. The grid becomes `(2P, 1, 1)`. Each workgroup owns one partition `p` and one head group `hg`, - and walks the rows `begin_row..end_row` of its descriptor. It runs one tile range `[tb, te)` of a row per pass, - re-entering the ASM's own prologue each time. The patches retarget the ASM's split-partial epilogue to the slot - layout of `o_acc` / `lse_acc`, apply the sink once, write the natural-log LSE of unsplit rows, and relax some - waits. Two never-executed `s_nop` pads keep the hot loop code at its original byte offset mod 256 - (`asm/align.py`). The docstring of `asm/patch.py` lists every patch. -* **In-kernel split planner.** `splice/pre.hip` holds the planner, written in HIP. It runs the closed-form cost-line - planner, a 64-way wave-cooperative boundary search, and an automatic split or no-split choice. The source is - compiled with `hipcc -S`, and `splice/splice.py` extracts the kernel body from that assembly. `asm/patch.py` - then splices the body into the kernel entry, so no separate planner launch is needed. Each workgroup publishes - its descriptor to `desc[p]`. -* **Fused last-arriver combine.** `splice/fin.hip` is also written in HIP and spliced the same way, at the end of - every split piece. Each piece counts its tiles into an agent-scope atomic counter (`cnt`). The piece that - completes a row merges all of that row's partials, adds the sink, and writes `out` / `lse_out`. Wide rows are - merged in two levels. The partial stores use sc1 (agent-scope write-through), so no separate combine launch is - needed, and the counters return to zero at the end of every completed call. - -## Layout - -| path | role | -|---|---| -| `gen.py` | the only build entry point (python3 stdlib only) | -| `asm/rt.s` | re-assemblable disassembly of the #5195 ASM (input, not edited) | -| `asm/patch.py` | rt.s -> persistent kernel (library, called by gen.py) | -| `asm/align.py` | alignment pads (assembles trial passes, reads label offsets with llvm-readelf) | -| `splice/splice.py` | `hipcc -S` + extraction of one kernel body as a splice block | -| `splice/pre.hip` | in-kernel split planner (`ps_plan`) | -| `splice/fin.hip` | fused last-arriver combine (`ps_merge`) | -| `splice/hca_common.hip` | shared definitions: cost line, `Desc`, wave-cooperative search, WG map | - -## Build and verify - -Run inside a ROCm container, for example with the toolchain listed below: - -```bash -python3 gen.py --out /tmp/x.co \ - --check /hsa/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co -# optional: keep the intermediates / check the rt.s provenance -python3 gen.py --out /tmp/x.co --keep /tmp/nm_ps_build \ - --verify-rt /hsa/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co -``` - -`gen.py` takes these steps: - -1. Check the sha256 of `asm/rt.s`. -2. Compile `splice/pre.hip` and `splice/fin.hip` with - `hipcc --offload-arch=gfx950 -O3 --offload-device-only -std=c++20 -w -S`. -3. Run `asm/patch.py`, including the alignment passes. -4. Rename the kernel symbol from `_ZN5aiter36..._nmE` to `_ZN5aiter39..._nm_psE`. This covers every occurrence: - the function symbol, `.kd`, and the metadata `.name` / `.symbol`. The Itanium length prefix is recomputed - (36 -> 39). -5. Assemble with `clang -x assembler -target amdgcn-amd-amdhsa -mcpu=gfx950 -c`. -6. Link with `ld.lld -shared`. -7. With `--check`, compare the sha256 against the given file. On a mismatch it exits with status 1. - -Tools are taken from `$ROCM_PATH` (default `/opt/rocm`): `llvm/bin/{clang,ld.lld,llvm-readelf,llvm-objcopy}` and -`bin/hipcc`. The output is byte-identical only with the same toolchain. `hipcc` codegen and `clang` / `lld` object -layout can change between ROCm releases. - -Toolchain used for the shipped `.co` (container image `rocm/atom-dev:nightly_202609161445`, `/opt/rocm` -> -`/opt/rocm-7.2.4`): - -``` -$ /opt/rocm/llvm/bin/clang --version -AMD clang version 22.0.0git (https://github.com/RadeonOpenCompute/llvm-project roc-7.2.4 26084 f58b06dce1f9c15707c5f808fd002e18c2accf7e) -Target: x86_64-unknown-linux-gnu -Thread model: posix -InstalledDir: /opt/rocm-7.2.4/lib/llvm/bin -$ hipcc --version -HIP version: 7.2.53211-97f5574fe2 -AMD clang version 22.0.0git (https://github.com/RadeonOpenCompute/llvm-project roc-7.2.4 26084 f58b06dce1f9c15707c5f808fd002e18c2accf7e) -$ /opt/rocm/llvm/bin/ld.lld --version -AMD LLD 22.0.0 (... f58b06dce1f9c15707c5f808fd002e18c2accf7e) (compatible with GNU linkers) -``` - -Expected sha256 of the output: -`889d8c653c3278f7c0df138476b1ca0d491b7fced25617f816d6818a68392af8`. - -## Launch and kernarg ABI - -* Grid `(2P, 1, 1)` and block `256`, with no dynamic LDS. `P` is the number of partitions (`P <= 1024`, and - `P % 8 == 0` selects the XCD-aware WG map). Rows: `N <= 32768`; the byte offsets - of `out` / `q` wrap above that (inherited from the #5195 ASM). The `cnt` / `arange` sizes below come from a - 65536-row counter layout and are larger than that limit needs. -* The kernarg block is 336 B: 21 slots of 16 B. Offsets are in bytes. Slots the kernel does not read are passed as - the values shown. - -| offset | type | content | -|---|---|---| -| 0x00 | ptr | `o_acc` fp32 `[2P][128][512]`: split-piece partial O (workspace) | -| 0x08 | ptr | `out` bf16 `[N][128][512]` | -| 0x10 | ptr | `lse_acc` fp32 `[2P][128]`: split-piece partial LSE, log2 domain, no sink (workspace) | -| 0x18 | ptr | `lse_out` fp32 `[N][128]`, natural log, sink included; 0 = not written | -| 0x20 | ptr | `q_packed` (Q NoPE, fp8 packed) | -| 0x30 | ptr | `kv_packed` (KV NoPE fp8 + scales, 512 B per token) | -| 0x40 | ptr | `kv_indptr` int32 `[N+1]` | -| 0x50 | ptr | `kv_page_indices` int32 | -| 0x60 | ptr | `desc` int32 `[P][8]`: per-partition descriptor, written by the kernel (workspace) | -| 0x68 | ptr | 0 (unused) | -| 0x70 | f32 | softmax scale `1/sqrt(512)` | -| 0x80 | u32 | 128 (gqa ratio) | -| 0x90 | u32 | 1 (num_kv_splits) | -| 0x98 | ptr | `cnt` int32 `[2*65536 + 16*512 + 4*1024]`, zero at rest: `[0, 2*65536)` row counters, `[2*65536, +16*512)` reserved (unused, but allocated), then `4*1024` group counters of the fused combine | -| 0xa0 | u32 | `P` | -| 0xa4 | u32 | `N` (rows) | -| 0xb0 | u32 | planner config `F | MT << 8` (default `6 | 1 << 8`) | -| 0xb8 | ptr | 0 (unused) | -| 0xc0 | u32 | 0 | -| 0xd0 | ptr | `qo_indptr` = int32 `arange(>= N+1)` | -| 0xe0 | ptr | `split_indptr` = int32 `arange(>= N+1)` (same buffer) | -| 0xf0 | u32 | 1 (bf16 output) | -| 0x100 | ptr | `q_rope` (bf16) | -| 0x110 | ptr | `kv_rope` (bf16) | -| 0x120 | ptr | `sink` fp32 `[128]` | -| 0x130 | ptr | 0 (valid_split) | -| 0x140 | u32 | 0 (use_valid_split) | - -Workspace rules: -* One workspace (`o_acc`, `lse_acc`, `desc`, `cnt`) can serve any number of calls that are ordered with respect to - each other. -* Calls that can run concurrently need separate workspaces. -* Rows with `K = 0` are not written. -* A kernel that aborts can leave `cnt` non-zero. Zero it again before the next call. diff --git a/hsa/gfx950/mla_v4/nm_ps_src/asm/align.py b/hsa/gfx950/mla_v4/nm_ps_src/asm/align.py deleted file mode 100644 index e47f34a79f3..00000000000 --- a/hsa/gfx950/mla_v4/nm_ps_src/asm/align.py +++ /dev/null @@ -1,65 +0,0 @@ -# SPDX-License-Identifier: MIT -# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. -"""Alignment-preserving pads for script-patched copies of rt.s. Two never-executed s_nop pads (@@PADA@@ before the -loop code, @@PADB@@ after the epilogue; each placed after an unconditional branch or s_endpgm) absorb inserted bytes so -the hot code keeps its original byte offset mod 256: loop copies label_1F54/3D74/5B68, drain label_795C, tail tile -label_8EFC, rescale subroutine label_9DB4. Offsets are read from the symbol table of an assembled pass. -""" - -import re -import subprocess -import tempfile - -A = 256 -PRE = ("label_1F54", "label_3D74", "label_5B68", "label_795C") -POST = ("label_8EFC", "label_9DB4") - - -def offsets(text, clang, readelf): - with tempfile.TemporaryDirectory() as d: - open(f"{d}/x.s", "w").write(text) - subprocess.run( - [ - clang, - "-x", - "assembler", - "-target", - "amdgcn-amd-amdhsa", - "-mcpu=gfx950", - "-c", - f"{d}/x.s", - "-o", - f"{d}/x.o", - ], - check=True, - ) - r = subprocess.run( - [readelf, "-s", f"{d}/x.o"], capture_output=True, text=True, check=True - ).stdout - return { - m[2]: int(m[1], 16) - for m in re.finditer( - r"^\s*\d+:\s+([0-9a-f]+)\s+0\s+NOTYPE\s+LOCAL\s+DEFAULT\s+\d+\s+(label_\w+)", - r, - re.M, - ) - } - - -def render(body, pads): - for k, n in pads.items(): - body = body.replace(k, "\n".join(["\ts_nop 0"] * (n // 4)) if n else "") - return body - - -def align(src_text, body, clang, readelf): - pads = {"@@PADA@@": 0, "@@PADB@@": 0} - orig = offsets(src_text, clang, readelf) - o = offsets(render(body, pads), clang, readelf) - pads["@@PADA@@"] = (-(o["label_1F54"] - orig["label_1F54"])) % A - o = offsets(render(body, pads), clang, readelf) - pads["@@PADB@@"] = (-(o["label_8EFC"] - orig["label_8EFC"])) % A - o = offsets(render(body, pads), clang, readelf) - for lab in PRE + POST: - assert (o[lab] - orig[lab]) % A == 0, (lab, o[lab], orig[lab]) - return render(body, pads), pads diff --git a/hsa/gfx950/mla_v4/nm_ps_src/asm/patch.py b/hsa/gfx950/mla_v4/nm_ps_src/asm/patch.py deleted file mode 100644 index 8925eea2a14..00000000000 --- a/hsa/gfx950/mla_v4/nm_ps_src/asm/patch.py +++ /dev/null @@ -1,437 +0,0 @@ -# SPDX-License-Identifier: MIT -# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. -"""rt.s (disassembly of the AITER #5195 v4 nm ASM, reassembles to the same .text / kernel descriptor) -> the persistent -HCA decode kernel. Script-driven rewrites only; the output is never hand-edited. Library: patch(src_text, ...) -> text. -The kernel symbol is left as in rt.s; gen.py renames it afterwards. - -Launch: grid (2P,1,1), 256 threads. WG w -> (p, hg) (xcd = w&7, loc = w>>3, p = xcd*(P/8) + loc/2, hg = loc&1 when -P%8 == 0, else p = w/2, hg = w&1). s2 := hg for the whole kernel. Kernarg: see README.md (21 x 16 B slots). -Entry (splice/pre.hip, the in-kernel split planner, spliced into the prologue): every WG plans its own partition p -(split and no-split plans from one search; the no-split plan is chosen when its max partition load stays within -NS_WANY of the split plan's), publishes desc[p] (hg == 0 WG) and leaves (begin_row, end_row, begin_tile, end_tile, -flags) in a243..a247, per / Pa / org in a248..a250. s0,s1,s2 are saved in a240..a242 around the splice. -Walk (per WG): rows r = begin_row .. end_row; piece (r, tb, te): - tb = r==begin_row ? begin_tile : 0, te = r==end_row ? end_tile : 2^20 (clamped by K), - split = (r==begin_row && flags&1) || (r==end_row && flags&2), slot = 2p + (r != begin_row). - Rows with K = 0 inside the walk are skipped without entering the ASM (they are left unwritten). - Each piece re-enters the ASM's own prologue with s3=r, s4=0, v0=thread id; at the ASM's s_endpgm the split piece - runs the fused combine (splice/fin.hip: last-arriver merge through agent-scope atomic counters at kernarg 0x98), - then all 4 waves barrier, r++, loop. Empty partition: begin_row=N > end_row=-1 -> exit. -Patches of the ASM body: - P1 chunk ([tb,te) contiguous tiles): s68 += 32tb, s71 = min(K-32tb, 32(te-tb)), s100=0 (no round-robin offset), - s65=s67, s94=4*s67 (index stride of consecutive tiles). - P2 s82 := split ? 2 : 1 (selects partial fp32 path vs bf16 path in the epilogue). - P3 sink: the ASM's "split 0 owns the sink" tests (s_cmp_eq_u32 s4, 0 in the epilogue) become "s82 == 1": sink - applied exactly once, in the unsplit epilogue; split pieces carry none (the fused combine adds it). - P4 partial LSE in the log2 domain: split pieces overwrite lse = m*s5 + log2(ln). - P5 partial store retargeted to the slot layout: at label_8068 o_acc base += slot*0x40000, lse_acc base += slot*512, - s89 (row) := 0 -> o_acc[slot][hg*64+h][:], lse_acc[slot][hg*64+h] (the ASM's own lane addressing). - P6 bf16 path (label_86D0): base = out (kernarg 0x08), row = s89 = r; plus (if lse_out != 0) - lse_out[r][hg*64+16w+ln%16] = lse (natural log, sink included). - P7 reloc: epilogue O-transpose LDS scratch 0x0..0x87ff -> 0x1f000..0x277ff. - sc1: the partial path's stores (label_8068 .. label_86CC) get sc1 (agent-scope write-through, read by fin.hip). - nowait / nowait2: epilogue store waits relaxed to lgkmcnt-only (stores need no completion wait before the next - piece; the fused combine waits vmcnt(0) itself before its counter atomic). - kfast: the entry touches the kernarg lines the ASM prologue reads (0x00, 0xc0, 0x100, 0x140) so its s_loads hit the - K$; the prologue's dependent indptr round trip becomes moves (kv_indptr[r], [r+1] from the wrapper's load in - s84/s85; split_indptr / qo_indptr = arange contract -> r, r+1). - kv0early: Q staging moved out of the KV ring (nope 0x2000.. -> 0x1f000.., rope 0x12800.. -> 0x17000..) so the KV - tile-0 loads (label_0A6C block) move up to right after the page indices land. - align: two never-executed s_nop pads keep the hot code at its original byte offset mod 256 (asm/align.py). -Registers used by the wrapper (never touched by rt.s): s30 (r), s31 (slot or -1), s38/s39 (tb/te), s42 s43 s84 s85 -(scratch), s101 (p), v248 (thread id), v249 (lse address), v250..v254 (readfirstlane temps), a230..a250. -""" - -import os -import sys - -HERE = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, HERE) -sys.path.insert(0, os.path.join(HERE, "..", "splice")) -import align as AL # noqa: E402 -import splice as SP # noqa: E402 - -K = "_ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE" # kernel symbol of rt.s -KF_MAP = { - "s_load_dword s68, s[28:29], 0x0": "\ts_mov_b32 s68, s84", - "s_load_dword s69, s[28:29], 0x4": "\ts_mov_b32 s69, s85", - "s_load_dword s93, s[40:41], 0x0": "\ts_mov_b32 s93, s3", - "s_load_dword s92, s[40:41], 0x4": "\ts_add_u32 s92, s3, 1", - "s_load_dword s89, s[36:37], 0x0": "\ts_mov_b32 s89, s3", - "s_load_dword s90, s[36:37], 0x4": "\ts_add_u32 s90, s3, 1", -} - - -def patch(src_text, clang, readelf, hipcc, workdir): - """src_text: rt.s. Returns (patched text, marks, pads).""" - _pre, _kd = SP.body( - hipcc, - os.path.join(SP.HERE, "pre.hip"), - "ps_plan", - "PS_PLAN", - workdir, - vmax=240, - smax=100, - allow_lds=True, - ) # uses <= 4.1 KB of LDS; ends with a barrier before the ASM - assert _kd["group_segment_fixed_size"] <= 8192, _kd["group_segment_fixed_size"] - _fin, _kf = SP.body( - hipcc, - os.path.join(SP.HERE, "fin.hip"), - "ps_merge", - "PS_MERGE", - workdir, - vmax=240, - smax=100, - allow_lds=True, - ) - assert _kf["group_segment_fixed_size"] <= 0x1F000 - - ENTRY = [ - "\tv_mov_b32_e32 v248, v0", - "\tv_accvgpr_write_b32 a240, s0", - "\tv_accvgpr_write_b32 a241, s1", - "\tv_accvgpr_write_b32 a242, s2", - _pre, - "PS_PLAN_RET:", - "\ts_waitcnt vmcnt(0) lgkmcnt(0)", - "\ts_nop 4", - "\tv_accvgpr_read_b32 v250, a240", - "\tv_accvgpr_read_b32 v251, a241", - "\tv_accvgpr_read_b32 v252, a242", - "\ts_nop 4", - "\tv_readfirstlane_b32 s0, v250", - "\tv_readfirstlane_b32 s1, v251", - "\tv_readfirstlane_b32 s2, v252", - "\ts_load_dword s42, s[0:1], 0xa0", # P - "\ts_load_dword s43, s[0:1], 0x0", - "\ts_load_dword s84, s[0:1], 0xc0", - "\ts_load_dword s85, s[0:1], 0x100", - "\ts_load_dword s101, s[0:1], 0x140", # kfast: warm the K$ lines of the ASM prologue - "\ts_waitcnt lgkmcnt(0)", - "\ts_and_b32 s43, s42, 7", - "\ts_cmp_eq_u32 s43, 0", - "\ts_cbranch_scc0 PS_MAP_PLAIN", - "\ts_and_b32 s84, s2, 7", - "\ts_lshr_b32 s85, s2, 3", - "\ts_lshr_b32 s43, s42, 3", - "\ts_mul_i32 s84, s84, s43", - "\ts_lshr_b32 s43, s85, 1", - "\ts_add_u32 s101, s84, s43", - "\ts_and_b32 s2, s85, 1", - "\ts_branch PS_MAPPED", - "PS_MAP_PLAIN:", - "\ts_lshr_b32 s101, s2, 1", - "\ts_and_b32 s2, s2, 1", - "PS_MAPPED:", - "\tv_accvgpr_read_b32 v250, a243", - "\ts_nop 4", - "\tv_readfirstlane_b32 s30, v250", # r = begin_row - ] - PIECE = [ - "PS_PIECE_START:", - "\ts_and_b32 s1, s1, 0xffff", - "\ts_load_dwordx2 s[42:43], s[0:1], 0x60", - "\ts_load_dwordx2 s[76:77], s[0:1], 0x40", - "\ts_load_dwordx2 s[74:75], s[0:1], 0xa0", - "\ts_waitcnt lgkmcnt(0)", - "\tv_accvgpr_read_b32 v250, a243", - "\tv_accvgpr_read_b32 v251, a244", - "\tv_accvgpr_read_b32 v252, a245", - "\tv_accvgpr_read_b32 v253, a246", - "\tv_accvgpr_read_b32 v254, a247", - "\ts_nop 4", - "\tv_readfirstlane_b32 s64, v250", - "\tv_readfirstlane_b32 s65, v251", - "\tv_readfirstlane_b32 s66, v252", - "\tv_readfirstlane_b32 s67, v253", - "\tv_readfirstlane_b32 s68, v254", - "\ts_add_u32 s85, s75, -1", - "\ts_min_i32 s85, s30, s85", - "\ts_max_i32 s85, s85, 0", - "\ts_lshl_b32 s85, s85, 2", - "\ts_load_dwordx2 s[78:79], s[76:77], s85", # kv_indptr[min(r, N-1)], kv_indptr[..+1] - "\ts_waitcnt lgkmcnt(0)", - # s64 begin_row s65 end_row s66 begin_tile s67 end_tile s68 flags - "\ts_cmp_gt_i32 s30, s65", - "\ts_cbranch_scc1 PS_DONE", - # K = 0 row inside the walk (e.g. bucket padding): skip without entering the ASM (no LDS use, no barrier) - "\ts_cmp_eq_u32 s78, s79", - "\ts_cbranch_scc0 PS_KNZ", - "\ts_add_u32 s30, s30, 1", - "\ts_branch PS_PIECE_START", - "PS_KNZ:", - "\tv_accvgpr_write_b32 a236, s78", - "\tv_accvgpr_write_b32 a237, s79", - "\ts_and_b32 s69, s68, 1", - "\ts_bfe_u32 s70, s68, 0x10001", - "\ts_cmp_eq_u32 s30, s64", - "\ts_cselect_b32 s38, s66, 0", - "\ts_cselect_b32 s71, s69, 0", - "\ts_cselect_b32 s72, 0, 1", - "\ts_cmp_eq_u32 s30, s65", - "\ts_cselect_b32 s39, s67, 0x100000", - "\ts_cselect_b32 s73, s70, 0", - "\ts_or_b32 s71, s71, s73", - "\ts_lshl_b32 s73, s101, 1", - "\ts_add_u32 s72, s73, s72", - "\ts_cmp_eq_u32 s71, 0", - "\ts_cselect_b32 s31, -1, s72", - "\ts_mov_b32 s3, s30", - "\ts_mov_b32 s4, 0", - "\ts_mov_b32 s84, s78", - "\ts_mov_b32 s85, s79", - "\ts_mov_b32 s88, 0", - "\ts_mov_b32 s70, 0", - "\tv_mov_b32_e32 v0, v248", - "\ts_branch PS_PADA_END", - "@@PADA@@", - "PS_PADA_END:", # alignment pad (skipped), see align.py - ] - EPI_PART = [ # label_8068: o_acc / lse_acc slot bases - "\ts_mul_i32 s75, s31, 0x40000", - "\ts_add_u32 s8, s8, s75", - "\ts_addc_u32 s9, s9, 0", - "\ts_lshl_b32 s75, s31, 9", - "\ts_add_u32 s12, s12, s75", - "\ts_addc_u32 s13, s13, 0", - "\ts_mov_b32 s89, 0", - ] - EPI_BF16 = [ # label_86D0: out base + optional natural-log LSE of the unsplit row - "\ts_load_dwordx2 s[8:9], s[0:1], 0x8", - "\ts_load_dwordx2 s[42:43], s[0:1], 0x18", - "\ts_waitcnt lgkmcnt(0)", - "\ts_and_b32 s9, s9, 0xffff", - "\ts_or_b32 s9, s9, 0x40000", - "\ts_cmp_eq_u64 s[42:43], 0", - "\ts_cbranch_scc1 PS_NOLSE", - "\ts_lshl_b32 s84, s89, 9", - "\ts_lshl_b32 s85, s2, 8", - "\ts_add_u32 s84, s84, s85", - "\ts_lshl_b32 s85, s7, 6", - "\ts_add_u32 s84, s84, s85", - "\ts_add_u32 s42, s42, s84", - "\ts_addc_u32 s43, s43, 0", - "\tv_and_b32_e32 v249, 15, v0", - "\tv_lshlrev_b32_e32 v249, 2, v249", - "\tglobal_store_dword v249, v33, s[42:43]", - "PS_NOLSE:", - ] - END = [ - "\ts_mov_b64 exec, -1", - # fused combine (split pieces only) - "\ts_cmp_eq_u32 s31, -1", - "\ts_cbranch_scc1 PS_MERGE_SKIP", - "\tv_accvgpr_write_b32 a230, s30", - "\tv_accvgpr_write_b32 a231, s31", - "\tv_accvgpr_write_b32 a232, s101", - "\tv_accvgpr_write_b32 a233, s2", - "\tv_accvgpr_write_b32 a234, s38", - "\tv_accvgpr_write_b32 a235, s39", - "\tv_mov_b32_e32 v0, v248", - "\ts_nop 4", - _fin, - "PS_MERGE_RET:", - "\ts_waitcnt vmcnt(0) lgkmcnt(0)", - "\ts_mov_b64 exec, -1", - "\ts_nop 4", - "\tv_accvgpr_read_b32 v250, a240", - "\tv_accvgpr_read_b32 v251, a241", - "\tv_accvgpr_read_b32 v252, a233", - "\tv_accvgpr_read_b32 v253, a230", - "\tv_accvgpr_read_b32 v254, a232", - "\ts_nop 4", - "\tv_readfirstlane_b32 s0, v250", - "\tv_readfirstlane_b32 s1, v251", - "\tv_readfirstlane_b32 s2, v252", - "\tv_readfirstlane_b32 s30, v253", - "\tv_readfirstlane_b32 s101, v254", - "PS_MERGE_SKIP:", - "\ts_barrier", - "\ts_add_u32 s30, s30, 1", - "\ts_branch PS_PIECE_START", - "@@PADB@@", - "PS_DONE:", - "\ts_endpgm", - ] - - out, done = [], {} - - def mark(k): - done[k] = done.get(k, 0) + 1 - - lines = src_text.split("\n") - _sa, _sb = lines.index("label_8068:"), lines.index("label_86CC:") - # kv0early: Q-wait position and the label_0A6C KV tile-0 block - _qa = next(i for i, ln in enumerate(lines) if ln.strip() == "s_waitcnt vmcnt(2)") - _k0 = lines.index("label_0A6C:") - _k1 = next( - i - for i in range(_k0, _k0 + 80) - if lines[i].strip() == "s_waitcnt lgkmcnt(0)" - and lines[i + 1].strip() == "s_barrier" - ) - KV0BLK = lines[_k0 + 1 : _k1] - assert ( - lines[_qa + 1].strip() == "s_barrier" - and any("global_load_lds_dwordx4" in x for x in KV0BLK) - and len(KV0BLK) == 50 - ), len(KV0BLK) - for i, ln in enumerate(lines): - t = ln.strip() - if _sa < i < _sb and t.startswith("buffer_store"): - ln = ln + " sc1" - t = ln.strip() - mark("sc1") - if ln == f"{K}:": - out += [ln] + ENTRY + PIECE - mark("entry") - continue - # P1 chunk - if t == "s_sub_u32 s71, s69, s68": - out += [ - ln, - "\ts_lshl_b32 s85, s38, 5", - "\ts_add_u32 s68, s68, s85", - "\ts_sub_u32 s71, s71, s85", - "\ts_sub_u32 s84, s39, s38", - "\ts_lshl_b32 s84, s84, 5", - "\ts_min_u32 s71, s71, s84", - ] - mark("P1a") - continue - if t == "s_mul_i32 s100, s4, s67": - out += [ln, "\ts_mov_b32 s100, 0"] - mark("P1b") - continue - if t == "s_mul_i32 s65, s82, s67": - out += ["\ts_mov_b32 s65, s67"] - mark("P1c") - continue - if t == "s_mul_i32 s94, s94, s82": - out += [ln, "\ts_mul_i32 s94, s67, 4"] - mark("P1d") - continue - # P2 - if t == "s_mov_b32 s82, s98": - out += [ln, "\ts_cmp_eq_u32 s31, -1", "\ts_cselect_b32 s82, 1, 2"] - mark("P2") - continue - # P3 (both "s_cmp_eq_u32 s4, 0" inside the epilogue region) - if t == "s_cmp_eq_u32 s4, 0" and i > 4000: - out += ["\ts_cmp_eq_u32 s82, 1"] - mark("P3") - continue - # P4 - if t == "v_fma_f32 v33, v27, s6, v26": - out += [ - ln, - "\ts_cmp_eq_u32 s82, 1", - "\ts_cbranch_scc1 PS_LSE_NAT", - "\tv_fma_f32 v33, v45, s5, v27", - "PS_LSE_NAT:", - ] - mark("P4") - continue - # P5 / P6 - if ln == "label_8068:": - out += [ln] + EPI_PART - mark("P5") - continue - if ln == "label_86D0:": - out += [ln] + EPI_BF16 - mark("P6") - continue - # P7 reloc - if ( - t in ("v_lshlrev_b32_e32 v16, 2, v16", "v_lshlrev_b32_e32 v10, 2, v10") - and i > 4000 - ): - r = t.split()[1].rstrip(",") - out += [ln, f"\tv_add_u32_e32 {r}, 0x1f000, {r}"] - mark("P7") - continue - # kv0early - if i < 520: - if t == "s_add_u32 s56, 0x2000, s75" and i < 300: - out += ["\ts_add_u32 s56, 0x1f000, s75"] - mark("kv0e") - continue - if t == "v_add_u32_e32 v16, 0x12800, v16" and i < 330: - out += ["\tv_add_u32_e32 v16, 0x17000, v16"] - mark("kv0e") - continue - if t == "s_add_u32 m0, 0x12800, s75" and i < 330: - out += ["\ts_add_u32 m0, 0x17000, s75"] - mark("kv0e") - continue - if i == _qa: - out += ["\ts_waitcnt vmcnt(10)"] + KV0BLK + ["\ts_waitcnt vmcnt(8)"] - mark("kv0e") - continue - if i == _qa + 1: - out += [ - ln, - "\tv_add_u32_e32 v10, 0x1d000, v10", - "\tv_add_u32_e32 v34, 0x1d000, v34", - ] - mark("kv0e") - continue - if t == "s_waitcnt vmcnt(0)" and lines[i - 1] == "label_09EC:": - out += ["\ts_waitcnt vmcnt(6)"] - mark("kv0e") - continue - if _k0 < i < _k1: - mark("kv0drop") - continue - # kfast - if i < 80 and t in KF_MAP: - out += [KF_MAP[t]] - mark("kfast") - continue - # nowait2 / nowait - if ( - t == "s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)" - and lines[i - 1] == "label_8EF4:" - ): - out += ["\ts_waitcnt lgkmcnt(0)"] - mark("nowait2") - continue - if t == "s_waitcnt vmcnt(0) lgkmcnt(0)" and 4641 < i < 5431: - out += ["\ts_waitcnt lgkmcnt(0)"] - mark("nowait") - continue - if t == "s_endpgm": - out += END - mark("end") - continue - out.append(ln) - exp = dict( - entry=1, - P1a=1, - P1b=1, - P1c=1, - P1d=1, - P2=1, - P3=2, - P4=1, - P5=1, - P6=1, - end=1, - sc1=33, - nowait=2, - nowait2=1, - kfast=6, - kv0e=6, - kv0drop=50, - ) - exp["P7"] = sum( - 1 - for i, ln in enumerate(lines) - if ln.strip() - in ("v_lshlrev_b32_e32 v16, 2, v16", "v_lshlrev_b32_e32 v10, 2, v10") - and i > 4000 - ) - assert done == exp, (done, exp) - text, pads = AL.align(src_text, "\n".join(out), clang, readelf) - return text, done, pads diff --git a/hsa/gfx950/mla_v4/nm_ps_src/asm/rt.s b/hsa/gfx950/mla_v4/nm_ps_src/asm/rt.s deleted file mode 100644 index 672bd13ec0c..00000000000 --- a/hsa/gfx950/mla_v4/nm_ps_src/asm/rt.s +++ /dev/null @@ -1,15413 +0,0 @@ -.amdgcn_target "amdgcn-amd-amdhsa--gfx950" -.amdhsa_code_object_version 5 -.text -.globl _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE -.protected _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE -.p2align 8 -.type _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE,@function -_ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE: - s_and_b32 s1, s1, 0xffff - s_mov_b32 s70, 0 - s_mov_b32 s73, 0x7060302 - s_mov_b32 s74, 0x5040100 - s_mov_b32 s78, 0 - s_load_dwordx2 s[8:9], s[0:1], 0x0 - s_load_dwordx2 s[12:13], s[0:1], 0x10 - s_load_dwordx2 s[16:17], s[0:1], 0x20 - s_load_dwordx2 s[20:21], s[0:1], 0x30 - s_load_dwordx2 s[28:29], s[0:1], 0x40 - s_load_dwordx2 s[24:25], s[0:1], 0x50 - s_load_dword s79, s[0:1], 0x70 - s_load_dword s80, s[0:1], 0x80 - s_load_dword s81, s[0:1], 0x90 - s_load_dword s86, s[0:1], 0xc0 - s_load_dwordx2 s[36:37], s[0:1], 0xd0 - s_load_dwordx2 s[40:41], s[0:1], 0xe0 - s_load_dword s83, s[0:1], 0xf0 - s_load_dwordx2 s[44:45], s[0:1], 0x100 - s_load_dwordx2 s[48:49], s[0:1], 0x110 - s_load_dwordx2 s[52:53], s[0:1], 0x120 - s_load_dwordx2 s[32:33], s[0:1], 0x130 - s_load_dword s99, s[0:1], 0x140 - v_lshrrev_b32_e32 v1, 10, v0 - v_lshrrev_b32_e32 v2, 10, v1 - v_and_b32_e32 v2, 0x3ff, v2 - v_and_b32_e32 v1, 0x3ff, v1 - v_and_b32_e32 v0, 0x3ff, v0 - v_lshrrev_b32_e32 v3, 6, v0 - v_and_b32_e32 v0, 63, v0 - s_mov_b32 s2, s2 - s_mov_b32 s3, s3 - s_mov_b32 s4, s4 - v_readfirstlane_b32 s7, v3 - s_waitcnt lgkmcnt(0) - s_mul_i32 s75, s3, 4 - s_and_b32 s29, s29, 0xffff - s_add_u32 s28, s75, s28 - s_addc_u32 s29, 0, s29 - s_load_dword s68, s[28:29], 0x0 - s_load_dword s69, s[28:29], 0x4 - s_mul_i32 s75, s3, 4 - s_and_b32 s41, s41, 0xffff - s_add_u32 s40, s75, s40 - s_addc_u32 s41, 0, s41 - s_and_b32 s37, s37, 0xffff - s_add_u32 s36, s75, s36 - s_addc_u32 s37, 0, s37 - s_load_dword s93, s[40:41], 0x0 - s_load_dword s92, s[40:41], 0x4 - s_load_dword s89, s[36:37], 0x0 - s_load_dword s90, s[36:37], 0x4 - s_waitcnt lgkmcnt(0) - s_sub_u32 s98, s92, s93 - s_cmp_le_u32 s98, s4 - s_cbranch_scc1 label_8EF4 - s_mov_b32 s82, s98 - s_sub_u32 s91, s90, s89 - s_cmp_eq_u32 s91, 0 - s_cbranch_scc1 label_8EF4 - s_mul_i32 s72, s91, s80 - s_mul_i32 s75, 0x800, s72 - s_mul_i32 s76, 0x400, s72 - s_cmp_eq_u32 s82, 1 - s_cselect_b32 s95, s76, s75 - s_mul_i32 s76, 0x200, s72 - s_mul_i32 s77, 0x80, s72 - s_mul_i32 s75, 4, s72 - s_mov_b32 s10, s95 - s_mov_b32 s18, s76 - s_mov_b32 s46, s77 - s_mov_b32 s14, s75 - s_mov_b32 s22, -16 - s_mov_b32 s50, -16 - s_mov_b32 s26, -16 - s_mov_b32 s11, 0x20000 - s_mov_b32 s19, 0x20000 - s_mov_b32 s47, 0x20000 - s_mov_b32 s15, 0x20000 - s_mov_b32 s23, 0x20000 - s_mov_b32 s51, 0x20000 - s_mov_b32 s27, 0x20000 - s_and_b32 s9, s9, 0xffff - s_and_b32 s17, s17, 0xffff - s_and_b32 s45, s45, 0xffff - s_and_b32 s13, s13, 0xffff - s_and_b32 s21, s21, 0xffff - s_and_b32 s49, s49, 0xffff - s_and_b32 s25, s25, 0xffff - s_or_b32 s9, s9, 0x40000 - s_or_b32 s17, s17, 0x40000 - s_or_b32 s45, s45, 0x40000 - s_or_b32 s13, s13, 0x40000 - s_or_b32 s21, s21, 0x40000 - s_or_b32 s49, s49, 0x40000 - s_or_b32 s25, s25, 0x40000 - s_and_b32 s53, s53, 0xffff - s_or_b32 s53, s53, 0x40000 - s_lshl_b32 s75, s80, 2 - s_mov_b32 s54, s75 - s_mov_b32 s55, 0x20000 - s_and_b32 s33, s33, 0xffff - s_or_b32 s33, s33, 0x40000 - s_mov_b32 s34, -16 - s_mov_b32 s35, 0x20000 - s_waitcnt lgkmcnt(0) - s_mov_b32 s86, 0 - s_lshr_b32 s67, 32, s86 - s_sub_u32 s71, s69, s68 - s_cmp_eq_u32 s71, 0 - s_cbranch_scc1 label_8EF4 - s_mul_i32 s100, s4, s67 - s_cmp_eq_u32 s99, 1 - s_cbranch_scc0 label_030C - s_sub_u32 s76, 5, s86 - s_sub_u32 s77, s67, 1 - s_add_u32 s77, s71, s77 - s_lshr_b32 s77, s77, s76 - s_cmp_lt_u32 s77, s82 - s_cselect_b32 s77, s77, s82 - s_cmp_eq_u32 s4, 0 - s_cbranch_scc0 label_030C - s_cmp_eq_u32 s2, 0 - s_cbranch_scc0 label_030C - s_cmp_eq_u32 s7, 0 - s_cbranch_scc0 label_030C - v_mov_b32_e32 v26, s77 - v_mov_b32_e32 v27, s3 - v_lshlrev_b32_e32 v27, 2, v27 - buffer_store_dword v26, v27, s[32:35], 0 offen -label_030C: - s_cmp_le_u32 s71, s100 - s_cbranch_scc1 label_8DAC - s_mov_b32 s87, 0 - s_sub_u32 s88, s71, s100 - s_mul_i32 s65, s82, s67 - s_mov_b32 s64, s88 - s_mov_b32 s66, 0 - s_mov_b32 s66, 0 - v_cvt_f32_u32_e32 v26, s65 - s_sub_i32 s75, 0, s65 - v_rcp_iflag_f32_e32 v26, v26 - s_nop 0 - v_mul_f32_e32 v26, 0x4f7ffffe, v26 - v_cvt_u32_f32_e32 v26, v26 - v_mul_lo_u32 v27, s75, v26 - v_mul_hi_u32 v27, v26, v27 - v_add_u32_e32 v26, v26, v27 - v_mul_hi_u32 v26, s64, v26 - v_mul_lo_u32 v27, v26, s65 - v_sub_u32_e32 v29, s64, v27 - v_add_u32_e32 v28, 1, v26 - v_cmp_le_u32_e32 vcc, s65, v29 - v_subrev_u32_e32 v27, s65, v29 - s_nop 0 - v_cndmask_b32_e32 v26, v26, v28, vcc - v_cndmask_b32_e32 v29, v29, v27, vcc - v_add_u32_e32 v27, 1, v26 - v_cmp_le_u32_e32 vcc, s65, v29 - s_nop 1 - v_cndmask_b32_e32 v29, v26, v27, vcc - s_nop 3 - v_readfirstlane_b32 s66, v29 - s_nop 3 - s_mov_b32 s88, s66 - s_mul_i32 s75, s88, s65 - s_sub_u32 s75, s64, s75 - s_mov_b32 s76, 0 - s_cmp_lt_u32 s75, s67 - s_cselect_b32 s76, s76, 1 - s_add_u32 s88, s76, s88 - s_cmpk_eq_u32 s76, 0x1 - s_cselect_b32 s70, 0, s75 - s_mul_i32 s94, s67, 4 - s_mul_i32 s94, s94, s82 - s_mul_i32 s75, s71, 4 - s_mov_b32 s26, s75 - s_mul_i32 s75, s68, 4 - s_add_u32 s24, s75, s24 - s_addc_u32 s25, 0, s25 - v_lshlrev_b32_e32 v26, 4, v0 - v_and_b32_e32 v27, 32, v26 - v_lshlrev_b32_e32 v26, 3, v0 - v_and_or_b32 v27, v26, 64, v27 - v_and_or_b32 v27, v0, 4, v27 - s_mul_i32 s75, 8, s7 - s_mul_i32 s76, 4, s100 - s_add_u32 s75, s76, s75 - v_add_u32_e32 v38, s75, v27 - v_bfe_u32 v44, v0, 1, 3 - v_lshlrev_b32_e32 v44, 2, v44 - s_mul_i32 s75, 32, s7 - v_add_u32_e32 v44, s75, v44 - s_mul_i32 s76, 4, s100 - v_add_u32_e32 v44, s76, v44 - buffer_load_dword v39, v38, s[24:27], 0 offen - buffer_load_dword v242, v44, s[24:27], 0 offen - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - buffer_load_dword v40, v38, s[24:27], 0 offen - buffer_load_dword v243, v44, s[24:27], 0 offen - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - s_cmp_eq_u32 s80, 16 - s_cselect_b32 s69, 16, 64 - s_cmp_eq_u32 s80, 32 - s_cselect_b32 s69, 32, s69 - s_cmp_eq_u32 s91, 1 - s_cselect_b32 s69, s69, 64 - s_lshr_b32 s72, s69, 4 - s_cmp_eq_u32 s70, 0 - s_cselect_b32 s72, s72, 4 - s_cmp_gt_u32 s88, 0 - s_cselect_b32 s72, s72, 4 - s_mul_i32 s75, s89, s80 - s_mul_i32 s75, s75, 0x200 - s_add_u32 s16, s75, s16 - s_addc_u32 s17, 0, s17 - s_mov_b32 s75, 0x8000 - s_mul_i32 s75, s75, s2 - s_mov_b32 s76, 0x200 - s_mul_i32 s76, s7, s76 - s_add_u32 s75, s75, s76 - v_and_b32_e32 v1, 31, v0 - v_lshlrev_b32_e32 v1, 4, v1 - v_bfe_u32 v26, v0, 5, 1 - v_lshlrev_b32_e32 v26, 11, v26 - v_add_u32_e32 v1, v26, v1 - v_add_u32_e32 v1, s75, v1 - s_mul_i32 s75, s7, 0x820 - s_add_u32 s56, 0x2000, s75 - s_add_u32 s57, 0x2080, s56 - s_add_u32 s58, 0x2080, s57 - s_add_u32 s59, 0x2080, s58 - s_mov_b32 m0, s56 - v_add_u32_e32 v2, 0, v1 - buffer_load_dwordx4 v2, s[16:19], 0 offen lds - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v2, 0x1000, v2 - buffer_load_dwordx4 v2, s[16:19], 0 offen lds - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v2, 0x1000, v2 - s_mov_b32 m0, s57 - v_add_u32_e32 v2, 0x2000, v1 - buffer_load_dwordx4 v2, s[16:19], 0 offen lds - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v2, 0x1000, v2 - buffer_load_dwordx4 v2, s[16:19], 0 offen lds - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v2, 0x1000, v2 - s_mov_b32 s6, 0x3fb8aa3b - v_mov_b32_e32 v27, s6 - v_mov_b32_e32 v26, s79 - v_mul_f32_e32 v26, s6, v26 - v_rcp_f32_e32 v27, v27 - v_mov_b32_e32 v45, 0xff7fffff - v_mov_b32_e32 v33, 0 - v_mov_b32_e32 v46, 0 - s_mov_b32 m0, s58 - v_add_u32_e32 v2, 0x4000, v1 - buffer_load_dwordx4 v2, s[16:19], 0 offen lds - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v2, 0x1000, v2 - buffer_load_dwordx4 v2, s[16:19], 0 offen lds - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v2, 0x1000, v2 - s_mov_b32 m0, s59 - v_add_u32_e32 v2, 0x6000, v1 - buffer_load_dwordx4 v2, s[16:19], 0 offen lds - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v2, 0x1000, v2 - buffer_load_dwordx4 v2, s[16:19], 0 offen lds - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v2, 0x1000, v2 - v_readfirstlane_b32 s5, v26 - v_readfirstlane_b32 s6, v27 - s_mul_i32 s75, s89, s80 - s_mul_i32 s75, s75, 0x80 - s_add_u32 s44, s75, s44 - s_addc_u32 s45, 0, s45 - s_mov_b32 s75, 0x2000 - s_mul_i32 s75, s75, s2 - v_bfe_u32 v26, v0, 1, 3 - v_lshlrev_b32_e32 v12, 7, v26 - v_lshrrev_b32_e32 v26, 4, v0 - v_lshl_add_u32 v12, v26, 5, v12 - v_and_b32_e32 v26, 1, v0 - v_lshl_add_u32 v12, v26, 4, v12 - v_add_u32_e32 v12, s75, v12 - s_add_u32 s60, 0x12800, 0 - s_add_u32 s61, 0x8c0, s60 - s_add_u32 s62, 0x8c0, s61 - s_add_u32 s63, 0x8c0, s62 - v_and_b32_e32 v26, 7, v0 - v_lshlrev_b32_e32 v26, 5, v26 - v_lshrrev_b32_e32 v29, 4, v0 - v_lshl_add_u32 v26, v29, 8, v26 - v_bfe_u32 v29, v0, 3, 1 - v_mul_i32_i24_e32 v29, 0x410, v29 - v_add_u32_e32 v16, v29, v26 - v_add_u32_e32 v16, 0x12800, v16 - s_mul_i32 s75, s7, 0x8c0 - s_add_u32 m0, 0x12800, s75 - s_mul_i32 s75, s7, 0x800 - v_add_u32_e32 v2, s75, v12 - buffer_load_dwordx4 v2, s[44:47], 0 offen lds - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v2, 0x400, v2 - buffer_load_dwordx4 v2, s[44:47], 0 offen lds - v_bfe_u32 v26, v0, 3, 1 - v_and_b32_e32 v27, 3, v0 - v_lshl_or_b32 v26, v27, 1, v26 - v_mul_i32_i24_e32 v27, 0x410, v26 - v_bfe_u32 v26, v0, 2, 1 - v_mul_i32_i24_e32 v26, 0x200, v26 - v_add_u32_e32 v27, v26, v27 - v_bfe_u32 v26, v0, 4, 2 - v_lshl_add_u32 v10, v26, 5, v27 - v_lshrrev_b32_e32 v28, 4, v0 - v_mul_i32_i24_e32 v29, 32, v28 - v_sub_u32_e32 v34, v10, v29 - v_add_u32_e32 v34, 0x1c0, v34 - v_lshlrev_b32_e32 v29, 1, v28 - v_lshrrev_b32_e32 v28, 1, v28 - v_or_b32_e32 v28, v28, v29 - v_and_b32_e32 v28, 3, v28 - v_add_u32_e32 v30, v28, v34 - v_sub_u32_e32 v25, v30, v34 - v_lshlrev_b32_e32 v25, 3, v25 - v_lshlrev_b32_e32 v43, 2, v0 - v_lshlrev_b32_e32 v17, 2, v0 - s_mul_i32 s75, 0x100, s7 - v_add_u32_e32 v17, s75, v17 - v_and_b32_e32 v26, 31, v0 - v_lshlrev_b32_e32 v27, 3, v26 - v_bfe_u32 v26, v0, 4, 1 - v_lshl_add_u32 v27, v26, 8, v27 - v_lshrrev_b32_e32 v26, 5, v0 - v_lshl_add_u32 v18, v26, 11, v27 - v_xor_b32_e32 v19, 0x80, v18 - v_lshlrev_b32_e32 v26, 2, v0 - v_and_b32_e32 v26, 0x80, v26 - s_mul_i32 s75, 0x200, s7 - v_lshl_add_u32 v20, v0, 3, s75 - v_xor_b32_e32 v20, v20, v26 - s_waitcnt vmcnt(2) - s_barrier - s_cmp_ge_u32 s7, s72 - s_cbranch_scc1 label_09EC - s_cmp_eq_i32 s7, 0 - s_cbranch_scc0 label_0854 - ds_read_b128 a[0:3], v10 offset:8192 - ds_read_b128 a[4:7], v10 offset:8208 - ds_read_b128 a[8:11], v10 offset:8320 - ds_read_b128 a[12:15], v10 offset:8336 - ds_read_b128 a[16:19], v10 offset:8448 - ds_read_b128 a[20:23], v10 offset:8464 - ds_read_b128 a[24:27], v10 offset:8576 - ds_read_b128 a[28:31], v10 offset:8592 - ds_read_b128 v[26:29], v34 offset:8192 - s_waitcnt lgkmcnt(0) - v_bfe_u32 v224, v26, v25, 8 - v_bfe_u32 v225, v27, v25, 8 - v_bfe_u32 v226, v28, v25, 8 - v_bfe_u32 v227, v29, v25, 8 - v_cmp_ge_u32_e64 vcc, v25, 16 - v_cndmask_b32_e64 v227, v227, 0, vcc - s_branch label_09EC -label_0854: - s_cmp_eq_i32 s7, 1 - s_cbranch_scc0 label_08DC - ds_read_b128 a[0:3], v10 offset:16640 - ds_read_b128 a[4:7], v10 offset:16656 - ds_read_b128 a[8:11], v10 offset:16512 - ds_read_b128 a[12:15], v10 offset:16528 - ds_read_b128 a[16:19], v10 offset:16768 - ds_read_b128 a[20:23], v10 offset:16784 - ds_read_b128 a[24:27], v10 offset:16896 - ds_read_b128 a[28:31], v10 offset:16912 - ds_read_b128 v[26:29], v34 offset:16512 - s_waitcnt lgkmcnt(0) - v_bfe_u32 v224, v27, v25, 8 - v_bfe_u32 v225, v26, v25, 8 - v_bfe_u32 v226, v28, v25, 8 - v_bfe_u32 v227, v29, v25, 8 - v_cmp_ge_u32_e64 vcc, v25, 16 - v_cndmask_b32_e64 v227, v227, 0, vcc - s_branch label_09EC -label_08DC: - s_cmp_eq_i32 s7, 2 - s_cbranch_scc0 label_0964 - ds_read_b128 a[0:3], v10 offset:24832 - ds_read_b128 a[4:7], v10 offset:24848 - ds_read_b128 a[8:11], v10 offset:24960 - ds_read_b128 a[12:15], v10 offset:24976 - ds_read_b128 a[16:19], v10 offset:25088 - ds_read_b128 a[20:23], v10 offset:25104 - ds_read_b128 a[24:27], v10 offset:25216 - ds_read_b128 a[28:31], v10 offset:25232 - ds_read_b128 v[26:29], v34 offset:24832 - s_waitcnt lgkmcnt(0) - v_bfe_u32 v224, v26, v25, 8 - v_bfe_u32 v225, v27, v25, 8 - v_bfe_u32 v226, v28, v25, 8 - v_bfe_u32 v227, v29, v25, 8 - v_cmp_ge_u32_e64 vcc, v25, 16 - v_cndmask_b32_e64 v227, v227, 0, vcc - s_branch label_09EC -label_0964: - s_cmp_eq_i32 s7, 3 - s_cbranch_scc0 label_09EC - ds_read_b128 a[0:3], v10 offset:33152 - ds_read_b128 a[4:7], v10 offset:33168 - ds_read_b128 a[8:11], v10 offset:33280 - ds_read_b128 a[12:15], v10 offset:33296 - ds_read_b128 a[16:19], v10 offset:33408 - ds_read_b128 a[20:23], v10 offset:33424 - ds_read_b128 a[24:27], v10 offset:33536 - ds_read_b128 a[28:31], v10 offset:33552 - ds_read_b128 v[26:29], v34 offset:33152 - s_waitcnt lgkmcnt(0) - v_bfe_u32 v224, v26, v25, 8 - v_bfe_u32 v225, v27, v25, 8 - v_bfe_u32 v226, v28, v25, 8 - v_bfe_u32 v227, v29, v25, 8 - v_cmp_ge_u32_e64 vcc, v25, 16 - v_cndmask_b32_e64 v227, v227, 0, vcc - s_branch label_09EC -label_09EC: - s_waitcnt vmcnt(0) - s_barrier - s_cmp_ge_u32 s7, s72 - s_cbranch_scc1 label_0A6C - s_cmp_eq_i32 s7, 0 - s_cbranch_scc0 label_0A18 - ds_read_b128 a[32:35], v16 - ds_read_b128 a[36:39], v16 offset:16 - s_branch label_0A6C -label_0A18: - s_cmp_eq_i32 s7, 1 - s_cbranch_scc0 label_0A34 - ds_read_b128 a[32:35], v16 offset:2240 - ds_read_b128 a[36:39], v16 offset:2256 - s_branch label_0A6C -label_0A34: - s_cmp_eq_i32 s7, 2 - s_cbranch_scc0 label_0A50 - ds_read_b128 a[32:35], v16 offset:4480 - ds_read_b128 a[36:39], v16 offset:4496 - s_branch label_0A6C -label_0A50: - s_cmp_eq_i32 s7, 3 - s_cbranch_scc0 label_0A6C - ds_read_b128 a[32:35], v16 offset:6720 - ds_read_b128 a[36:39], v16 offset:6736 - s_branch label_0A6C -label_0A6C: - v_bfe_u32 v26, v0, 2, 2 - v_lshlrev_b32_e32 v1, 7, v26 - v_lshrrev_b32_e32 v26, 4, v0 - v_lshl_add_u32 v1, v26, 5, v1 - v_and_b32_e32 v26, 1, v0 - v_lshl_add_u32 v1, v26, 4, v1 - s_mov_b32 s92, 0xcccccccc - s_mov_b32 s93, 0xcccccccc - s_mov_b32 s74, 0xff7fffff - s_mul_i32 s75, s7, 0x1040 - s_add_u32 s56, 0x2000, s75 - s_add_u32 s57, 0x4100, s56 - s_add_u32 s58, 0x4100, s57 - s_add_u32 s59, 0x4100, s58 - s_movk_i32 s73, 0x200 - s_and_b32 s75, s21, 0xffff - v_add_co_u32_e32 v14, vcc, s20, v1 - v_mov_b32_e32 v15, s75 - v_addc_co_u32_e64 v15, vcc, v15, 0, vcc - v_max_i32_e32 v39, 0, v39 - s_nop 1 - v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - s_mov_b32 m0, s56 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - buffer_load_dword v41, v38, s[24:27], 0 offen - buffer_load_dword v244, v44, s[24:27], 0 offen - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - s_waitcnt lgkmcnt(0) - s_barrier - s_cmp_ge_u32 s7, s72 - s_cbranch_scc1 label_0C34 - v_lshrrev_b32_e32 v32, 4, v0 - v_cmp_ge_u32_e64 vcc, v32, 2 - s_mov_b64 exec, vcc - v_accvgpr_write_b32 a24, 0 - v_accvgpr_write_b32 a25, 0 - v_accvgpr_write_b32 a26, 0 - v_accvgpr_write_b32 a27, 0 - v_accvgpr_write_b32 a28, 0 - v_accvgpr_write_b32 a29, 0 - v_accvgpr_write_b32 a30, 0 - v_accvgpr_write_b32 a31, 0 - s_mov_b64 exec, -1 -label_0C34: - v_and_b32_e32 v26, 48, v0 - v_lshlrev_b32_e32 v12, 1, v26 - v_and_b32_e32 v26, 1, v0 - v_lshl_or_b32 v12, v26, 4, v12 - s_mul_i32 s75, s7, 0x410 - s_add_u32 s60, 0x12800, s75 - s_add_u32 s61, 0x1040, s60 - s_add_u32 s62, 0x1040, s61 - s_add_u32 s63, 0x1040, s62 - s_movk_i32 s86, 0x80 - s_and_b32 s75, s49, 0xffff - v_add_co_u32_e32 v246, vcc, s48, v12 - v_mov_b32_e32 v247, s75 - v_addc_co_u32_e64 v247, vcc, v247, 0, vcc - v_max_i32_e32 v242, 0, v242 - v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] - s_mov_b32 m0, s60 - global_load_lds_dwordx4 v[2:3], off - v_lshrrev_b32_e32 v26, 4, v0 - v_lshlrev_b32_e32 v27, 8, v26 - v_and_b32_e32 v26, 7, v0 - v_lshlrev_b32_e32 v28, 5, v26 - v_add_u32_e32 v27, v28, v27 - v_bfe_u32 v26, v0, 3, 1 - v_mul_i32_i24_e32 v28, 0x410, v26 - v_add_u32_e32 v27, v28, v27 - v_add_u32_e32 v16, 0x12800, v27 - v_and_b32_e32 v26, 6, v0 - v_lshlrev_b32_e32 v26, 1, v26 - v_and_or_b32 v26, v0, 1, v26 - v_mul_i32_i24_e32 v27, 0x410, v26 - v_bfe_u32 v26, v0, 3, 1 - v_mul_i32_i24_e32 v26, 32, v26 - v_add_u32_e32 v27, v26, v27 - v_bfe_u32 v26, v0, 4, 2 - v_lshl_add_u32 v13, v26, 8, v27 - v_max_i32_e32 v40, 0, v40 - s_nop 1 - v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - s_mov_b32 m0, s57 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_lshrrev_b32_e32 v28, 4, v0 - v_mul_i32_i24_e32 v29, 0x100, v28 - v_sub_u32_e32 v1, v13, v29 - v_add_u32_e32 v1, 0x2c0, v1 - v_lshlrev_b32_e32 v29, 1, v28 - v_lshrrev_b32_e32 v28, 1, v28 - v_or_b32_e32 v28, v28, v29 - v_and_b32_e32 v28, 3, v28 - v_add_u32_e32 v30, v28, v1 - v_sub_u32_e32 v25, v30, v1 - v_lshlrev_b32_e32 v25, 3, v25 - s_mul_i32 s75, s7, 64 - s_lshl_b32 s76, s7, 2 - v_lshrrev_b32_e32 v32, 4, v0 - s_cmp_eq_i32 s7, 3 - s_cselect_b64 vcc, -1, 0 - v_and_b32_e32 v33, 32, v0 - v_lshlrev_b32_e32 v33, 4, v33 - v_sub_u32_e32 v241, v13, v33 - v_cndmask_b32_e32 v33, v13, v241, vcc - v_add_u32_e32 v10, s75, v33 - v_add3_u32 v238, s76, v1, v32 - s_xor_b32 s78, s75, 64 - s_cmp_lt_u32 s7, 2 - s_cselect_b32 s76, s75, 0 - s_cselect_b32 s78, s78, 64 - v_add_u32_e32 v239, s76, v13 - v_add_u32_e32 v240, s78, v13 - s_cmp_eq_i32 s7, 1 - s_cselect_b32 s78, 4, 0 - s_or_b32 s78, s78, 0xc0c0c00 - v_lshrrev_b32_e32 v12, 3, v25 - v_or_b32_e32 v12, s78, v12 - v_max_i32_e32 v243, 0, v243 - v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] - s_mov_b32 m0, s61 - global_load_lds_dwordx4 v[2:3], off - v_and_b32_e32 v26, 15, v0 - v_mul_i32_i24_e32 v26, 0x390, v26 - v_lshrrev_b32_e32 v27, 4, v0 - v_lshl_add_u32 v26, v27, 6, v26 - s_mul_i32 s75, 0x100, s7 - s_add_u32 s75, 0x17000, s75 - v_add_u32_e32 v11, s75, v26 - v_cmp_ge_u32_e64 vcc, v27, 2 - s_cmp_lg_u32 s7, 3 - s_nop 4 - s_cselect_b64 vcc, 0, vcc - v_and_b32_e32 v28, 31, v0 - v_lshlrev_b32_e32 v28, 4, v28 - v_add_u32_e32 v28, 0x1e200, v28 - v_cndmask_b32_e32 v11, v11, v28, vcc - v_bfe_u32 v26, v0, 2, 2 - v_bfe_u32 v27, v0, 4, 2 - v_and_b32_e32 v28, 3, v0 - v_lshlrev_b32_e32 v28, 3, v28 - v_lshl_add_u32 v29, v27, 3, v26 - v_mul_i32_i24_e32 v29, 0x390, v29 - v_add_u32_e32 v29, v28, v29 - s_mul_i32 s75, 0x100, s7 - s_add_u32 s75, 0x17000, s75 - v_add_u32_e32 v21, s75, v29 - v_add_u32_e32 v22, 32, v21 - v_mul_i32_i24_e32 v29, 0x410, v27 - v_add_u32_e32 v29, v28, v29 - v_lshl_add_u32 v29, v26, 5, v29 - v_add_u32_e32 v23, 0x12800, v29 - v_add_u32_e32 v24, 0x100, v23 - buffer_load_dword v42, v38, s[24:27], 0 offen - buffer_load_dword v245, v44, s[24:27], 0 offen - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mov_b64_e32 v[96:97], 0 - v_mov_b64_e32 v[98:99], 0 - v_mov_b64_e32 v[100:101], 0 - v_mov_b64_e32 v[102:103], 0 - v_mov_b64_e32 v[104:105], 0 - v_mov_b64_e32 v[106:107], 0 - v_mov_b64_e32 v[108:109], 0 - v_mov_b64_e32 v[110:111], 0 - v_mov_b64_e32 v[112:113], 0 - v_mov_b64_e32 v[114:115], 0 - v_mov_b64_e32 v[116:117], 0 - v_mov_b64_e32 v[118:119], 0 - v_mov_b64_e32 v[120:121], 0 - v_mov_b64_e32 v[122:123], 0 - v_mov_b64_e32 v[124:125], 0 - v_mov_b64_e32 v[126:127], 0 - s_cmp_le_u32 s72, 1 - s_cbranch_scc1 label_1068 - v_mov_b64_e32 v[128:129], 0 - v_mov_b64_e32 v[130:131], 0 - v_mov_b64_e32 v[132:133], 0 - v_mov_b64_e32 v[134:135], 0 - v_mov_b64_e32 v[136:137], 0 - v_mov_b64_e32 v[138:139], 0 - v_mov_b64_e32 v[140:141], 0 - v_mov_b64_e32 v[142:143], 0 - v_mov_b64_e32 v[144:145], 0 - v_mov_b64_e32 v[146:147], 0 - v_mov_b64_e32 v[148:149], 0 - v_mov_b64_e32 v[150:151], 0 - v_mov_b64_e32 v[152:153], 0 - v_mov_b64_e32 v[154:155], 0 - v_mov_b64_e32 v[156:157], 0 - v_mov_b64_e32 v[158:159], 0 - s_cmp_le_u32 s72, 2 - s_cbranch_scc1 label_1068 - v_mov_b64_e32 v[160:161], 0 - v_mov_b64_e32 v[162:163], 0 - v_mov_b64_e32 v[164:165], 0 - v_mov_b64_e32 v[166:167], 0 - v_mov_b64_e32 v[168:169], 0 - v_mov_b64_e32 v[170:171], 0 - v_mov_b64_e32 v[172:173], 0 - v_mov_b64_e32 v[174:175], 0 - v_mov_b64_e32 v[176:177], 0 - v_mov_b64_e32 v[178:179], 0 - v_mov_b64_e32 v[180:181], 0 - v_mov_b64_e32 v[182:183], 0 - v_mov_b64_e32 v[184:185], 0 - v_mov_b64_e32 v[186:187], 0 - v_mov_b64_e32 v[188:189], 0 - v_mov_b64_e32 v[190:191], 0 - v_mov_b64_e32 v[192:193], 0 - v_mov_b64_e32 v[194:195], 0 - v_mov_b64_e32 v[196:197], 0 - v_mov_b64_e32 v[198:199], 0 - v_mov_b64_e32 v[200:201], 0 - v_mov_b64_e32 v[202:203], 0 - v_mov_b64_e32 v[204:205], 0 - v_mov_b64_e32 v[206:207], 0 - v_mov_b64_e32 v[208:209], 0 - v_mov_b64_e32 v[210:211], 0 - v_mov_b64_e32 v[212:213], 0 - v_mov_b64_e32 v[214:215], 0 - v_mov_b64_e32 v[216:217], 0 - v_mov_b64_e32 v[218:219], 0 - v_mov_b64_e32 v[220:221], 0 - v_mov_b64_e32 v[222:223], 0 -label_1068: - s_waitcnt vmcnt(6) - s_barrier - s_cmp_ge_u32 s7, s72 - s_cbranch_scc1 label_1098 - ds_read_b128 a[104:107], v16 - ds_read_b128 a[108:111], v16 offset:16 - ds_read_b128 a[112:115], v16 offset:2080 - ds_read_b128 a[116:119], v16 offset:2096 -label_1098: - ds_read_b128 v[80:83], v10 offset:8192 - ds_read_b128 v[84:87], v10 offset:8208 - ds_read_b128 v[88:91], v10 offset:10272 - ds_read_b128 v[92:95], v10 offset:10288 - s_cmp_ge_u32 s7, s72 - s_cbranch_scc1 label_11FC - v_accvgpr_write_b32 a64, 0 - v_accvgpr_write_b32 a65, 0 - v_accvgpr_write_b32 a66, 0 - v_accvgpr_write_b32 a67, 0 - v_accvgpr_write_b32 a68, 0 - v_accvgpr_write_b32 a69, 0 - v_accvgpr_write_b32 a70, 0 - v_accvgpr_write_b32 a71, 0 - v_accvgpr_write_b32 a96, 0 - v_accvgpr_write_b32 a97, 0 - v_accvgpr_write_b32 a98, 0 - v_accvgpr_write_b32 a99, 0 - v_accvgpr_write_b32 a100, 0 - v_accvgpr_write_b32 a101, 0 - v_accvgpr_write_b32 a102, 0 - v_accvgpr_write_b32 a103, 0 - v_lshrrev_b32_e32 v32, 4, v0 - v_cmp_eq_u32_e32 vcc, 3, v32 - v_mov_b32_e32 v47, 0xff - v_cndmask_b32_e64 v47, v47, 0, vcc - ds_read_b128 a[40:43], v239 offset:8192 - ds_read_b128 a[44:47], v239 offset:8208 - ds_read_b128 a[72:75], v239 offset:10272 - ds_read_b128 a[76:79], v239 offset:10288 - ds_read_b128 a[48:51], v240 offset:8192 - ds_read_b128 a[52:55], v240 offset:8208 - ds_read_b128 a[80:83], v240 offset:10272 - ds_read_b128 a[84:87], v240 offset:10288 - ds_read_b128 a[56:59], v13 offset:8320 - ds_read_b128 a[60:63], v13 offset:8336 - ds_read_b128 a[88:91], v13 offset:10400 - ds_read_b128 a[92:95], v13 offset:10416 - ds_read_b128 a[64:67], v241 offset:8384 - ds_read_b128 a[68:71], v241 offset:8400 - ds_read_b128 a[96:99], v241 offset:10464 - ds_read_b128 a[100:103], v241 offset:10480 - ds_read_b128 v[228:231], v1 offset:8192 - ds_read_u8 v236, v238 offset:8192 - ds_read_b128 v[232:235], v1 offset:10272 - ds_read_u8 v237, v238 offset:10272 - s_branch label_120C -label_11FC: - ds_read_u8 v236, v238 offset:8192 - ds_read_u8 v237, v238 offset:10272 -label_120C: - s_cmp_eq_u32 s88, 0 - s_cbranch_scc1 label_1F14 - s_cmp_ge_u32 s7, s72 - s_cbranch_scc1 label_12E14 - s_cmp_lt_u32 s7, 2 - s_cbranch_scc0 label_18A0 - s_waitcnt lgkmcnt(0) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - buffer_load_dword v39, v38, s[24:27], 0 offen - buffer_load_dword v242, v44, s[24:27], 0 offen - v_max_i32_e32 v41, 0, v41 - v_bfe_u32 v230, v230, v25, 8 - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - s_mov_b32 m0, s58 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_cvt_scalef32_pk_bf16_fp8 v34, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - ds_write_b128 v11, v[34:37] - v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cvt_scalef32_pk_bf16_fp8 v34, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - ds_write_b128 v11, v[34:37] offset:16 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_cvt_scalef32_pk_bf16_fp8 v34, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v85, v33 op_sel:[1,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_write_b128 v11, v[34:37] offset:32 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_bfe_u32 v234, v234, v25, 8 - v_cvt_scalef32_pk_bf16_fp8 v34, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v87, v33 op_sel:[1,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - ds_write_b128 v11, v[34:37] offset:48 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_and_b32_e32 v235, v47, v235 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - v_cvt_scalef32_pk_bf16_fp8 v34, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v89, v33 op_sel:[1,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - ds_write_b128 v11, v[34:37] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v34, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - ds_write_b128 v11, v[34:37] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v34, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - ds_write_b128 v11, v[34:37] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v34, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v95, v33 op_sel:[1,0,0] - ds_write_b128 v11, v[34:37] offset:14640 - v_max_i32_e32 v244, 0, v244 - v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] - s_mov_b32 m0, s62 - global_load_lds_dwordx4 v[2:3], off - s_waitcnt vmcnt(5) - s_barrier - ds_read_b128 v[228:231], v1 offset:24832 - ds_read_u8 v236, v238 offset:24832 - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - ds_read_b128 v[232:235], v1 offset:26912 - ds_read_u8 v237, v238 offset:26912 - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - ds_read_b128 v[80:83], v10 offset:24832 - ds_read_b128 v[84:87], v10 offset:24848 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - ds_read_b128 v[88:91], v10 offset:26912 - ds_read_b128 v[92:95], v10 offset:26928 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - ds_read_b128 a[40:43], v239 offset:24832 - ds_read_b128 a[44:47], v239 offset:24848 - v_exp_f32_e32 v33, v33 - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - ds_read_b128 a[72:75], v239 offset:26912 - ds_read_b128 a[76:79], v239 offset:26928 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - ds_read_b128 a[48:51], v240 offset:24832 - ds_read_b128 a[52:55], v240 offset:24848 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - ds_read_b128 a[80:83], v240 offset:26912 - ds_read_b128 a[84:87], v240 offset:26928 - v_exp_f32_e32 v50, v50 - v_exp_f32_e32 v51, v51 - ds_read_b128 a[56:59], v13 offset:24960 - ds_read_b128 a[60:63], v13 offset:24976 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - ds_read_b128 a[88:91], v13 offset:27040 - ds_read_b128 a[92:95], v13 offset:27056 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - ds_read_b128 a[64:67], v241 offset:25024 - ds_read_b128 a[68:71], v241 offset:25040 - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - ds_read_b128 a[96:99], v241 offset:27104 - ds_read_b128 a[100:103], v241 offset:27120 - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_add_f32_e32 v26, v55, v26 - ds_read_b128 a[104:107], v16 offset:4160 - ds_read_b128 a[108:111], v16 offset:4176 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - ds_write_b32 v17, v33 offset:4096 - ds_write_b64 v20, v[48:49] - ds_write_b64 v20, v[50:51] offset:2048 - v_max_i32_e32 v42, 0, v42 - s_nop 1 - v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - ds_read_b128 a[112:115], v16 offset:6240 - ds_read_b128 a[116:119], v16 offset:6256 - buffer_load_dword v40, v38, s[24:27], 0 offen - buffer_load_dword v243, v44, s[24:27], 0 offen - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - s_mov_b32 m0, s59 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_max_i32_e32 v245, 0, v245 - v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] - s_mov_b32 m0, s63 - global_load_lds_dwordx4 v[2:3], off - s_branch label_1F1C -label_18A0: - s_waitcnt lgkmcnt(0) - s_barrier - v_bfe_u32 v228, v228, v25, 8 - v_bfe_u32 v229, v229, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - buffer_load_dword v39, v38, s[24:27], 0 offen - buffer_load_dword v242, v44, s[24:27], 0 offen - v_max_i32_e32 v41, 0, v41 - v_bfe_u32 v230, v230, v25, 8 - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - s_mov_b32 m0, s58 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_cvt_scalef32_pk_bf16_fp8 v34, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - ds_write_b128 v11, v[34:37] - v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cvt_scalef32_pk_bf16_fp8 v34, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - ds_write_b128 v11, v[34:37] offset:16 - v_bfe_u32 v232, v232, v25, 8 - v_bfe_u32 v233, v233, v25, 8 - v_cvt_scalef32_pk_bf16_fp8 v34, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v85, v33 op_sel:[1,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_write_b128 v11, v[34:37] offset:32 - v_bfe_u32 v234, v234, v25, 8 - v_cvt_scalef32_pk_bf16_fp8 v34, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v87, v33 op_sel:[1,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - ds_write_b128 v11, v[34:37] offset:48 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_and_b32_e32 v235, v47, v235 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_cvt_scalef32_pk_bf16_fp8 v34, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v89, v33 op_sel:[1,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - ds_write_b128 v11, v[34:37] offset:14592 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_cvt_scalef32_pk_bf16_fp8 v34, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - ds_write_b128 v11, v[34:37] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v34, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - ds_write_b128 v11, v[34:37] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v34, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v95, v33 op_sel:[1,0,0] - ds_write_b128 v11, v[34:37] offset:14640 - v_max_i32_e32 v244, 0, v244 - v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] - s_mov_b32 m0, s62 - global_load_lds_dwordx4 v[2:3], off - s_waitcnt vmcnt(5) - s_barrier - ds_read_b128 v[228:231], v1 offset:24832 - ds_read_u8 v236, v238 offset:24832 - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - ds_read_b128 v[232:235], v1 offset:26912 - ds_read_u8 v237, v238 offset:26912 - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - ds_read_b128 v[80:83], v10 offset:24832 - ds_read_b128 v[84:87], v10 offset:24848 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - ds_read_b128 v[88:91], v10 offset:26912 - ds_read_b128 v[92:95], v10 offset:26928 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - ds_read_b128 a[40:43], v239 offset:24832 - ds_read_b128 a[44:47], v239 offset:24848 - v_exp_f32_e32 v33, v33 - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - ds_read_b128 a[72:75], v239 offset:26912 - ds_read_b128 a[76:79], v239 offset:26928 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - ds_read_b128 a[48:51], v240 offset:24832 - ds_read_b128 a[52:55], v240 offset:24848 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - ds_read_b128 a[80:83], v240 offset:26912 - ds_read_b128 a[84:87], v240 offset:26928 - v_exp_f32_e32 v50, v50 - v_exp_f32_e32 v51, v51 - ds_read_b128 a[56:59], v13 offset:24960 - ds_read_b128 a[60:63], v13 offset:24976 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - ds_read_b128 a[88:91], v13 offset:27040 - ds_read_b128 a[92:95], v13 offset:27056 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - ds_read_b128 a[64:67], v241 offset:25024 - ds_read_b128 a[68:71], v241 offset:25040 - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - ds_read_b128 a[96:99], v241 offset:27104 - ds_read_b128 a[100:103], v241 offset:27120 - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_add_f32_e32 v26, v55, v26 - ds_read_b128 a[104:107], v16 offset:4160 - ds_read_b128 a[108:111], v16 offset:4176 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - ds_write_b32 v17, v33 offset:4096 - ds_write_b64 v20, v[48:49] - ds_write_b64 v20, v[50:51] offset:2048 - v_max_i32_e32 v42, 0, v42 - s_nop 1 - v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - ds_read_b128 a[112:115], v16 offset:6240 - ds_read_b128 a[116:119], v16 offset:6256 - buffer_load_dword v40, v38, s[24:27], 0 offen - buffer_load_dword v243, v44, s[24:27], 0 offen - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - s_mov_b32 m0, s59 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_max_i32_e32 v245, 0, v245 - v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] - s_mov_b32 m0, s63 - global_load_lds_dwordx4 v[2:3], off - s_branch label_1F1C -label_1F14: - s_mov_b32 s72, s69 - s_branch label_8EFC -label_1F1C: - s_mov_b32 s72, s69 - s_cmp_eq_u32 s88, 1 - s_cbranch_scc1 label_795C - s_mov_b32 s87, 1 - s_cmp_eq_u32 s72, 64 - s_cbranch_scc1 label_1F44 - s_cmp_eq_u32 s72, 16 - s_cbranch_scc1 label_9FB8 - s_cmp_eq_u32 s72, 32 - s_cbranch_scc1 label_EDDC -label_1F44: - s_cmp_eq_i32 s7, 2 - s_cbranch_scc1 label_3D74 - s_cmp_eq_i32 s7, 3 - s_cbranch_scc1 label_5B68 -label_1F54: - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b32 v36, v43 offset:4608 - ds_read_b32 v37, v43 offset:4864 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v39, 0, v39 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s56 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[50:51], v19 - v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] - buffer_load_dword v41, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] - buffer_load_dword v244, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[52:53], v18 offset:512 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64 v[54:55], v19 offset:512 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64 v[56:57], v18 offset:1024 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - ds_read_b64 v[58:59], v19 offset:1024 - ds_read_b64 v[60:61], v18 offset:1536 - v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_min_f32_e32 v26, v26, v36 - v_min_f32_e32 v26, v26, v37 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] - ds_read_b64 v[62:63], v19 offset:1536 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] - s_cbranch_vccz label_221C - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9DB4 -label_221C: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v64, v65, v64 - v_max3_f32 v30, v66, v67, v30 - v_max3_f32 v30, v68, v69, v30 - v_max3_f32 v30, v70, v71, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v242, 0, v242 - v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - s_mov_b32 m0, s60 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_waitcnt vmcnt(12) - s_barrier - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:41472 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] - ds_read_b128 a[48:51], v240 offset:41472 - ds_read_b128 a[52:55], v240 offset:41488 - ds_read_u8 v236, v238 offset:41472 - v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] - ds_read_b128 a[80:83], v240 offset:43552 - ds_read_b128 a[84:87], v240 offset:43568 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:43552 - v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] - v_fma_f32 v64, v64, s5, -v32 - v_fma_f32 v65, v65, s5, -v32 - v_fma_f32 v66, v66, s5, -v32 - v_fma_f32 v67, v67, s5, -v32 - v_fma_f32 v68, v68, s5, -v32 - v_fma_f32 v69, v69, s5, -v32 - v_fma_f32 v70, v70, s5, -v32 - v_fma_f32 v71, v71, s5, -v32 - v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] - ds_read_b128 a[56:59], v13 offset:41600 - ds_read_b128 a[60:63], v13 offset:41616 - v_exp_f32_e32 v64, v64 - v_exp_f32_e32 v65, v65 - v_exp_f32_e32 v66, v66 - ds_read_u8 v237, v238 offset:43552 - v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] - v_exp_f32_e32 v67, v67 - v_exp_f32_e32 v68, v68 - v_exp_f32_e32 v69, v69 - v_exp_f32_e32 v70, v70 - v_exp_f32_e32 v71, v71 - v_mul_f32_e32 v46, v33, v46 - v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] - ds_read_b128 a[88:91], v13 offset:43680 - ds_read_b128 a[92:95], v13 offset:43696 - v_add_f32_e32 v26, v65, v64 - v_add_f32_e32 v26, v66, v26 - v_add_f32_e32 v26, v67, v26 - v_add_f32_e32 v26, v68, v26 - v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] - v_add_f32_e32 v26, v69, v26 - v_add_f32_e32 v26, v70, v26 - v_add_f32_e32 v26, v71, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v64, v64, v65 - v_cvt_pk_bf16_f32 v65, v66, v67 - v_cvt_pk_bf16_f32 v66, v68, v69 - v_cvt_pk_bf16_f32 v67, v70, v71 - v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] - ds_read_b128 a[64:67], v241 offset:41664 - ds_read_b128 a[68:71], v241 offset:41680 - ds_write_b32 v17, v33 offset:4096 - v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] - v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] - ds_read_b128 a[96:99], v241 offset:43744 - ds_read_b128 a[100:103], v241 offset:43760 - ds_write_b64 v20, v[64:65] - v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] - v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] - ds_read_b128 a[104:107], v16 offset:8320 - ds_read_b128 a[108:111], v16 offset:8336 - ds_write_b64 v20, v[66:67] offset:2048 - v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] - v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] - ds_read_b128 a[112:115], v16 offset:10400 - ds_read_b128 a[116:119], v16 offset:10416 - v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b32 v36, v43 offset:4608 - ds_read_b32 v37, v43 offset:4864 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v40, 0, v40 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s57 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[66:67], v19 - v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - buffer_load_dword v42, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - buffer_load_dword v245, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[68:69], v18 offset:512 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64 v[70:71], v19 offset:512 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64 v[72:73], v18 offset:1024 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - ds_read_b64 v[74:75], v19 offset:1024 - ds_read_b64 v[76:77], v18 offset:1536 - v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_min_f32_e32 v26, v26, v36 - v_min_f32_e32 v26, v26, v37 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - ds_read_b64 v[78:79], v19 offset:1536 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - s_cbranch_vccz label_29A0 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9DB4 -label_29A0: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v243, 0, v243 - v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] - s_mov_b32 m0, s61 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] - s_waitcnt vmcnt(12) - s_barrier - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:58112 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[72:75], v[160:163] - ds_read_b128 a[48:51], v240 offset:58112 - ds_read_b128 a[52:55], v240 offset:58128 - ds_read_u8 v236, v238 offset:58112 - v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[72:75], v[164:167] - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[72:75], v[168:171] - ds_read_b128 a[80:83], v240 offset:60192 - ds_read_b128 a[84:87], v240 offset:60208 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:60192 - v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[72:75], v[172:175] - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[72:75], v[176:179] - ds_read_b128 a[56:59], v13 offset:58240 - ds_read_b128 a[60:63], v13 offset:58256 - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - v_exp_f32_e32 v50, v50 - ds_read_u8 v237, v238 offset:60192 - v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[72:75], v[180:183] - v_exp_f32_e32 v51, v51 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[72:75], v[184:187] - ds_read_b128 a[88:91], v13 offset:60320 - ds_read_b128 a[92:95], v13 offset:60336 - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[72:75], v[188:191] - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_add_f32_e32 v26, v55, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[76:79], v[192:195] - ds_read_b128 a[64:67], v241 offset:58304 - ds_read_b128 a[68:71], v241 offset:58320 - ds_write_b32 v17, v33 offset:4096 - v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[76:79], v[196:199] - v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[76:79], v[200:203] - ds_read_b128 a[96:99], v241 offset:60384 - ds_read_b128 a[100:103], v241 offset:60400 - ds_write_b64 v20, v[48:49] - v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[76:79], v[204:207] - v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[76:79], v[208:211] - ds_read_b128 a[104:107], v16 offset:12480 - ds_read_b128 a[108:111], v16 offset:12496 - ds_write_b64 v20, v[50:51] offset:2048 - v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[76:79], v[212:215] - v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[76:79], v[216:219] - ds_read_b128 a[112:115], v16 offset:14560 - ds_read_b128 a[116:119], v16 offset:14576 - v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[76:79], v[220:223] - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b32 v36, v43 offset:4608 - ds_read_b32 v37, v43 offset:4864 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v41, 0, v41 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s58 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[50:51], v19 - v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] - buffer_load_dword v39, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] - buffer_load_dword v242, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[52:53], v18 offset:512 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64 v[54:55], v19 offset:512 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64 v[56:57], v18 offset:1024 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - ds_read_b64 v[58:59], v19 offset:1024 - ds_read_b64 v[60:61], v18 offset:1536 - v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_min_f32_e32 v26, v26, v36 - v_min_f32_e32 v26, v26, v37 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] - ds_read_b64 v[62:63], v19 offset:1536 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] - s_cbranch_vccz label_3124 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9DB4 -label_3124: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v64, v65, v64 - v_max3_f32 v30, v66, v67, v30 - v_max3_f32 v30, v68, v69, v30 - v_max3_f32 v30, v70, v71, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v244, 0, v244 - v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - s_mov_b32 m0, s62 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_waitcnt vmcnt(12) - s_barrier - ds_read_b128 v[80:83], v10 offset:8192 - ds_read_b128 v[84:87], v10 offset:8208 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - ds_read_b128 v[88:91], v10 offset:10272 - ds_read_b128 v[92:95], v10 offset:10288 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:8192 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] - ds_read_b128 a[48:51], v240 offset:8192 - ds_read_b128 a[52:55], v240 offset:8208 - ds_read_u8 v236, v238 offset:8192 - v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] - ds_read_b128 a[80:83], v240 offset:10272 - ds_read_b128 a[84:87], v240 offset:10288 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:10272 - v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] - v_fma_f32 v64, v64, s5, -v32 - v_fma_f32 v65, v65, s5, -v32 - v_fma_f32 v66, v66, s5, -v32 - v_fma_f32 v67, v67, s5, -v32 - v_fma_f32 v68, v68, s5, -v32 - v_fma_f32 v69, v69, s5, -v32 - v_fma_f32 v70, v70, s5, -v32 - v_fma_f32 v71, v71, s5, -v32 - v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] - ds_read_b128 a[56:59], v13 offset:8320 - ds_read_b128 a[60:63], v13 offset:8336 - v_exp_f32_e32 v64, v64 - v_exp_f32_e32 v65, v65 - v_exp_f32_e32 v66, v66 - ds_read_u8 v237, v238 offset:10272 - v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] - v_exp_f32_e32 v67, v67 - v_exp_f32_e32 v68, v68 - v_exp_f32_e32 v69, v69 - v_exp_f32_e32 v70, v70 - v_exp_f32_e32 v71, v71 - v_mul_f32_e32 v46, v33, v46 - v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] - ds_read_b128 a[88:91], v13 offset:10400 - ds_read_b128 a[92:95], v13 offset:10416 - v_add_f32_e32 v26, v65, v64 - v_add_f32_e32 v26, v66, v26 - v_add_f32_e32 v26, v67, v26 - v_add_f32_e32 v26, v68, v26 - v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] - v_add_f32_e32 v26, v69, v26 - v_add_f32_e32 v26, v70, v26 - v_add_f32_e32 v26, v71, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v64, v64, v65 - v_cvt_pk_bf16_f32 v65, v66, v67 - v_cvt_pk_bf16_f32 v66, v68, v69 - v_cvt_pk_bf16_f32 v67, v70, v71 - v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] - ds_read_b128 a[64:67], v241 offset:8384 - ds_read_b128 a[68:71], v241 offset:8400 - ds_write_b32 v17, v33 offset:4096 - v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] - v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] - ds_read_b128 a[96:99], v241 offset:10464 - ds_read_b128 a[100:103], v241 offset:10480 - ds_write_b64 v20, v[64:65] - v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] - v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] - ds_read_b128 a[104:107], v16 - ds_read_b128 a[108:111], v16 offset:16 - ds_write_b64 v20, v[66:67] offset:2048 - v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] - v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] - ds_read_b128 a[112:115], v16 offset:2080 - ds_read_b128 a[116:119], v16 offset:2096 - v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b32 v36, v43 offset:4608 - ds_read_b32 v37, v43 offset:4864 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v42, 0, v42 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s59 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[66:67], v19 - v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - buffer_load_dword v40, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - buffer_load_dword v243, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[68:69], v18 offset:512 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64 v[70:71], v19 offset:512 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64 v[72:73], v18 offset:1024 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - ds_read_b64 v[74:75], v19 offset:1024 - ds_read_b64 v[76:77], v18 offset:1536 - v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_min_f32_e32 v26, v26, v36 - v_min_f32_e32 v26, v26, v37 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - ds_read_b64 v[78:79], v19 offset:1536 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - s_cbranch_vccz label_38A8 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9DB4 -label_38A8: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v245, 0, v245 - v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] - s_mov_b32 m0, s63 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] - s_waitcnt vmcnt(12) - s_barrier - ds_read_b128 v[80:83], v10 offset:24832 - ds_read_b128 v[84:87], v10 offset:24848 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] - ds_read_b128 v[88:91], v10 offset:26912 - ds_read_b128 v[92:95], v10 offset:26928 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:24832 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[72:75], v[160:163] - ds_read_b128 a[48:51], v240 offset:24832 - ds_read_b128 a[52:55], v240 offset:24848 - ds_read_u8 v236, v238 offset:24832 - v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[72:75], v[164:167] - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[72:75], v[168:171] - ds_read_b128 a[80:83], v240 offset:26912 - ds_read_b128 a[84:87], v240 offset:26928 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:26912 - v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[72:75], v[172:175] - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[72:75], v[176:179] - ds_read_b128 a[56:59], v13 offset:24960 - ds_read_b128 a[60:63], v13 offset:24976 - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - v_exp_f32_e32 v50, v50 - ds_read_u8 v237, v238 offset:26912 - v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[72:75], v[180:183] - v_exp_f32_e32 v51, v51 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[72:75], v[184:187] - ds_read_b128 a[88:91], v13 offset:27040 - ds_read_b128 a[92:95], v13 offset:27056 - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[72:75], v[188:191] - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_add_f32_e32 v26, v55, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[76:79], v[192:195] - ds_read_b128 a[64:67], v241 offset:25024 - ds_read_b128 a[68:71], v241 offset:25040 - ds_write_b32 v17, v33 offset:4096 - v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[76:79], v[196:199] - v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[76:79], v[200:203] - ds_read_b128 a[96:99], v241 offset:27104 - ds_read_b128 a[100:103], v241 offset:27120 - ds_write_b64 v20, v[48:49] - v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[76:79], v[204:207] - v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[76:79], v[208:211] - ds_read_b128 a[104:107], v16 offset:4160 - ds_read_b128 a[108:111], v16 offset:4176 - ds_write_b64 v20, v[50:51] offset:2048 - v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[76:79], v[212:215] - v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[76:79], v[216:219] - ds_read_b128 a[112:115], v16 offset:6240 - ds_read_b128 a[116:119], v16 offset:6256 - v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[76:79], v[220:223] - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_branch label_1F54 -label_3D68: - s_nop 0 - s_nop 0 - s_branch label_795C -label_3D74: - s_waitcnt lgkmcnt(2) - s_barrier - v_bfe_u32 v228, v228, v25, 8 - v_bfe_u32 v229, v229, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b32 v36, v43 offset:4608 - ds_read_b32 v37, v43 offset:4864 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v39, 0, v39 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s56 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] - buffer_load_dword v41, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] - buffer_load_dword v244, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[52:53], v18 offset:512 - v_bfe_u32 v232, v232, v25, 8 - v_bfe_u32 v233, v233, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64 v[54:55], v19 offset:512 - v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64 v[56:57], v18 offset:1024 - ds_read_b64 v[58:59], v19 offset:1024 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - ds_read_b64 v[60:61], v18 offset:1536 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_min_f32_e32 v26, v26, v36 - v_min_f32_e32 v26, v26, v37 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] - ds_read_b64 v[62:63], v19 offset:1536 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_cbranch_vccz label_4034 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9DB4 -label_4034: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - v_max3_f32 v30, v64, v65, v64 - v_max3_f32 v30, v66, v67, v30 - v_max3_f32 v30, v68, v69, v30 - v_max3_f32 v30, v70, v71, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v242, 0, v242 - v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - s_mov_b32 m0, s60 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_waitcnt vmcnt(12) - s_barrier - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - ds_read_b128 v[228:231], v1 offset:41472 - v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] - ds_read_b128 a[40:43], v239 offset:41472 - ds_read_b128 a[44:47], v239 offset:41488 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_u8 v236, v238 offset:41472 - v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] - v_fma_f32 v64, v64, s5, -v32 - v_fma_f32 v65, v65, s5, -v32 - v_fma_f32 v66, v66, s5, -v32 - v_fma_f32 v67, v67, s5, -v32 - v_fma_f32 v68, v68, s5, -v32 - v_fma_f32 v69, v69, s5, -v32 - v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] - ds_read_b128 a[72:75], v239 offset:43552 - ds_read_b128 a[76:79], v239 offset:43568 - v_fma_f32 v70, v70, s5, -v32 - v_fma_f32 v71, v71, s5, -v32 - ds_read_b128 v[232:235], v1 offset:43552 - v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] - v_exp_f32_e32 v64, v64 - v_exp_f32_e32 v65, v65 - v_exp_f32_e32 v66, v66 - v_exp_f32_e32 v67, v67 - v_exp_f32_e32 v68, v68 - v_exp_f32_e32 v69, v69 - v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] - ds_read_b128 a[48:51], v240 offset:41472 - ds_read_b128 a[52:55], v240 offset:41488 - v_exp_f32_e32 v70, v70 - v_exp_f32_e32 v71, v71 - v_mul_f32_e32 v46, v33, v46 - ds_read_u8 v237, v238 offset:43552 - v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] - v_add_f32_e32 v26, v65, v64 - v_add_f32_e32 v26, v66, v26 - v_add_f32_e32 v26, v67, v26 - v_add_f32_e32 v26, v68, v26 - v_add_f32_e32 v26, v69, v26 - v_add_f32_e32 v26, v70, v26 - v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] - ds_read_b128 a[80:83], v240 offset:43552 - ds_read_b128 a[84:87], v240 offset:43568 - v_add_f32_e32 v26, v71, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v64, v64, v65 - v_cvt_pk_bf16_f32 v65, v66, v67 - v_cvt_pk_bf16_f32 v66, v68, v69 - v_cvt_pk_bf16_f32 v67, v70, v71 - v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] - v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] - ds_read_b128 a[64:67], v241 offset:41664 - ds_read_b128 a[68:71], v241 offset:41680 - ds_write_b32 v17, v33 offset:4096 - v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] - v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] - ds_read_b128 a[96:99], v241 offset:43744 - ds_read_b128 a[100:103], v241 offset:43760 - ds_write_b64 v20, v[64:65] - v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] - v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] - ds_read_b128 a[104:107], v16 offset:8320 - ds_read_b128 a[108:111], v16 offset:8336 - ds_write_b64 v20, v[66:67] offset:2048 - v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] - v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] - ds_read_b128 a[112:115], v16 offset:10400 - ds_read_b128 a[116:119], v16 offset:10416 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_bfe_u32 v228, v228, v25, 8 - v_bfe_u32 v229, v229, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b32 v36, v43 offset:4608 - ds_read_b32 v37, v43 offset:4864 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v40, 0, v40 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s57 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - buffer_load_dword v42, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - buffer_load_dword v245, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[68:69], v18 offset:512 - v_bfe_u32 v232, v232, v25, 8 - v_bfe_u32 v233, v233, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64 v[70:71], v19 offset:512 - v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64 v[72:73], v18 offset:1024 - ds_read_b64 v[74:75], v19 offset:1024 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - ds_read_b64 v[76:77], v18 offset:1536 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_min_f32_e32 v26, v26, v36 - v_min_f32_e32 v26, v26, v37 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - ds_read_b64 v[78:79], v19 offset:1536 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_cbranch_vccz label_47B0 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9DB4 -label_47B0: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v243, 0, v243 - v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] - s_mov_b32 m0, s61 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] - s_waitcnt vmcnt(12) - s_barrier - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - ds_read_b128 v[228:231], v1 offset:58112 - v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[72:75], v[160:163] - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[72:75], v[164:167] - ds_read_b128 a[40:43], v239 offset:58112 - ds_read_b128 a[44:47], v239 offset:58128 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_u8 v236, v238 offset:58112 - v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[72:75], v[168:171] - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[72:75], v[172:175] - ds_read_b128 a[72:75], v239 offset:60192 - ds_read_b128 a[76:79], v239 offset:60208 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - ds_read_b128 v[232:235], v1 offset:60192 - v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[72:75], v[176:179] - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - v_exp_f32_e32 v50, v50 - v_exp_f32_e32 v51, v51 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[72:75], v[180:183] - ds_read_b128 a[48:51], v240 offset:58112 - ds_read_b128 a[52:55], v240 offset:58128 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - ds_read_u8 v237, v238 offset:60192 - v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[72:75], v[184:187] - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[72:75], v[188:191] - ds_read_b128 a[80:83], v240 offset:60192 - ds_read_b128 a[84:87], v240 offset:60208 - v_add_f32_e32 v26, v55, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[76:79], v[192:195] - v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[76:79], v[196:199] - ds_read_b128 a[64:67], v241 offset:58304 - ds_read_b128 a[68:71], v241 offset:58320 - ds_write_b32 v17, v33 offset:4096 - v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[76:79], v[200:203] - v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[76:79], v[204:207] - ds_read_b128 a[96:99], v241 offset:60384 - ds_read_b128 a[100:103], v241 offset:60400 - ds_write_b64 v20, v[48:49] - v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[76:79], v[208:211] - v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[76:79], v[212:215] - ds_read_b128 a[104:107], v16 offset:12480 - ds_read_b128 a[108:111], v16 offset:12496 - ds_write_b64 v20, v[50:51] offset:2048 - v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[76:79], v[216:219] - v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[76:79], v[220:223] - ds_read_b128 a[112:115], v16 offset:14560 - ds_read_b128 a[116:119], v16 offset:14576 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_bfe_u32 v228, v228, v25, 8 - v_bfe_u32 v229, v229, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b32 v36, v43 offset:4608 - ds_read_b32 v37, v43 offset:4864 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v41, 0, v41 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s58 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] - buffer_load_dword v39, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] - buffer_load_dword v242, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[52:53], v18 offset:512 - v_bfe_u32 v232, v232, v25, 8 - v_bfe_u32 v233, v233, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64 v[54:55], v19 offset:512 - v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64 v[56:57], v18 offset:1024 - ds_read_b64 v[58:59], v19 offset:1024 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - ds_read_b64 v[60:61], v18 offset:1536 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_min_f32_e32 v26, v26, v36 - v_min_f32_e32 v26, v26, v37 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] - ds_read_b64 v[62:63], v19 offset:1536 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_cbranch_vccz label_4F2C - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9DB4 -label_4F2C: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - v_max3_f32 v30, v64, v65, v64 - v_max3_f32 v30, v66, v67, v30 - v_max3_f32 v30, v68, v69, v30 - v_max3_f32 v30, v70, v71, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v244, 0, v244 - v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - s_mov_b32 m0, s62 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_waitcnt vmcnt(12) - s_barrier - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - ds_read_b128 v[80:83], v10 offset:8192 - ds_read_b128 v[84:87], v10 offset:8208 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - ds_read_b128 v[88:91], v10 offset:10272 - ds_read_b128 v[92:95], v10 offset:10288 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - ds_read_b128 v[228:231], v1 offset:8192 - v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] - ds_read_b128 a[40:43], v239 offset:8192 - ds_read_b128 a[44:47], v239 offset:8208 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_u8 v236, v238 offset:8192 - v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] - v_fma_f32 v64, v64, s5, -v32 - v_fma_f32 v65, v65, s5, -v32 - v_fma_f32 v66, v66, s5, -v32 - v_fma_f32 v67, v67, s5, -v32 - v_fma_f32 v68, v68, s5, -v32 - v_fma_f32 v69, v69, s5, -v32 - v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] - ds_read_b128 a[72:75], v239 offset:10272 - ds_read_b128 a[76:79], v239 offset:10288 - v_fma_f32 v70, v70, s5, -v32 - v_fma_f32 v71, v71, s5, -v32 - ds_read_b128 v[232:235], v1 offset:10272 - v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] - v_exp_f32_e32 v64, v64 - v_exp_f32_e32 v65, v65 - v_exp_f32_e32 v66, v66 - v_exp_f32_e32 v67, v67 - v_exp_f32_e32 v68, v68 - v_exp_f32_e32 v69, v69 - v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] - ds_read_b128 a[48:51], v240 offset:8192 - ds_read_b128 a[52:55], v240 offset:8208 - v_exp_f32_e32 v70, v70 - v_exp_f32_e32 v71, v71 - v_mul_f32_e32 v46, v33, v46 - ds_read_u8 v237, v238 offset:10272 - v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] - v_add_f32_e32 v26, v65, v64 - v_add_f32_e32 v26, v66, v26 - v_add_f32_e32 v26, v67, v26 - v_add_f32_e32 v26, v68, v26 - v_add_f32_e32 v26, v69, v26 - v_add_f32_e32 v26, v70, v26 - v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] - ds_read_b128 a[80:83], v240 offset:10272 - ds_read_b128 a[84:87], v240 offset:10288 - v_add_f32_e32 v26, v71, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v64, v64, v65 - v_cvt_pk_bf16_f32 v65, v66, v67 - v_cvt_pk_bf16_f32 v66, v68, v69 - v_cvt_pk_bf16_f32 v67, v70, v71 - v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] - v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] - ds_read_b128 a[64:67], v241 offset:8384 - ds_read_b128 a[68:71], v241 offset:8400 - ds_write_b32 v17, v33 offset:4096 - v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] - v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] - ds_read_b128 a[96:99], v241 offset:10464 - ds_read_b128 a[100:103], v241 offset:10480 - ds_write_b64 v20, v[64:65] - v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] - v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] - ds_read_b128 a[104:107], v16 - ds_read_b128 a[108:111], v16 offset:16 - ds_write_b64 v20, v[66:67] offset:2048 - v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] - v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] - ds_read_b128 a[112:115], v16 offset:2080 - ds_read_b128 a[116:119], v16 offset:2096 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_bfe_u32 v228, v228, v25, 8 - v_bfe_u32 v229, v229, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b32 v36, v43 offset:4608 - ds_read_b32 v37, v43 offset:4864 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v42, 0, v42 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s59 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - buffer_load_dword v40, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - buffer_load_dword v243, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[68:69], v18 offset:512 - v_bfe_u32 v232, v232, v25, 8 - v_bfe_u32 v233, v233, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64 v[70:71], v19 offset:512 - v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64 v[72:73], v18 offset:1024 - ds_read_b64 v[74:75], v19 offset:1024 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - ds_read_b64 v[76:77], v18 offset:1536 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_min_f32_e32 v26, v26, v36 - v_min_f32_e32 v26, v26, v37 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - ds_read_b64 v[78:79], v19 offset:1536 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_cbranch_vccz label_56A8 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9DB4 -label_56A8: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v245, 0, v245 - v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] - s_mov_b32 m0, s63 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] - s_waitcnt vmcnt(12) - s_barrier - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] - ds_read_b128 v[80:83], v10 offset:24832 - ds_read_b128 v[84:87], v10 offset:24848 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] - ds_read_b128 v[88:91], v10 offset:26912 - ds_read_b128 v[92:95], v10 offset:26928 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - ds_read_b128 v[228:231], v1 offset:24832 - v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[72:75], v[160:163] - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[72:75], v[164:167] - ds_read_b128 a[40:43], v239 offset:24832 - ds_read_b128 a[44:47], v239 offset:24848 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_u8 v236, v238 offset:24832 - v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[72:75], v[168:171] - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[72:75], v[172:175] - ds_read_b128 a[72:75], v239 offset:26912 - ds_read_b128 a[76:79], v239 offset:26928 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - ds_read_b128 v[232:235], v1 offset:26912 - v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[72:75], v[176:179] - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - v_exp_f32_e32 v50, v50 - v_exp_f32_e32 v51, v51 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[72:75], v[180:183] - ds_read_b128 a[48:51], v240 offset:24832 - ds_read_b128 a[52:55], v240 offset:24848 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - ds_read_u8 v237, v238 offset:26912 - v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[72:75], v[184:187] - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[72:75], v[188:191] - ds_read_b128 a[80:83], v240 offset:26912 - ds_read_b128 a[84:87], v240 offset:26928 - v_add_f32_e32 v26, v55, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[76:79], v[192:195] - v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[76:79], v[196:199] - ds_read_b128 a[64:67], v241 offset:25024 - ds_read_b128 a[68:71], v241 offset:25040 - ds_write_b32 v17, v33 offset:4096 - v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[76:79], v[200:203] - v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[76:79], v[204:207] - ds_read_b128 a[96:99], v241 offset:27104 - ds_read_b128 a[100:103], v241 offset:27120 - ds_write_b64 v20, v[48:49] - v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[76:79], v[208:211] - v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[76:79], v[212:215] - ds_read_b128 a[104:107], v16 offset:4160 - ds_read_b128 a[108:111], v16 offset:4176 - ds_write_b64 v20, v[50:51] offset:2048 - v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[76:79], v[216:219] - v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[76:79], v[220:223] - ds_read_b128 a[112:115], v16 offset:6240 - ds_read_b128 a[116:119], v16 offset:6256 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_branch label_3D74 -label_5B68: - s_waitcnt lgkmcnt(2) - s_barrier - v_bfe_u32 v228, v228, v25, 8 - v_bfe_u32 v229, v229, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b32 v36, v43 offset:4608 - ds_read_b32 v37, v43 offset:4864 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v39, 0, v39 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s56 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] - buffer_load_dword v41, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 - v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] - buffer_load_dword v244, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v23 offset:128 - ds_read_b64 v[52:53], v18 offset:512 - v_bfe_u32 v232, v232, v25, 8 - v_bfe_u32 v233, v233, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v24 - ds_read_b64_tr_b16 a[142:143], v24 offset:128 - ds_read_b64 v[54:55], v19 offset:512 - v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v23 offset:512 - ds_read_b64_tr_b16 a[146:147], v23 offset:640 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v24 offset:512 - ds_read_b64 v[56:57], v18 offset:1024 - ds_read_b64 v[58:59], v19 offset:1024 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v24 offset:640 - ds_read_b64 v[60:61], v18 offset:1536 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_min_f32_e32 v26, v26, v36 - v_min_f32_e32 v26, v26, v37 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] - ds_read_b64 v[62:63], v19 offset:1536 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_cbranch_vccz label_5E28 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9DB4 -label_5E28: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - v_max3_f32 v30, v64, v65, v64 - v_max3_f32 v30, v66, v67, v30 - v_max3_f32 v30, v68, v69, v30 - v_max3_f32 v30, v70, v71, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v242, 0, v242 - v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - s_mov_b32 m0, s60 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_waitcnt vmcnt(12) - s_barrier - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - ds_read_b128 v[228:231], v1 offset:41472 - v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] - ds_read_b128 a[40:43], v239 offset:41472 - ds_read_b128 a[44:47], v239 offset:41488 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_u8 v236, v238 offset:41472 - v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] - v_fma_f32 v64, v64, s5, -v32 - v_fma_f32 v65, v65, s5, -v32 - v_fma_f32 v66, v66, s5, -v32 - v_fma_f32 v67, v67, s5, -v32 - v_fma_f32 v68, v68, s5, -v32 - v_fma_f32 v69, v69, s5, -v32 - v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] - ds_read_b128 a[72:75], v239 offset:43552 - ds_read_b128 a[76:79], v239 offset:43568 - v_fma_f32 v70, v70, s5, -v32 - v_fma_f32 v71, v71, s5, -v32 - ds_read_b128 v[232:235], v1 offset:43552 - v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] - v_exp_f32_e32 v64, v64 - v_exp_f32_e32 v65, v65 - v_exp_f32_e32 v66, v66 - v_exp_f32_e32 v67, v67 - v_exp_f32_e32 v68, v68 - v_exp_f32_e32 v69, v69 - v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] - ds_read_b128 a[48:51], v240 offset:41472 - ds_read_b128 a[52:55], v240 offset:41488 - v_exp_f32_e32 v70, v70 - v_exp_f32_e32 v71, v71 - v_mul_f32_e32 v46, v33, v46 - ds_read_u8 v237, v238 offset:43552 - v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] - v_add_f32_e32 v26, v65, v64 - v_add_f32_e32 v26, v66, v26 - v_add_f32_e32 v26, v67, v26 - v_add_f32_e32 v26, v68, v26 - v_add_f32_e32 v26, v69, v26 - v_add_f32_e32 v26, v70, v26 - v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] - ds_read_b128 a[80:83], v240 offset:43552 - ds_read_b128 a[84:87], v240 offset:43568 - v_add_f32_e32 v26, v71, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v64, v64, v65 - v_cvt_pk_bf16_f32 v65, v66, v67 - v_cvt_pk_bf16_f32 v66, v68, v69 - v_cvt_pk_bf16_f32 v67, v70, v71 - v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] - v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] - ds_read_b128 a[56:59], v13 offset:41600 - ds_read_b128 a[60:63], v13 offset:41616 - ds_write_b32 v17, v33 offset:4096 - v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] - v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] - ds_read_b128 a[88:91], v13 offset:43680 - ds_read_b128 a[92:95], v13 offset:43696 - ds_write_b64 v20, v[64:65] - v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] - v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] - ds_read_b128 a[104:107], v16 offset:8320 - ds_read_b128 a[108:111], v16 offset:8336 - ds_write_b64 v20, v[66:67] offset:2048 - v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] - v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] - ds_read_b128 a[112:115], v16 offset:10400 - ds_read_b128 a[116:119], v16 offset:10416 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_bfe_u32 v228, v228, v25, 8 - v_bfe_u32 v229, v229, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b32 v36, v43 offset:4608 - ds_read_b32 v37, v43 offset:4864 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v40, 0, v40 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s57 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - buffer_load_dword v42, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 offset:4160 - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - buffer_load_dword v245, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v23 offset:4288 - ds_read_b64 v[68:69], v18 offset:512 - v_bfe_u32 v232, v232, v25, 8 - v_bfe_u32 v233, v233, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v24 offset:4160 - ds_read_b64_tr_b16 a[142:143], v24 offset:4288 - ds_read_b64 v[70:71], v19 offset:512 - v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v23 offset:4672 - ds_read_b64_tr_b16 a[146:147], v23 offset:4800 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v24 offset:4672 - ds_read_b64 v[72:73], v18 offset:1024 - ds_read_b64 v[74:75], v19 offset:1024 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v24 offset:4800 - ds_read_b64 v[76:77], v18 offset:1536 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_min_f32_e32 v26, v26, v36 - v_min_f32_e32 v26, v26, v37 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - ds_read_b64 v[78:79], v19 offset:1536 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_cbranch_vccz label_65A4 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9DB4 -label_65A4: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v243, 0, v243 - v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] - s_mov_b32 m0, s61 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] - s_waitcnt vmcnt(12) - s_barrier - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - ds_read_b128 v[228:231], v1 offset:58112 - v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[72:75], v[160:163] - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[72:75], v[164:167] - ds_read_b128 a[40:43], v239 offset:58112 - ds_read_b128 a[44:47], v239 offset:58128 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_u8 v236, v238 offset:58112 - v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[72:75], v[168:171] - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[72:75], v[172:175] - ds_read_b128 a[72:75], v239 offset:60192 - ds_read_b128 a[76:79], v239 offset:60208 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - ds_read_b128 v[232:235], v1 offset:60192 - v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[72:75], v[176:179] - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - v_exp_f32_e32 v50, v50 - v_exp_f32_e32 v51, v51 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[72:75], v[180:183] - ds_read_b128 a[48:51], v240 offset:58112 - ds_read_b128 a[52:55], v240 offset:58128 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - ds_read_u8 v237, v238 offset:60192 - v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[72:75], v[184:187] - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[72:75], v[188:191] - ds_read_b128 a[80:83], v240 offset:60192 - ds_read_b128 a[84:87], v240 offset:60208 - v_add_f32_e32 v26, v55, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[76:79], v[192:195] - v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[76:79], v[196:199] - ds_read_b128 a[56:59], v13 offset:58240 - ds_read_b128 a[60:63], v13 offset:58256 - ds_write_b32 v17, v33 offset:4096 - v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[76:79], v[200:203] - v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[76:79], v[204:207] - ds_read_b128 a[88:91], v13 offset:60320 - ds_read_b128 a[92:95], v13 offset:60336 - ds_write_b64 v20, v[48:49] - v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[76:79], v[208:211] - v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[76:79], v[212:215] - ds_read_b128 a[104:107], v16 offset:12480 - ds_read_b128 a[108:111], v16 offset:12496 - ds_write_b64 v20, v[50:51] offset:2048 - v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[76:79], v[216:219] - v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[76:79], v[220:223] - ds_read_b128 a[112:115], v16 offset:14560 - ds_read_b128 a[116:119], v16 offset:14576 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_bfe_u32 v228, v228, v25, 8 - v_bfe_u32 v229, v229, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b32 v36, v43 offset:4608 - ds_read_b32 v37, v43 offset:4864 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v41, 0, v41 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s58 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] - buffer_load_dword v39, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 offset:8320 - v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] - buffer_load_dword v242, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v23 offset:8448 - ds_read_b64 v[52:53], v18 offset:512 - v_bfe_u32 v232, v232, v25, 8 - v_bfe_u32 v233, v233, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v24 offset:8320 - ds_read_b64_tr_b16 a[142:143], v24 offset:8448 - ds_read_b64 v[54:55], v19 offset:512 - v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v23 offset:8832 - ds_read_b64_tr_b16 a[146:147], v23 offset:8960 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v24 offset:8832 - ds_read_b64 v[56:57], v18 offset:1024 - ds_read_b64 v[58:59], v19 offset:1024 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v24 offset:8960 - ds_read_b64 v[60:61], v18 offset:1536 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_min_f32_e32 v26, v26, v36 - v_min_f32_e32 v26, v26, v37 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] - ds_read_b64 v[62:63], v19 offset:1536 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_cbranch_vccz label_6D20 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9DB4 -label_6D20: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - v_max3_f32 v30, v64, v65, v64 - v_max3_f32 v30, v66, v67, v30 - v_max3_f32 v30, v68, v69, v30 - v_max3_f32 v30, v70, v71, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v244, 0, v244 - v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - s_mov_b32 m0, s62 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_waitcnt vmcnt(12) - s_barrier - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - ds_read_b128 v[80:83], v10 offset:8192 - ds_read_b128 v[84:87], v10 offset:8208 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - ds_read_b128 v[88:91], v10 offset:10272 - ds_read_b128 v[92:95], v10 offset:10288 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - ds_read_b128 v[228:231], v1 offset:8192 - v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] - ds_read_b128 a[40:43], v239 offset:8192 - ds_read_b128 a[44:47], v239 offset:8208 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_u8 v236, v238 offset:8192 - v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] - v_fma_f32 v64, v64, s5, -v32 - v_fma_f32 v65, v65, s5, -v32 - v_fma_f32 v66, v66, s5, -v32 - v_fma_f32 v67, v67, s5, -v32 - v_fma_f32 v68, v68, s5, -v32 - v_fma_f32 v69, v69, s5, -v32 - v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] - ds_read_b128 a[72:75], v239 offset:10272 - ds_read_b128 a[76:79], v239 offset:10288 - v_fma_f32 v70, v70, s5, -v32 - v_fma_f32 v71, v71, s5, -v32 - ds_read_b128 v[232:235], v1 offset:10272 - v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] - v_exp_f32_e32 v64, v64 - v_exp_f32_e32 v65, v65 - v_exp_f32_e32 v66, v66 - v_exp_f32_e32 v67, v67 - v_exp_f32_e32 v68, v68 - v_exp_f32_e32 v69, v69 - v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] - ds_read_b128 a[48:51], v240 offset:8192 - ds_read_b128 a[52:55], v240 offset:8208 - v_exp_f32_e32 v70, v70 - v_exp_f32_e32 v71, v71 - v_mul_f32_e32 v46, v33, v46 - ds_read_u8 v237, v238 offset:10272 - v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] - v_add_f32_e32 v26, v65, v64 - v_add_f32_e32 v26, v66, v26 - v_add_f32_e32 v26, v67, v26 - v_add_f32_e32 v26, v68, v26 - v_add_f32_e32 v26, v69, v26 - v_add_f32_e32 v26, v70, v26 - v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] - ds_read_b128 a[80:83], v240 offset:10272 - ds_read_b128 a[84:87], v240 offset:10288 - v_add_f32_e32 v26, v71, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v64, v64, v65 - v_cvt_pk_bf16_f32 v65, v66, v67 - v_cvt_pk_bf16_f32 v66, v68, v69 - v_cvt_pk_bf16_f32 v67, v70, v71 - v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] - v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] - ds_read_b128 a[56:59], v13 offset:8320 - ds_read_b128 a[60:63], v13 offset:8336 - ds_write_b32 v17, v33 offset:4096 - v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] - v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] - ds_read_b128 a[88:91], v13 offset:10400 - ds_read_b128 a[92:95], v13 offset:10416 - ds_write_b64 v20, v[64:65] - v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] - v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] - ds_read_b128 a[104:107], v16 - ds_read_b128 a[108:111], v16 offset:16 - ds_write_b64 v20, v[66:67] offset:2048 - v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] - v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] - ds_read_b128 a[112:115], v16 offset:2080 - ds_read_b128 a[116:119], v16 offset:2096 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_bfe_u32 v228, v228, v25, 8 - v_bfe_u32 v229, v229, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b32 v36, v43 offset:4608 - ds_read_b32 v37, v43 offset:4864 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v42, 0, v42 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s59 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - buffer_load_dword v40, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 offset:12480 - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - buffer_load_dword v243, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v23 offset:12608 - ds_read_b64 v[68:69], v18 offset:512 - v_bfe_u32 v232, v232, v25, 8 - v_bfe_u32 v233, v233, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v24 offset:12480 - ds_read_b64_tr_b16 a[142:143], v24 offset:12608 - ds_read_b64 v[70:71], v19 offset:512 - v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v23 offset:12992 - ds_read_b64_tr_b16 a[146:147], v23 offset:13120 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v24 offset:12992 - ds_read_b64 v[72:73], v18 offset:1024 - ds_read_b64 v[74:75], v19 offset:1024 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v24 offset:13120 - ds_read_b64 v[76:77], v18 offset:1536 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_min_f32_e32 v26, v26, v36 - v_min_f32_e32 v26, v26, v37 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - ds_read_b64 v[78:79], v19 offset:1536 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_cbranch_vccz label_749C - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9DB4 -label_749C: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v245, 0, v245 - v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] - s_mov_b32 m0, s63 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] - s_waitcnt vmcnt(12) - s_barrier - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] - ds_read_b128 v[80:83], v10 offset:24832 - ds_read_b128 v[84:87], v10 offset:24848 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] - ds_read_b128 v[88:91], v10 offset:26912 - ds_read_b128 v[92:95], v10 offset:26928 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - ds_read_b128 v[228:231], v1 offset:24832 - v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[72:75], v[160:163] - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[72:75], v[164:167] - ds_read_b128 a[40:43], v239 offset:24832 - ds_read_b128 a[44:47], v239 offset:24848 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_u8 v236, v238 offset:24832 - v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[72:75], v[168:171] - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[72:75], v[172:175] - ds_read_b128 a[72:75], v239 offset:26912 - ds_read_b128 a[76:79], v239 offset:26928 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - ds_read_b128 v[232:235], v1 offset:26912 - v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[72:75], v[176:179] - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - v_exp_f32_e32 v50, v50 - v_exp_f32_e32 v51, v51 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[72:75], v[180:183] - ds_read_b128 a[48:51], v240 offset:24832 - ds_read_b128 a[52:55], v240 offset:24848 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - ds_read_u8 v237, v238 offset:26912 - v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[72:75], v[184:187] - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[72:75], v[188:191] - ds_read_b128 a[80:83], v240 offset:26912 - ds_read_b128 a[84:87], v240 offset:26928 - v_add_f32_e32 v26, v55, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[76:79], v[192:195] - v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[76:79], v[196:199] - ds_read_b128 a[56:59], v13 offset:24960 - ds_read_b128 a[60:63], v13 offset:24976 - ds_write_b32 v17, v33 offset:4096 - v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[76:79], v[200:203] - v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[76:79], v[204:207] - ds_read_b128 a[88:91], v13 offset:27040 - ds_read_b128 a[92:95], v13 offset:27056 - ds_write_b64 v20, v[48:49] - v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[76:79], v[208:211] - v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[76:79], v[212:215] - ds_read_b128 a[104:107], v16 offset:4160 - ds_read_b128 a[108:111], v16 offset:4176 - ds_write_b64 v20, v[50:51] offset:2048 - v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[76:79], v[216:219] - v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[76:79], v[220:223] - ds_read_b128 a[112:115], v16 offset:6240 - ds_read_b128 a[116:119], v16 offset:6256 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_branch label_5B68 -label_795C: - s_add_i32 s75, s88, -1 - s_and_b32 s75, s75, 3 - s_mul_i32 s75, s75, 0x1040 - v_add_u32_e32 v23, s75, v23 - v_add_u32_e32 v24, s75, v24 - s_cmp_eq_u32 s72, 64 - s_cbranch_scc1 label_7A20 - s_cmp_eq_u32 s72, 16 - s_cbranch_scc0 label_79C4 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b64 v[48:49], v18 - ds_read_b64 v[50:51], v19 - ds_read_b32 v34, v43 offset:4096 - s_waitcnt lgkmcnt(0) - s_barrier - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_cbranch_vccz label_79C0 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_79C0: - s_branch label_7AB0 -label_79C4: - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b64 v[48:49], v18 - ds_read_b64 v[50:51], v19 - ds_read_b64 v[52:53], v18 offset:512 - ds_read_b64 v[54:55], v19 offset:512 - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - s_waitcnt lgkmcnt(0) - s_barrier - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_cbranch_vccz label_7A1C - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_7A1C: - s_branch label_7AB0 -label_7A20: - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b64 v[48:49], v18 - ds_read_b64 v[50:51], v19 - ds_read_b64 v[52:53], v18 offset:512 - ds_read_b64 v[54:55], v19 offset:512 - ds_read_b64 v[56:57], v18 offset:1024 - ds_read_b64 v[58:59], v19 offset:1024 - ds_read_b64 v[60:61], v18 offset:1536 - ds_read_b64 v[62:63], v19 offset:1536 - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b32 v36, v43 offset:4608 - ds_read_b32 v37, v43 offset:4864 - s_waitcnt lgkmcnt(0) - s_barrier - v_min_f32_e32 v26, v34, v35 - v_min_f32_e32 v26, v26, v36 - v_min_f32_e32 v26, v26, v37 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_cbranch_vccz label_7AB0 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9DB4 -label_7AB0: - s_cmp_eq_i32 s7, 3 - s_cbranch_scc1 label_7B3C - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - s_branch label_7BBC -label_7B3C: - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 - ds_read_b64_tr_b16 a[138:139], v23 offset:128 - ds_read_b64_tr_b16 a[140:141], v24 - ds_read_b64_tr_b16 a[142:143], v24 offset:128 - ds_read_b64_tr_b16 a[144:145], v23 offset:512 - ds_read_b64_tr_b16 a[146:147], v23 offset:640 - ds_read_b64_tr_b16 a[148:149], v24 offset:512 - ds_read_b64_tr_b16 a[150:151], v24 offset:640 -label_7BBC: - s_waitcnt lgkmcnt(0) - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - s_cmp_le_u32 s72, 16 - s_cbranch_scc1 label_7CD0 - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - s_cmp_le_u32 s72, 32 - s_cbranch_scc1 label_7CD0 - v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] - v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] - v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] - v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] - v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] - v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] - v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] - v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] - v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] - v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] - v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] - v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] - v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] - v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] - v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] - v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] -label_7CD0: - s_add_i32 s75, s88, -1 - s_and_b32 s75, s75, 3 - s_mul_i32 s75, s75, 0xffffefc0 - v_add_u32_e32 v23, s75, v23 - v_add_u32_e32 v24, s75, v24 - s_cmp_eq_i32 s70, 0 - s_cbranch_scc1 label_7CF4 - s_branch label_8EFC -label_7CF4: - s_cmp_eq_u32 s4, 0 - s_cbranch_scc0 label_7D78 - v_and_b32_e64 v33, v0, 15 - v_lshlrev_b32_e32 v33, 2, v33 - s_lshl_b32 s75, s7, 4 - s_sub_u32 s76, s80, 1 - s_min_u32 s76, s76, 63 - s_and_b32 s75, s75, s76 - s_lshl_b32 s76, s2, 6 - s_add_u32 s75, s75, s76 - s_lshl_b32 s75, s75, 2 - v_add_u32_e32 v33, s75, v33 - buffer_load_dword v30, v33, s[52:55], 0 offen - s_waitcnt vmcnt(0) - v_mul_f32_e32 v30, 0x41b504f3, v30 - v_max_f32_e32 v31, v45, v30 - v_sub_f32_e32 v33, v45, v31 - v_mul_f32_e64 v33, v33, s5 - v_exp_f32_e32 v32, v33 - v_sub_f32_e32 v30, v30, v31 - v_mul_f32_e64 v30, v30, s5 - v_exp_f32_e32 v30, v30 - s_nop 0 - v_mul_f32_e32 v30, 0x3e800000, v30 - v_fma_f32 v46, v46, v32, v30 - v_mov_b32_e32 v45, v31 -label_7D78: - v_mov_b32_e32 v26, v46 - v_mov_b32_e32 v27, v46 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - v_mov_b32_e32 v46, 0 - v_add_f32_e32 v46, v26, v46 - v_add_f32_e32 v46, v27, v46 - v_add_f32_e32 v46, v28, v46 - v_add_f32_e32 v46, v29, v46 - v_mov_b32_e32 v26, 0 - v_cmp_eq_u32_e64 s[64:65], v26, v46 - v_mul_f32_e64 v26, v45, s79 - v_log_f32_e32 v27, v46 - v_cndmask_b32_e64 v46, v46, 1.0, s[64:65] - s_nop 1 - v_rcp_f32_e32 v46, v46 - s_nop 1 - v_fma_f32 v33, v27, s6, v26 - s_cmp_eq_u32 s4, 0 - s_cbranch_scc0 label_7DF0 - v_mul_f32_e32 v46, v46, v32 -label_7DF0: - s_mov_b32 s87, s72 - v_lshlrev_b32_e32 v30, 2, v0 - s_mul_i32 s75, 0x100, s7 - v_add_u32_e32 v30, s75, v30 - ds_write_b32 v30, v46 offset:4096 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v26, v43 offset:4096 - s_cmp_le_u32 s87, 16 - s_cbranch_scc1 label_7E44 - ds_read_b32 v27, v43 offset:4352 - s_cmp_le_u32 s87, 32 - s_cbranch_scc1 label_7E44 - ds_read_b32 v28, v43 offset:4608 - ds_read_b32 v29, v43 offset:4864 -label_7E44: - s_waitcnt lgkmcnt(0) - v_pk_mul_f32 v[96:97], v[26:27], v[96:97] op_sel_hi:[0,1] - v_pk_mul_f32 v[98:99], v[26:27], v[98:99] op_sel_hi:[0,1] - v_pk_mul_f32 v[100:101], v[26:27], v[100:101] op_sel_hi:[0,1] - v_pk_mul_f32 v[102:103], v[26:27], v[102:103] op_sel_hi:[0,1] - v_pk_mul_f32 v[104:105], v[26:27], v[104:105] op_sel_hi:[0,1] - v_pk_mul_f32 v[106:107], v[26:27], v[106:107] op_sel_hi:[0,1] - v_pk_mul_f32 v[108:109], v[26:27], v[108:109] op_sel_hi:[0,1] - v_pk_mul_f32 v[110:111], v[26:27], v[110:111] op_sel_hi:[0,1] - v_pk_mul_f32 v[112:113], v[26:27], v[112:113] op_sel_hi:[0,1] - v_pk_mul_f32 v[114:115], v[26:27], v[114:115] op_sel_hi:[0,1] - v_pk_mul_f32 v[116:117], v[26:27], v[116:117] op_sel_hi:[0,1] - v_pk_mul_f32 v[118:119], v[26:27], v[118:119] op_sel_hi:[0,1] - v_pk_mul_f32 v[120:121], v[26:27], v[120:121] op_sel_hi:[0,1] - v_pk_mul_f32 v[122:123], v[26:27], v[122:123] op_sel_hi:[0,1] - v_pk_mul_f32 v[124:125], v[26:27], v[124:125] op_sel_hi:[0,1] - v_pk_mul_f32 v[126:127], v[26:27], v[126:127] op_sel_hi:[0,1] - s_cmp_le_u32 s87, 16 - s_cbranch_scc1 label_8058 - v_pk_mul_f32 v[128:129], v[26:27], v[128:129] op_sel:[1,0] - v_pk_mul_f32 v[130:131], v[26:27], v[130:131] op_sel:[1,0] - v_pk_mul_f32 v[132:133], v[26:27], v[132:133] op_sel:[1,0] - v_pk_mul_f32 v[134:135], v[26:27], v[134:135] op_sel:[1,0] - v_pk_mul_f32 v[136:137], v[26:27], v[136:137] op_sel:[1,0] - v_pk_mul_f32 v[138:139], v[26:27], v[138:139] op_sel:[1,0] - v_pk_mul_f32 v[140:141], v[26:27], v[140:141] op_sel:[1,0] - v_pk_mul_f32 v[142:143], v[26:27], v[142:143] op_sel:[1,0] - v_pk_mul_f32 v[144:145], v[26:27], v[144:145] op_sel:[1,0] - v_pk_mul_f32 v[146:147], v[26:27], v[146:147] op_sel:[1,0] - v_pk_mul_f32 v[148:149], v[26:27], v[148:149] op_sel:[1,0] - v_pk_mul_f32 v[150:151], v[26:27], v[150:151] op_sel:[1,0] - v_pk_mul_f32 v[152:153], v[26:27], v[152:153] op_sel:[1,0] - v_pk_mul_f32 v[154:155], v[26:27], v[154:155] op_sel:[1,0] - v_pk_mul_f32 v[156:157], v[26:27], v[156:157] op_sel:[1,0] - v_pk_mul_f32 v[158:159], v[26:27], v[158:159] op_sel:[1,0] - s_cmp_le_u32 s87, 32 - s_cbranch_scc1 label_8058 - v_pk_mul_f32 v[160:161], v[28:29], v[160:161] op_sel_hi:[0,1] - v_pk_mul_f32 v[162:163], v[28:29], v[162:163] op_sel_hi:[0,1] - v_pk_mul_f32 v[164:165], v[28:29], v[164:165] op_sel_hi:[0,1] - v_pk_mul_f32 v[166:167], v[28:29], v[166:167] op_sel_hi:[0,1] - v_pk_mul_f32 v[168:169], v[28:29], v[168:169] op_sel_hi:[0,1] - v_pk_mul_f32 v[170:171], v[28:29], v[170:171] op_sel_hi:[0,1] - v_pk_mul_f32 v[172:173], v[28:29], v[172:173] op_sel_hi:[0,1] - v_pk_mul_f32 v[174:175], v[28:29], v[174:175] op_sel_hi:[0,1] - v_pk_mul_f32 v[176:177], v[28:29], v[176:177] op_sel_hi:[0,1] - v_pk_mul_f32 v[178:179], v[28:29], v[178:179] op_sel_hi:[0,1] - v_pk_mul_f32 v[180:181], v[28:29], v[180:181] op_sel_hi:[0,1] - v_pk_mul_f32 v[182:183], v[28:29], v[182:183] op_sel_hi:[0,1] - v_pk_mul_f32 v[184:185], v[28:29], v[184:185] op_sel_hi:[0,1] - v_pk_mul_f32 v[186:187], v[28:29], v[186:187] op_sel_hi:[0,1] - v_pk_mul_f32 v[188:189], v[28:29], v[188:189] op_sel_hi:[0,1] - v_pk_mul_f32 v[190:191], v[28:29], v[190:191] op_sel_hi:[0,1] - v_pk_mul_f32 v[192:193], v[28:29], v[192:193] op_sel:[1,0] - v_pk_mul_f32 v[194:195], v[28:29], v[194:195] op_sel:[1,0] - v_pk_mul_f32 v[196:197], v[28:29], v[196:197] op_sel:[1,0] - v_pk_mul_f32 v[198:199], v[28:29], v[198:199] op_sel:[1,0] - v_pk_mul_f32 v[200:201], v[28:29], v[200:201] op_sel:[1,0] - v_pk_mul_f32 v[202:203], v[28:29], v[202:203] op_sel:[1,0] - v_pk_mul_f32 v[204:205], v[28:29], v[204:205] op_sel:[1,0] - v_pk_mul_f32 v[206:207], v[28:29], v[206:207] op_sel:[1,0] - v_pk_mul_f32 v[208:209], v[28:29], v[208:209] op_sel:[1,0] - v_pk_mul_f32 v[210:211], v[28:29], v[210:211] op_sel:[1,0] - v_pk_mul_f32 v[212:213], v[28:29], v[212:213] op_sel:[1,0] - v_pk_mul_f32 v[214:215], v[28:29], v[214:215] op_sel:[1,0] - v_pk_mul_f32 v[216:217], v[28:29], v[216:217] op_sel:[1,0] - v_pk_mul_f32 v[218:219], v[28:29], v[218:219] op_sel:[1,0] - v_pk_mul_f32 v[220:221], v[28:29], v[220:221] op_sel:[1,0] - v_pk_mul_f32 v[222:223], v[28:29], v[222:223] op_sel:[1,0] -label_8058: - s_cmp_gt_u32 s82, 1 - s_cbranch_scc1 label_8068 - s_cmp_eq_u32 s83, 1 - s_cbranch_scc1 label_86D0 -label_8068: - s_mul_i32 s95, 0x800, s80 - s_mul_i32 s96, s81, s95 - s_mul_i32 s75, s89, s96 - s_add_u32 s8, s75, s8 - s_addc_u32 s9, 0, s9 - s_mov_b32 s75, 0x20000 - s_mul_i32 s75, s75, s2 - s_add_u32 s8, s75, s8 - s_addc_u32 s9, 0, s9 - s_mul_i32 s75, s96, s91 - s_mov_b32 s10, s75 - v_and_b32_e32 v26, 15, v0 - v_lshlrev_b32_e32 v2, 4, v26 - v_lshrrev_b32_e32 v26, 4, v0 - v_mul_i32_i24_e32 v26, 0x800, v26 - v_add_u32_e32 v2, v2, v26 - s_mul_i32 s75, s4, s95 - v_add_u32_e64 v2, v2, s75 - s_mul_i32 s75, s7, 0x200 - v_add_u32_e64 v2, v2, s75 - v_mov_b32_e32 v1, v2 - s_lshl_b32 s64, s7, 4 - s_ff1_i32_b32 s78, s80 - s_lshr_b32 s66, s64, s78 - s_sub_u32 s78, s80, 1 - s_and_b32 s76, s64, s78 - s_mul_i32 s97, 4, s80 - s_mul_i32 s77, s81, s97 - s_mul_i32 s75, s89, s77 - s_mul_i32 s78, s4, s97 - s_add_u32 s75, s75, s78 - s_mov_b32 s78, 0x100 - s_mul_i32 s78, s78, s2 - s_add_u32 s75, s75, s78 - s_add_u32 s12, s75, s12 - s_addc_u32 s13, 0, s13 - s_mul_i32 s75, s91, s77 - s_mov_b32 s14, s75 - v_and_b32_e32 v26, 15, v0 - v_lshlrev_b32_e32 v26, 2, v26 - s_lshl_b32 s76, s76, 2 - v_add_u32_e64 v26, v26, s76 - s_mul_i32 s76, s66, s77 - v_add_u32_e64 v26, v26, s76 - buffer_store_dword v33, v26, s[12:15], 0 offen - s_waitcnt vmcnt(0) lgkmcnt(0) - s_barrier - v_and_b32_e32 v26, 3, v0 - v_lshlrev_b32_e32 v16, 2, v26 - v_bfe_u32 v26, v0, 2, 2 - v_lshl_add_u32 v16, v26, 4, v16 - v_lshrrev_b32_e32 v26, 4, v0 - v_mul_i32_i24_e32 v26, 0x44, v26 - v_add_u32_e32 v16, v26, v16 - s_mul_i32 s75, s7, 0x880 - v_add_u32_e32 v16, s75, v16 - v_lshlrev_b32_e32 v16, 2, v16 - v_lshrrev_b32_e32 v26, 4, v0 - v_mul_i32_i24_e32 v10, 4, v26 - v_and_b32_e32 v26, 3, v0 - v_mul_i32_i24_e32 v26, 0x44, v26 - v_add_u32_e32 v10, v26, v10 - v_and_b32_e32 v26, 15, v0 - v_lshrrev_b32_e32 v26, 2, v26 - v_mul_i32_i24_e32 v26, 0x110, v26 - v_add_u32_e32 v10, v26, v10 - s_mul_i32 s75, s7, 0x880 - v_add_u32_e32 v10, s75, v10 - v_lshlrev_b32_e32 v10, 2, v10 - v_mov_b32_e32 v2, v1 - v_mov_b64_e32 v[26:27], v[96:97] - v_mov_b64_e32 v[28:29], v[98:99] - ds_write_b128 v16, v[26:29] - v_mov_b64_e32 v[26:27], v[100:101] - v_mov_b64_e32 v[28:29], v[102:103] - ds_write_b128 v16, v[26:29] offset:1088 - v_mov_b64_e32 v[26:27], v[104:105] - v_mov_b64_e32 v[28:29], v[106:107] - ds_write_b128 v16, v[26:29] offset:2176 - v_mov_b64_e32 v[26:27], v[108:109] - v_mov_b64_e32 v[28:29], v[110:111] - ds_write_b128 v16, v[26:29] offset:3264 - v_mov_b64_e32 v[26:27], v[112:113] - v_mov_b64_e32 v[28:29], v[114:115] - ds_write_b128 v16, v[26:29] offset:4352 - v_mov_b64_e32 v[26:27], v[116:117] - v_mov_b64_e32 v[28:29], v[118:119] - ds_write_b128 v16, v[26:29] offset:5440 - v_mov_b64_e32 v[26:27], v[120:121] - v_mov_b64_e32 v[28:29], v[122:123] - ds_write_b128 v16, v[26:29] offset:6528 - v_mov_b64_e32 v[26:27], v[124:125] - v_mov_b64_e32 v[28:29], v[126:127] - ds_write_b128 v16, v[26:29] offset:7616 - s_waitcnt lgkmcnt(4) - ds_read_b128 v[96:99], v10 - ds_read_b128 v[100:103], v10 offset:64 - ds_read_b128 v[104:107], v10 offset:128 - ds_read_b128 v[108:111], v10 offset:192 - s_waitcnt lgkmcnt(4) - ds_read_b128 v[112:115], v10 offset:4352 - ds_read_b128 v[116:119], v10 offset:4416 - ds_read_b128 v[120:123], v10 offset:4480 - ds_read_b128 v[124:127], v10 offset:4544 - s_waitcnt lgkmcnt(0) - buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[112:115], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[116:119], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[120:123], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[124:127], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 - s_cmp_le_u32 s87, 16 - s_cbranch_scc1 label_86CC - s_sub_u32 s76, s96, s95 - s_sub_u32 s77, s80, 1 - s_and_b32 s75, s77, 16 - s_cmp_eq_u32 s75, 0 - s_cselect_b32 s75, s76, 0 - v_add_u32_e32 v2, s75, v2 - v_mov_b64_e32 v[26:27], v[128:129] - v_mov_b64_e32 v[28:29], v[130:131] - ds_write_b128 v16, v[26:29] - v_mov_b64_e32 v[26:27], v[132:133] - v_mov_b64_e32 v[28:29], v[134:135] - ds_write_b128 v16, v[26:29] offset:1088 - v_mov_b64_e32 v[26:27], v[136:137] - v_mov_b64_e32 v[28:29], v[138:139] - ds_write_b128 v16, v[26:29] offset:2176 - v_mov_b64_e32 v[26:27], v[140:141] - v_mov_b64_e32 v[28:29], v[142:143] - ds_write_b128 v16, v[26:29] offset:3264 - v_mov_b64_e32 v[26:27], v[144:145] - v_mov_b64_e32 v[28:29], v[146:147] - ds_write_b128 v16, v[26:29] offset:4352 - v_mov_b64_e32 v[26:27], v[148:149] - v_mov_b64_e32 v[28:29], v[150:151] - ds_write_b128 v16, v[26:29] offset:5440 - v_mov_b64_e32 v[26:27], v[152:153] - v_mov_b64_e32 v[28:29], v[154:155] - ds_write_b128 v16, v[26:29] offset:6528 - v_mov_b64_e32 v[26:27], v[156:157] - v_mov_b64_e32 v[28:29], v[158:159] - ds_write_b128 v16, v[26:29] offset:7616 - s_waitcnt lgkmcnt(4) - ds_read_b128 v[96:99], v10 - ds_read_b128 v[100:103], v10 offset:64 - ds_read_b128 v[104:107], v10 offset:128 - ds_read_b128 v[108:111], v10 offset:192 - s_waitcnt lgkmcnt(4) - ds_read_b128 v[112:115], v10 offset:4352 - ds_read_b128 v[116:119], v10 offset:4416 - ds_read_b128 v[120:123], v10 offset:4480 - ds_read_b128 v[124:127], v10 offset:4544 - s_waitcnt lgkmcnt(0) - buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[112:115], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[116:119], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[120:123], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[124:127], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 - s_cmp_le_u32 s87, 32 - s_cbranch_scc1 label_86CC - s_sub_u32 s76, s96, s95 - s_sub_u32 s77, s80, 1 - s_and_b32 s75, s77, 32 - s_cmp_eq_u32 s75, 0 - s_cselect_b32 s75, s76, 0 - v_add_u32_e32 v2, s75, v2 - v_mov_b64_e32 v[26:27], v[160:161] - v_mov_b64_e32 v[28:29], v[162:163] - ds_write_b128 v16, v[26:29] - v_mov_b64_e32 v[26:27], v[164:165] - v_mov_b64_e32 v[28:29], v[166:167] - ds_write_b128 v16, v[26:29] offset:1088 - v_mov_b64_e32 v[26:27], v[168:169] - v_mov_b64_e32 v[28:29], v[170:171] - ds_write_b128 v16, v[26:29] offset:2176 - v_mov_b64_e32 v[26:27], v[172:173] - v_mov_b64_e32 v[28:29], v[174:175] - ds_write_b128 v16, v[26:29] offset:3264 - v_mov_b64_e32 v[26:27], v[176:177] - v_mov_b64_e32 v[28:29], v[178:179] - ds_write_b128 v16, v[26:29] offset:4352 - v_mov_b64_e32 v[26:27], v[180:181] - v_mov_b64_e32 v[28:29], v[182:183] - ds_write_b128 v16, v[26:29] offset:5440 - v_mov_b64_e32 v[26:27], v[184:185] - v_mov_b64_e32 v[28:29], v[186:187] - ds_write_b128 v16, v[26:29] offset:6528 - v_mov_b64_e32 v[26:27], v[188:189] - v_mov_b64_e32 v[28:29], v[190:191] - ds_write_b128 v16, v[26:29] offset:7616 - s_waitcnt lgkmcnt(4) - ds_read_b128 v[96:99], v10 - ds_read_b128 v[100:103], v10 offset:64 - ds_read_b128 v[104:107], v10 offset:128 - ds_read_b128 v[108:111], v10 offset:192 - s_waitcnt lgkmcnt(4) - ds_read_b128 v[112:115], v10 offset:4352 - ds_read_b128 v[116:119], v10 offset:4416 - ds_read_b128 v[120:123], v10 offset:4480 - ds_read_b128 v[124:127], v10 offset:4544 - s_waitcnt lgkmcnt(0) - buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[112:115], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[116:119], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[120:123], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[124:127], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 - s_sub_u32 s76, s96, s95 - s_sub_u32 s77, s80, 1 - s_and_b32 s75, s77, 48 - s_cmp_eq_u32 s75, 0 - s_cselect_b32 s75, s76, 0 - v_add_u32_e32 v2, s75, v2 - v_mov_b64_e32 v[26:27], v[192:193] - v_mov_b64_e32 v[28:29], v[194:195] - ds_write_b128 v16, v[26:29] - v_mov_b64_e32 v[26:27], v[196:197] - v_mov_b64_e32 v[28:29], v[198:199] - ds_write_b128 v16, v[26:29] offset:1088 - v_mov_b64_e32 v[26:27], v[200:201] - v_mov_b64_e32 v[28:29], v[202:203] - ds_write_b128 v16, v[26:29] offset:2176 - v_mov_b64_e32 v[26:27], v[204:205] - v_mov_b64_e32 v[28:29], v[206:207] - ds_write_b128 v16, v[26:29] offset:3264 - v_mov_b64_e32 v[26:27], v[208:209] - v_mov_b64_e32 v[28:29], v[210:211] - ds_write_b128 v16, v[26:29] offset:4352 - v_mov_b64_e32 v[26:27], v[212:213] - v_mov_b64_e32 v[28:29], v[214:215] - ds_write_b128 v16, v[26:29] offset:5440 - v_mov_b64_e32 v[26:27], v[216:217] - v_mov_b64_e32 v[28:29], v[218:219] - ds_write_b128 v16, v[26:29] offset:6528 - v_mov_b64_e32 v[26:27], v[220:221] - v_mov_b64_e32 v[28:29], v[222:223] - ds_write_b128 v16, v[26:29] offset:7616 - s_waitcnt lgkmcnt(4) - ds_read_b128 v[96:99], v10 - ds_read_b128 v[100:103], v10 offset:64 - ds_read_b128 v[104:107], v10 offset:128 - ds_read_b128 v[108:111], v10 offset:192 - s_waitcnt lgkmcnt(4) - ds_read_b128 v[112:115], v10 offset:4352 - ds_read_b128 v[116:119], v10 offset:4416 - ds_read_b128 v[120:123], v10 offset:4480 - ds_read_b128 v[124:127], v10 offset:4544 - s_waitcnt lgkmcnt(0) - buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[112:115], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[116:119], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[120:123], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[124:127], v2, s[8:11], 0 offen offset:256 - v_add_u32_e32 v2, 0x2000, v2 -label_86CC: - s_branch label_8EF4 -label_86D0: - s_mul_i32 s95, 0x400, s80 - s_mul_i32 s96, s82, s95 - s_mul_i32 s75, s89, s96 - s_add_u32 s8, s75, s8 - s_addc_u32 s9, 0, s9 - s_mov_b32 s75, 0x10000 - s_mul_i32 s75, s75, s2 - s_add_u32 s8, s75, s8 - s_addc_u32 s9, 0, s9 - s_mul_i32 s75, s96, s91 - s_mov_b32 s10, s75 - v_and_b32_e32 v26, 7, v0 - v_lshlrev_b32_e32 v2, 4, v26 - v_lshrrev_b32_e32 v26, 3, v0 - v_mul_i32_i24_e32 v26, 0x400, v26 - v_add_u32_e32 v2, v2, v26 - s_mul_i32 s75, s4, s95 - v_add_u32_e64 v2, v2, s75 - s_mul_i32 s75, s7, 0x100 - v_add_u32_e64 v2, v2, s75 - v_mov_b32_e32 v1, v2 - s_waitcnt vmcnt(0) lgkmcnt(0) - s_barrier - v_lshrrev_b32_e32 v26, 4, v0 - v_mul_i32_i24_e32 v16, 0x48, v26 - v_and_b32_e32 v26, 15, v0 - v_mul_i32_i24_e32 v26, 2, v26 - v_add_u32_e32 v16, v26, v16 - s_mul_i32 s75, s7, 0x480 - v_add_u32_e32 v16, s75, v16 - v_lshlrev_b32_e32 v16, 2, v16 - v_lshrrev_b32_e32 v26, 3, v0 - v_mul_i32_i24_e32 v10, 2, v26 - v_and_b32_e32 v26, 3, v0 - v_mul_i32_i24_e32 v26, 0x90, v26 - v_add_u32_e32 v10, v26, v10 - v_lshrrev_b32_e32 v26, 2, v0 - v_and_b32_e32 v26, 1, v26 - v_mul_i32_i24_e32 v26, 36, v26 - v_add_u32_e32 v10, v26, v10 - s_mul_i32 s75, s7, 0x480 - v_add_u32_e32 v10, s75, v10 - v_lshlrev_b32_e32 v10, 2, v10 - v_mov_b32_e32 v2, v1 - v_mov_b64_e32 v[30:31], v[96:97] - v_mov_b64_e32 v[32:33], v[98:99] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] - v_mov_b64_e32 v[30:31], v[100:101] - v_mov_b64_e32 v[32:33], v[102:103] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:1152 - v_mov_b64_e32 v[30:31], v[104:105] - v_mov_b64_e32 v[32:33], v[106:107] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:144 - v_mov_b64_e32 v[30:31], v[108:109] - v_mov_b64_e32 v[32:33], v[110:111] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:1296 - v_mov_b64_e32 v[30:31], v[112:113] - v_mov_b64_e32 v[32:33], v[114:115] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:2304 - v_mov_b64_e32 v[30:31], v[116:117] - v_mov_b64_e32 v[32:33], v[118:119] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:3456 - v_mov_b64_e32 v[30:31], v[120:121] - v_mov_b64_e32 v[32:33], v[122:123] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:2448 - v_mov_b64_e32 v[30:31], v[124:125] - v_mov_b64_e32 v[32:33], v[126:127] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:3600 - s_waitcnt lgkmcnt(4) - ds_read_b64 v[96:97], v10 - ds_read_b64 v[100:101], v10 offset:64 - ds_read_b64 v[98:99], v10 offset:288 - ds_read_b64 v[102:103], v10 offset:352 - s_waitcnt lgkmcnt(4) - ds_read_b64 v[104:105], v10 offset:2304 - ds_read_b64 v[108:109], v10 offset:2368 - ds_read_b64 v[106:107], v10 offset:2592 - ds_read_b64 v[110:111], v10 offset:2656 - s_waitcnt lgkmcnt(0) - buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen offset:128 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen offset:128 - v_add_u32_e32 v2, 0x2000, v2 - s_cmp_le_u32 s87, 16 - s_cbranch_scc1 label_8DA8 - v_mov_b64_e32 v[30:31], v[128:129] - v_mov_b64_e32 v[32:33], v[130:131] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] - v_mov_b64_e32 v[30:31], v[132:133] - v_mov_b64_e32 v[32:33], v[134:135] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:1152 - v_mov_b64_e32 v[30:31], v[136:137] - v_mov_b64_e32 v[32:33], v[138:139] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:144 - v_mov_b64_e32 v[30:31], v[140:141] - v_mov_b64_e32 v[32:33], v[142:143] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:1296 - v_mov_b64_e32 v[30:31], v[144:145] - v_mov_b64_e32 v[32:33], v[146:147] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:2304 - v_mov_b64_e32 v[30:31], v[148:149] - v_mov_b64_e32 v[32:33], v[150:151] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:3456 - v_mov_b64_e32 v[30:31], v[152:153] - v_mov_b64_e32 v[32:33], v[154:155] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:2448 - v_mov_b64_e32 v[30:31], v[156:157] - v_mov_b64_e32 v[32:33], v[158:159] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:3600 - s_waitcnt lgkmcnt(4) - ds_read_b64 v[96:97], v10 - ds_read_b64 v[100:101], v10 offset:64 - ds_read_b64 v[98:99], v10 offset:288 - ds_read_b64 v[102:103], v10 offset:352 - s_waitcnt lgkmcnt(4) - ds_read_b64 v[104:105], v10 offset:2304 - ds_read_b64 v[108:109], v10 offset:2368 - ds_read_b64 v[106:107], v10 offset:2592 - ds_read_b64 v[110:111], v10 offset:2656 - s_waitcnt lgkmcnt(0) - buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen offset:128 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen offset:128 - v_add_u32_e32 v2, 0x2000, v2 - s_cmp_le_u32 s87, 32 - s_cbranch_scc1 label_8DA8 - v_mov_b64_e32 v[30:31], v[160:161] - v_mov_b64_e32 v[32:33], v[162:163] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] - v_mov_b64_e32 v[30:31], v[164:165] - v_mov_b64_e32 v[32:33], v[166:167] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:1152 - v_mov_b64_e32 v[30:31], v[168:169] - v_mov_b64_e32 v[32:33], v[170:171] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:144 - v_mov_b64_e32 v[30:31], v[172:173] - v_mov_b64_e32 v[32:33], v[174:175] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:1296 - v_mov_b64_e32 v[30:31], v[176:177] - v_mov_b64_e32 v[32:33], v[178:179] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:2304 - v_mov_b64_e32 v[30:31], v[180:181] - v_mov_b64_e32 v[32:33], v[182:183] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:3456 - v_mov_b64_e32 v[30:31], v[184:185] - v_mov_b64_e32 v[32:33], v[186:187] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:2448 - v_mov_b64_e32 v[30:31], v[188:189] - v_mov_b64_e32 v[32:33], v[190:191] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:3600 - s_waitcnt lgkmcnt(4) - ds_read_b64 v[96:97], v10 - ds_read_b64 v[100:101], v10 offset:64 - ds_read_b64 v[98:99], v10 offset:288 - ds_read_b64 v[102:103], v10 offset:352 - s_waitcnt lgkmcnt(4) - ds_read_b64 v[104:105], v10 offset:2304 - ds_read_b64 v[108:109], v10 offset:2368 - ds_read_b64 v[106:107], v10 offset:2592 - ds_read_b64 v[110:111], v10 offset:2656 - s_waitcnt lgkmcnt(0) - buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen offset:128 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen offset:128 - v_add_u32_e32 v2, 0x2000, v2 - v_mov_b64_e32 v[30:31], v[192:193] - v_mov_b64_e32 v[32:33], v[194:195] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] - v_mov_b64_e32 v[30:31], v[196:197] - v_mov_b64_e32 v[32:33], v[198:199] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:1152 - v_mov_b64_e32 v[30:31], v[200:201] - v_mov_b64_e32 v[32:33], v[202:203] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:144 - v_mov_b64_e32 v[30:31], v[204:205] - v_mov_b64_e32 v[32:33], v[206:207] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:1296 - v_mov_b64_e32 v[30:31], v[208:209] - v_mov_b64_e32 v[32:33], v[210:211] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:2304 - v_mov_b64_e32 v[30:31], v[212:213] - v_mov_b64_e32 v[32:33], v[214:215] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:3456 - v_mov_b64_e32 v[30:31], v[216:217] - v_mov_b64_e32 v[32:33], v[218:219] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:2448 - v_mov_b64_e32 v[30:31], v[220:221] - v_mov_b64_e32 v[32:33], v[222:223] - v_cvt_pk_bf16_f32 v30, v30, v31 - v_cvt_pk_bf16_f32 v31, v32, v33 - ds_write_b64 v16, v[30:31] offset:3600 - s_waitcnt lgkmcnt(4) - ds_read_b64 v[96:97], v10 - ds_read_b64 v[100:101], v10 offset:64 - ds_read_b64 v[98:99], v10 offset:288 - ds_read_b64 v[102:103], v10 offset:352 - s_waitcnt lgkmcnt(4) - ds_read_b64 v[104:105], v10 offset:2304 - ds_read_b64 v[108:109], v10 offset:2368 - ds_read_b64 v[106:107], v10 offset:2592 - ds_read_b64 v[110:111], v10 offset:2656 - s_waitcnt lgkmcnt(0) - buffer_store_dwordx4 v[96:99], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[104:107], v2, s[8:11], 0 offen offset:128 - v_add_u32_e32 v2, 0x2000, v2 - buffer_store_dwordx4 v[100:103], v2, s[8:11], 0 offen - buffer_store_dwordx4 v[108:111], v2, s[8:11], 0 offen offset:128 - v_add_u32_e32 v2, 0x2000, v2 -label_8DA8: - s_branch label_8EF4 -label_8DAC: - s_cmp_eq_u32 s99, 1 - s_cbranch_scc0 label_8DC4 - s_waitcnt vmcnt(0) - s_waitcnt lgkmcnt(0) - s_barrier - s_branch label_8EF4 -label_8DC4: - s_cmp_eq_u32 s80, 16 - s_cselect_b32 s69, 16, 64 - s_cmp_eq_u32 s80, 32 - s_cselect_b32 s69, 32, s69 - s_cmp_eq_u32 s91, 1 - s_cselect_b32 s69, s69, 64 - s_mov_b32 s72, s69 - v_mov_b32_e32 v33, 0xe0ad78ec - v_mov_b32_e32 v46, 0 - v_mov_b32_e32 v43, 0 - v_mov_b64_e32 v[96:97], 0 - v_mov_b64_e32 v[98:99], 0 - v_mov_b64_e32 v[100:101], 0 - v_mov_b64_e32 v[102:103], 0 - v_mov_b64_e32 v[104:105], 0 - v_mov_b64_e32 v[106:107], 0 - v_mov_b64_e32 v[108:109], 0 - v_mov_b64_e32 v[110:111], 0 - v_mov_b64_e32 v[112:113], 0 - v_mov_b64_e32 v[114:115], 0 - v_mov_b64_e32 v[116:117], 0 - v_mov_b64_e32 v[118:119], 0 - v_mov_b64_e32 v[120:121], 0 - v_mov_b64_e32 v[122:123], 0 - v_mov_b64_e32 v[124:125], 0 - v_mov_b64_e32 v[126:127], 0 - v_mov_b64_e32 v[128:129], 0 - v_mov_b64_e32 v[130:131], 0 - v_mov_b64_e32 v[132:133], 0 - v_mov_b64_e32 v[134:135], 0 - v_mov_b64_e32 v[136:137], 0 - v_mov_b64_e32 v[138:139], 0 - v_mov_b64_e32 v[140:141], 0 - v_mov_b64_e32 v[142:143], 0 - v_mov_b64_e32 v[144:145], 0 - v_mov_b64_e32 v[146:147], 0 - v_mov_b64_e32 v[148:149], 0 - v_mov_b64_e32 v[150:151], 0 - v_mov_b64_e32 v[152:153], 0 - v_mov_b64_e32 v[154:155], 0 - v_mov_b64_e32 v[156:157], 0 - v_mov_b64_e32 v[158:159], 0 - v_mov_b64_e32 v[160:161], 0 - v_mov_b64_e32 v[162:163], 0 - v_mov_b64_e32 v[164:165], 0 - v_mov_b64_e32 v[166:167], 0 - v_mov_b64_e32 v[168:169], 0 - v_mov_b64_e32 v[170:171], 0 - v_mov_b64_e32 v[172:173], 0 - v_mov_b64_e32 v[174:175], 0 - v_mov_b64_e32 v[176:177], 0 - v_mov_b64_e32 v[178:179], 0 - v_mov_b64_e32 v[180:181], 0 - v_mov_b64_e32 v[182:183], 0 - v_mov_b64_e32 v[184:185], 0 - v_mov_b64_e32 v[186:187], 0 - v_mov_b64_e32 v[188:189], 0 - v_mov_b64_e32 v[190:191], 0 - v_mov_b64_e32 v[192:193], 0 - v_mov_b64_e32 v[194:195], 0 - v_mov_b64_e32 v[196:197], 0 - v_mov_b64_e32 v[198:199], 0 - v_mov_b64_e32 v[200:201], 0 - v_mov_b64_e32 v[202:203], 0 - v_mov_b64_e32 v[204:205], 0 - v_mov_b64_e32 v[206:207], 0 - v_mov_b64_e32 v[208:209], 0 - v_mov_b64_e32 v[210:211], 0 - v_mov_b64_e32 v[212:213], 0 - v_mov_b64_e32 v[214:215], 0 - v_mov_b64_e32 v[216:217], 0 - v_mov_b64_e32 v[218:219], 0 - v_mov_b64_e32 v[220:221], 0 - v_mov_b64_e32 v[222:223], 0 - s_branch label_7DF0 -label_8EF4: - s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) - s_endpgm -label_8EFC: - s_waitcnt vmcnt(0) - s_waitcnt lgkmcnt(0) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_bfe_u32 v230, v230, v25, 8 - v_and_b32_e32 v231, v47, v231 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_bfe_u32 v234, v234, v25, 8 - v_and_b32_e32 v235, v47, v235 - s_and_b32 s77, s88, 3 - s_mul_i32 s75, s77, 0x4100 - v_add_u32_e32 v30, s75, v13 - ds_read_b128 a[56:59], v30 offset:8320 - ds_read_b128 a[60:63], v30 offset:8336 - ds_read_b128 a[88:91], v30 offset:10400 - ds_read_b128 a[92:95], v30 offset:10416 - v_add_u32_e32 v30, s75, v241 - ds_read_b128 a[64:67], v30 offset:8384 - ds_read_b128 a[68:71], v30 offset:8400 - ds_read_b128 a[96:99], v30 offset:10464 - ds_read_b128 a[100:103], v30 offset:10480 - v_add_u32_e32 v31, s75, v239 - ds_read_b128 a[40:43], v31 offset:8192 - ds_read_b128 a[44:47], v31 offset:8208 - ds_read_b128 a[72:75], v31 offset:10272 - ds_read_b128 a[76:79], v31 offset:10288 - v_add_u32_e32 v31, s75, v238 - v_add_u32_e32 v32, s75, v1 - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - ds_write_b128 v11, v[26:29] offset:14640 - s_mul_i32 s76, s77, 0x1040 - v_add_u32_e32 v31, s76, v16 - v_mov_b64_e32 v[26:27], 0 - v_mov_b64_e32 v[28:29], 0 - v_and_b32_e32 v30, 15, v0 - s_and_b32 s75, s70, 0xff - s_sub_i32 s76, s75, 0 - v_cmp_le_i32_e32 vcc, s76, v30 - s_nop 4 - s_mov_b64 exec, vcc - ds_write_b128 v11, v[26:29] - ds_write_b128 v11, v[26:29] offset:16 - ds_write_b128 v11, v[26:29] offset:32 - ds_write_b128 v11, v[26:29] offset:48 - ds_write_b128 v31, v[26:29] - ds_write_b128 v31, v[26:29] offset:16 - s_mov_b64 exec, -1 - s_sub_i32 s76, s75, 16 - v_cmp_le_i32_e32 vcc, s76, v30 - s_nop 4 - s_mov_b64 exec, vcc - ds_write_b128 v11, v[26:29] offset:14592 - ds_write_b128 v11, v[26:29] offset:14608 - ds_write_b128 v11, v[26:29] offset:14624 - ds_write_b128 v11, v[26:29] offset:14640 - ds_write_b128 v31, v[26:29] offset:2080 - ds_write_b128 v31, v[26:29] offset:2096 - s_mov_b64 exec, -1 - s_cmp_eq_i32 s7, 3 - s_cbranch_scc1 label_97C0 - s_waitcnt lgkmcnt(0) - s_waitcnt vmcnt(0) - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - s_waitcnt lgkmcnt(0) - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - v_mov_b32_e32 v31, 0xff800000 - s_and_b32 s75, s70, 0xff - v_mov_b32_e32 v30, s75 - v_lshrrev_b32_e32 v26, 4, v0 - v_mul_i32_i24_e32 v26, 4, v26 - v_add_u32_e32 v27, 1, v26 - v_add_u32_e32 v28, 2, v26 - v_add_u32_e32 v29, 3, v26 - v_cmp_lt_u32_e64 s[64:65], v26, v30 - v_add_u32_e32 v26, 16, v26 - s_nop 0 - v_cndmask_b32_e64 v48, v31, v48, s[64:65] - v_cmp_lt_u32_e64 s[64:65], v27, v30 - v_add_u32_e32 v27, 16, v27 - s_nop 0 - v_cndmask_b32_e64 v49, v31, v49, s[64:65] - v_cmp_lt_u32_e64 s[64:65], v28, v30 - v_add_u32_e32 v28, 16, v28 - s_nop 0 - v_cndmask_b32_e64 v50, v31, v50, s[64:65] - v_cmp_lt_u32_e64 s[64:65], v29, v30 - v_add_u32_e32 v29, 16, v29 - s_nop 0 - v_cndmask_b32_e64 v51, v31, v51, s[64:65] - v_cmp_lt_u32_e64 s[64:65], v26, v30 - v_add_u32_e32 v26, 16, v26 - s_nop 0 - v_cndmask_b32_e64 v52, v31, v52, s[64:65] - v_cmp_lt_u32_e64 s[64:65], v27, v30 - v_add_u32_e32 v27, 16, v27 - s_nop 0 - v_cndmask_b32_e64 v53, v31, v53, s[64:65] - v_cmp_lt_u32_e64 s[64:65], v28, v30 - v_add_u32_e32 v28, 16, v28 - s_nop 0 - v_cndmask_b32_e64 v54, v31, v54, s[64:65] - v_cmp_lt_u32_e64 s[64:65], v29, v30 - v_add_u32_e32 v29, 16, v29 - s_nop 0 - v_cndmask_b32_e64 v55, v31, v55, s[64:65] - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - v_exp_f32_e32 v50, v50 - v_exp_f32_e32 v51, v51 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_add_f32_e32 v26, v55, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - s_cmp_eq_u32 s72, 64 - s_cbranch_scc1 label_9604 - s_cmp_eq_u32 s72, 16 - s_cbranch_scc0 label_9590 - ds_write_b32 v17, v33 offset:4096 - ds_write_b64 v20, v[48:49] - ds_write_b64 v20, v[50:51] offset:2048 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b64 v[48:49], v18 - ds_read_b64 v[50:51], v19 - ds_read_b32 v34, v43 offset:4096 - s_waitcnt lgkmcnt(0) - s_barrier - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_cbranch_vccz label_958C - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_958C: - s_branch label_96AC -label_9590: - ds_write_b32 v17, v33 offset:4096 - ds_write_b64 v20, v[48:49] - ds_write_b64 v20, v[50:51] offset:2048 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b64 v[48:49], v18 - ds_read_b64 v[50:51], v19 - ds_read_b64 v[52:53], v18 offset:512 - ds_read_b64 v[54:55], v19 offset:512 - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - s_waitcnt lgkmcnt(0) - s_barrier - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_cbranch_vccz label_9600 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_9600: - s_branch label_96AC -label_9604: - ds_write_b32 v17, v33 offset:4096 - ds_write_b64 v20, v[48:49] - ds_write_b64 v20, v[50:51] offset:2048 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b64 v[48:49], v18 - ds_read_b64 v[50:51], v19 - ds_read_b64 v[52:53], v18 offset:512 - ds_read_b64 v[54:55], v19 offset:512 - ds_read_b64 v[56:57], v18 offset:1024 - ds_read_b64 v[58:59], v19 offset:1024 - ds_read_b64 v[60:61], v18 offset:1536 - ds_read_b64 v[62:63], v19 offset:1536 - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b32 v36, v43 offset:4608 - ds_read_b32 v37, v43 offset:4864 - s_waitcnt lgkmcnt(0) - s_barrier - v_min_f32_e32 v26, v34, v35 - v_min_f32_e32 v26, v26, v36 - v_min_f32_e32 v26, v26, v37 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_cbranch_vccz label_96AC - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9DB4 -label_96AC: - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - s_cmp_le_u32 s72, 16 - s_cbranch_scc1 label_97BC - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - s_cmp_le_u32 s72, 32 - s_cbranch_scc1 label_97BC - v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] - v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] - v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] - v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] - v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] - v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] - v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] - v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] - v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] - v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] - v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] - v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] - v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] - v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] - v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] - v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] -label_97BC: - s_branch label_7CF4 -label_97C0: - s_and_b32 s75, s88, 3 - s_mul_i32 s75, s75, 0x1040 - v_add_u32_e32 v23, s75, v23 - v_add_u32_e32 v24, s75, v24 - s_waitcnt lgkmcnt(0) - s_waitcnt vmcnt(0) - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 - ds_read_b64_tr_b16 a[138:139], v23 offset:128 - ds_read_b64_tr_b16 a[140:141], v24 - ds_read_b64_tr_b16 a[142:143], v24 offset:128 - ds_read_b64_tr_b16 a[144:145], v23 offset:512 - ds_read_b64_tr_b16 a[146:147], v23 offset:640 - ds_read_b64_tr_b16 a[148:149], v24 offset:512 - ds_read_b64_tr_b16 a[150:151], v24 offset:640 - s_waitcnt lgkmcnt(0) - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[40:47], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[72:79], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - v_mov_b32_e32 v31, 0xff800000 - s_and_b32 s75, s70, 0xff - v_mov_b32_e32 v30, s75 - v_lshrrev_b32_e32 v26, 4, v0 - v_mul_i32_i24_e32 v26, 4, v26 - v_add_u32_e32 v27, 1, v26 - v_add_u32_e32 v28, 2, v26 - v_add_u32_e32 v29, 3, v26 - v_cmp_lt_u32_e64 s[64:65], v26, v30 - v_add_u32_e32 v26, 16, v26 - s_nop 0 - v_cndmask_b32_e64 v48, v31, v48, s[64:65] - v_cmp_lt_u32_e64 s[64:65], v27, v30 - v_add_u32_e32 v27, 16, v27 - s_nop 0 - v_cndmask_b32_e64 v49, v31, v49, s[64:65] - v_cmp_lt_u32_e64 s[64:65], v28, v30 - v_add_u32_e32 v28, 16, v28 - s_nop 0 - v_cndmask_b32_e64 v50, v31, v50, s[64:65] - v_cmp_lt_u32_e64 s[64:65], v29, v30 - v_add_u32_e32 v29, 16, v29 - s_nop 0 - v_cndmask_b32_e64 v51, v31, v51, s[64:65] - v_cmp_lt_u32_e64 s[64:65], v26, v30 - v_add_u32_e32 v26, 16, v26 - s_nop 0 - v_cndmask_b32_e64 v52, v31, v52, s[64:65] - v_cmp_lt_u32_e64 s[64:65], v27, v30 - v_add_u32_e32 v27, 16, v27 - s_nop 0 - v_cndmask_b32_e64 v53, v31, v53, s[64:65] - v_cmp_lt_u32_e64 s[64:65], v28, v30 - v_add_u32_e32 v28, 16, v28 - s_nop 0 - v_cndmask_b32_e64 v54, v31, v54, s[64:65] - v_cmp_lt_u32_e64 s[64:65], v29, v30 - v_add_u32_e32 v29, 16, v29 - s_nop 0 - v_cndmask_b32_e64 v55, v31, v55, s[64:65] - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - v_exp_f32_e32 v50, v50 - v_exp_f32_e32 v51, v51 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_add_f32_e32 v26, v55, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - s_cmp_eq_u32 s72, 64 - s_cbranch_scc1 label_9BF8 - s_cmp_eq_u32 s72, 16 - s_cbranch_scc0 label_9B84 - ds_write_b32 v17, v33 offset:4096 - ds_write_b64 v20, v[48:49] - ds_write_b64 v20, v[50:51] offset:2048 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b64 v[48:49], v18 - ds_read_b64 v[50:51], v19 - ds_read_b32 v34, v43 offset:4096 - s_waitcnt lgkmcnt(0) - s_barrier - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_cbranch_vccz label_9B80 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_9B80: - s_branch label_9CA0 -label_9B84: - ds_write_b32 v17, v33 offset:4096 - ds_write_b64 v20, v[48:49] - ds_write_b64 v20, v[50:51] offset:2048 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b64 v[48:49], v18 - ds_read_b64 v[50:51], v19 - ds_read_b64 v[52:53], v18 offset:512 - ds_read_b64 v[54:55], v19 offset:512 - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - s_waitcnt lgkmcnt(0) - s_barrier - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_cbranch_vccz label_9BF4 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_9BF4: - s_branch label_9CA0 -label_9BF8: - ds_write_b32 v17, v33 offset:4096 - ds_write_b64 v20, v[48:49] - ds_write_b64 v20, v[50:51] offset:2048 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b64 v[48:49], v18 - ds_read_b64 v[50:51], v19 - ds_read_b64 v[52:53], v18 offset:512 - ds_read_b64 v[54:55], v19 offset:512 - ds_read_b64 v[56:57], v18 offset:1024 - ds_read_b64 v[58:59], v19 offset:1024 - ds_read_b64 v[60:61], v18 offset:1536 - ds_read_b64 v[62:63], v19 offset:1536 - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b32 v36, v43 offset:4608 - ds_read_b32 v37, v43 offset:4864 - s_waitcnt lgkmcnt(0) - s_barrier - v_min_f32_e32 v26, v34, v35 - v_min_f32_e32 v26, v26, v36 - v_min_f32_e32 v26, v26, v37 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_cbranch_vccz label_9CA0 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9DB4 -label_9CA0: - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - s_cmp_le_u32 s72, 16 - s_cbranch_scc1 label_9DB0 - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - s_cmp_le_u32 s72, 32 - s_cbranch_scc1 label_9DB0 - v_mfma_f32_16x16x32_bf16 v[160:163], a[120:123], v[56:59], v[160:163] - v_mfma_f32_16x16x32_bf16 v[164:167], a[124:127], v[56:59], v[164:167] - v_mfma_f32_16x16x32_bf16 v[168:171], a[128:131], v[56:59], v[168:171] - v_mfma_f32_16x16x32_bf16 v[172:175], a[132:135], v[56:59], v[172:175] - v_mfma_f32_16x16x32_bf16 v[176:179], a[136:139], v[56:59], v[176:179] - v_mfma_f32_16x16x32_bf16 v[180:183], a[140:143], v[56:59], v[180:183] - v_mfma_f32_16x16x32_bf16 v[184:187], a[144:147], v[56:59], v[184:187] - v_mfma_f32_16x16x32_bf16 v[188:191], a[148:151], v[56:59], v[188:191] - v_mfma_f32_16x16x32_bf16 v[192:195], a[120:123], v[60:63], v[192:195] - v_mfma_f32_16x16x32_bf16 v[196:199], a[124:127], v[60:63], v[196:199] - v_mfma_f32_16x16x32_bf16 v[200:203], a[128:131], v[60:63], v[200:203] - v_mfma_f32_16x16x32_bf16 v[204:207], a[132:135], v[60:63], v[204:207] - v_mfma_f32_16x16x32_bf16 v[208:211], a[136:139], v[60:63], v[208:211] - v_mfma_f32_16x16x32_bf16 v[212:215], a[140:143], v[60:63], v[212:215] - v_mfma_f32_16x16x32_bf16 v[216:219], a[144:147], v[60:63], v[216:219] - v_mfma_f32_16x16x32_bf16 v[220:223], a[148:151], v[60:63], v[220:223] -label_9DB0: - s_branch label_7CF4 -label_9DB4: - v_pk_mul_f32 v[192:193], v[36:37], v[192:193] op_sel:[1,0] - v_pk_mul_f32 v[194:195], v[36:37], v[194:195] op_sel:[1,0] - v_pk_mul_f32 v[196:197], v[36:37], v[196:197] op_sel:[1,0] - v_pk_mul_f32 v[198:199], v[36:37], v[198:199] op_sel:[1,0] - v_pk_mul_f32 v[200:201], v[36:37], v[200:201] op_sel:[1,0] - v_pk_mul_f32 v[202:203], v[36:37], v[202:203] op_sel:[1,0] - v_pk_mul_f32 v[204:205], v[36:37], v[204:205] op_sel:[1,0] - v_pk_mul_f32 v[206:207], v[36:37], v[206:207] op_sel:[1,0] - v_pk_mul_f32 v[208:209], v[36:37], v[208:209] op_sel:[1,0] - v_pk_mul_f32 v[210:211], v[36:37], v[210:211] op_sel:[1,0] - v_pk_mul_f32 v[212:213], v[36:37], v[212:213] op_sel:[1,0] - v_pk_mul_f32 v[214:215], v[36:37], v[214:215] op_sel:[1,0] - v_pk_mul_f32 v[216:217], v[36:37], v[216:217] op_sel:[1,0] - v_pk_mul_f32 v[218:219], v[36:37], v[218:219] op_sel:[1,0] - v_pk_mul_f32 v[220:221], v[36:37], v[220:221] op_sel:[1,0] - v_pk_mul_f32 v[222:223], v[36:37], v[222:223] op_sel:[1,0] - v_pk_mul_f32 v[160:161], v[36:37], v[160:161] op_sel_hi:[0,1] - v_pk_mul_f32 v[162:163], v[36:37], v[162:163] op_sel_hi:[0,1] - v_pk_mul_f32 v[164:165], v[36:37], v[164:165] op_sel_hi:[0,1] - v_pk_mul_f32 v[166:167], v[36:37], v[166:167] op_sel_hi:[0,1] - v_pk_mul_f32 v[168:169], v[36:37], v[168:169] op_sel_hi:[0,1] - v_pk_mul_f32 v[170:171], v[36:37], v[170:171] op_sel_hi:[0,1] - v_pk_mul_f32 v[172:173], v[36:37], v[172:173] op_sel_hi:[0,1] - v_pk_mul_f32 v[174:175], v[36:37], v[174:175] op_sel_hi:[0,1] - v_pk_mul_f32 v[176:177], v[36:37], v[176:177] op_sel_hi:[0,1] - v_pk_mul_f32 v[178:179], v[36:37], v[178:179] op_sel_hi:[0,1] - v_pk_mul_f32 v[180:181], v[36:37], v[180:181] op_sel_hi:[0,1] - v_pk_mul_f32 v[182:183], v[36:37], v[182:183] op_sel_hi:[0,1] - v_pk_mul_f32 v[184:185], v[36:37], v[184:185] op_sel_hi:[0,1] - v_pk_mul_f32 v[186:187], v[36:37], v[186:187] op_sel_hi:[0,1] - v_pk_mul_f32 v[188:189], v[36:37], v[188:189] op_sel_hi:[0,1] - v_pk_mul_f32 v[190:191], v[36:37], v[190:191] op_sel_hi:[0,1] -label_9EB4: - v_pk_mul_f32 v[128:129], v[34:35], v[128:129] op_sel:[1,0] - v_pk_mul_f32 v[130:131], v[34:35], v[130:131] op_sel:[1,0] - v_pk_mul_f32 v[132:133], v[34:35], v[132:133] op_sel:[1,0] - v_pk_mul_f32 v[134:135], v[34:35], v[134:135] op_sel:[1,0] - v_pk_mul_f32 v[136:137], v[34:35], v[136:137] op_sel:[1,0] - v_pk_mul_f32 v[138:139], v[34:35], v[138:139] op_sel:[1,0] - v_pk_mul_f32 v[140:141], v[34:35], v[140:141] op_sel:[1,0] - v_pk_mul_f32 v[142:143], v[34:35], v[142:143] op_sel:[1,0] - v_pk_mul_f32 v[144:145], v[34:35], v[144:145] op_sel:[1,0] - v_pk_mul_f32 v[146:147], v[34:35], v[146:147] op_sel:[1,0] - v_pk_mul_f32 v[148:149], v[34:35], v[148:149] op_sel:[1,0] - v_pk_mul_f32 v[150:151], v[34:35], v[150:151] op_sel:[1,0] - v_pk_mul_f32 v[152:153], v[34:35], v[152:153] op_sel:[1,0] - v_pk_mul_f32 v[154:155], v[34:35], v[154:155] op_sel:[1,0] - v_pk_mul_f32 v[156:157], v[34:35], v[156:157] op_sel:[1,0] - v_pk_mul_f32 v[158:159], v[34:35], v[158:159] op_sel:[1,0] -label_9F34: - v_pk_mul_f32 v[96:97], v[34:35], v[96:97] op_sel_hi:[0,1] - v_pk_mul_f32 v[98:99], v[34:35], v[98:99] op_sel_hi:[0,1] - v_pk_mul_f32 v[100:101], v[34:35], v[100:101] op_sel_hi:[0,1] - v_pk_mul_f32 v[102:103], v[34:35], v[102:103] op_sel_hi:[0,1] - v_pk_mul_f32 v[104:105], v[34:35], v[104:105] op_sel_hi:[0,1] - v_pk_mul_f32 v[106:107], v[34:35], v[106:107] op_sel_hi:[0,1] - v_pk_mul_f32 v[108:109], v[34:35], v[108:109] op_sel_hi:[0,1] - v_pk_mul_f32 v[110:111], v[34:35], v[110:111] op_sel_hi:[0,1] - v_pk_mul_f32 v[112:113], v[34:35], v[112:113] op_sel_hi:[0,1] - v_pk_mul_f32 v[114:115], v[34:35], v[114:115] op_sel_hi:[0,1] - v_pk_mul_f32 v[116:117], v[34:35], v[116:117] op_sel_hi:[0,1] - v_pk_mul_f32 v[118:119], v[34:35], v[118:119] op_sel_hi:[0,1] - v_pk_mul_f32 v[120:121], v[34:35], v[120:121] op_sel_hi:[0,1] - v_pk_mul_f32 v[122:123], v[34:35], v[122:123] op_sel_hi:[0,1] - v_pk_mul_f32 v[124:125], v[34:35], v[124:125] op_sel_hi:[0,1] - v_pk_mul_f32 v[126:127], v[34:35], v[126:127] op_sel_hi:[0,1] - s_setpc_b64 vcc -label_9FB8: - s_cmp_eq_u32 s88, 8 - s_cselect_b32 s69, 1, 0 - s_cmp_eq_u32 s70, 0 - s_cselect_b32 s69, s69, 0 - s_cmp_lt_u32 s7, 1 - s_cbranch_scc1 label_A02C - s_cmp_eq_u32 s70, 0 - s_cbranch_scc1 label_A018 - v_mov_b64_e32 v[48:49], 0 - v_mov_b64_e32 v[50:51], 0 - v_mov_b64_e32 v[52:53], 0 - v_mov_b64_e32 v[54:55], 0 - v_mov_b64_e32 v[56:57], 0 - v_mov_b64_e32 v[58:59], 0 - v_mov_b64_e32 v[60:61], 0 - v_mov_b64_e32 v[62:63], 0 - v_mov_b64_e32 v[64:65], 0 - v_mov_b64_e32 v[66:67], 0 - v_mov_b64_e32 v[68:69], 0 - v_mov_b64_e32 v[70:71], 0 - v_mov_b64_e32 v[72:73], 0 - v_mov_b64_e32 v[74:75], 0 - v_mov_b64_e32 v[76:77], 0 - v_mov_b64_e32 v[78:79], 0 -label_A018: - s_cmp_eq_u32 s7, 1 - s_cbranch_scc1 label_BBC0 - s_cmp_eq_u32 s7, 2 - s_cbranch_scc1 label_CC74 - s_branch label_DD28 -label_A02C: - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v39, 0, v39 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s56 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[50:51], v19 - v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] - buffer_load_dword v41, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] - buffer_load_dword v244, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] - s_cbranch_vccz label_A2A0 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_A2A0: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v64, v65, v64 - v_max3_f32 v30, v66, v67, v30 - v_max3_f32 v30, v68, v69, v30 - v_max3_f32 v30, v70, v71, v30 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v242, 0, v242 - v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s60 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_waitcnt vmcnt(12) - s_barrier - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - s_nop 1 - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:41472 - s_nop 1 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - s_nop 1 - ds_read_b128 a[48:51], v240 offset:41472 - ds_read_b128 a[52:55], v240 offset:41488 - ds_read_u8 v236, v238 offset:41472 - s_nop 1 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - s_nop 1 - ds_read_b128 a[80:83], v240 offset:43552 - ds_read_b128 a[84:87], v240 offset:43568 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:43552 - s_nop 1 - v_fma_f32 v64, v64, s5, -v32 - v_fma_f32 v65, v65, s5, -v32 - v_fma_f32 v66, v66, s5, -v32 - v_fma_f32 v67, v67, s5, -v32 - v_fma_f32 v68, v68, s5, -v32 - v_fma_f32 v69, v69, s5, -v32 - v_fma_f32 v70, v70, s5, -v32 - v_fma_f32 v71, v71, s5, -v32 - s_nop 1 - ds_read_b128 a[56:59], v13 offset:41600 - ds_read_b128 a[60:63], v13 offset:41616 - v_exp_f32_e32 v64, v64 - v_exp_f32_e32 v65, v65 - v_exp_f32_e32 v66, v66 - ds_read_u8 v237, v238 offset:43552 - s_nop 1 - v_exp_f32_e32 v67, v67 - v_exp_f32_e32 v68, v68 - v_exp_f32_e32 v69, v69 - v_exp_f32_e32 v70, v70 - v_exp_f32_e32 v71, v71 - v_mul_f32_e32 v46, v33, v46 - s_nop 1 - ds_read_b128 a[88:91], v13 offset:43680 - ds_read_b128 a[92:95], v13 offset:43696 - v_add_f32_e32 v26, v65, v64 - v_add_f32_e32 v26, v66, v26 - v_add_f32_e32 v26, v67, v26 - v_add_f32_e32 v26, v68, v26 - s_nop 1 - v_add_f32_e32 v26, v69, v26 - v_add_f32_e32 v26, v70, v26 - v_add_f32_e32 v26, v71, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v64, v64, v65 - v_cvt_pk_bf16_f32 v65, v66, v67 - v_cvt_pk_bf16_f32 v66, v68, v69 - v_cvt_pk_bf16_f32 v67, v70, v71 - s_nop 1 - ds_read_b128 a[64:67], v241 offset:41664 - ds_read_b128 a[68:71], v241 offset:41680 - ds_write_b32 v17, v33 offset:4096 - s_nop 1 - s_nop 1 - ds_read_b128 a[96:99], v241 offset:43744 - ds_read_b128 a[100:103], v241 offset:43760 - ds_write_b64 v20, v[64:65] - s_nop 1 - s_nop 1 - ds_read_b128 a[104:107], v16 offset:8320 - ds_read_b128 a[108:111], v16 offset:8336 - ds_write_b64 v20, v[66:67] offset:2048 - s_nop 1 - s_nop 1 - ds_read_b128 a[112:115], v16 offset:10400 - ds_read_b128 a[116:119], v16 offset:10416 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v40, 0, v40 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s57 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[66:67], v19 - v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - buffer_load_dword v42, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - buffer_load_dword v245, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - s_cbranch_vccz label_A970 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_A970: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v243, 0, v243 - v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s61 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_waitcnt vmcnt(11) - s_barrier - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - s_nop 1 - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:58112 - s_nop 1 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - s_nop 1 - ds_read_b128 a[48:51], v240 offset:58112 - ds_read_b128 a[52:55], v240 offset:58128 - ds_read_u8 v236, v238 offset:58112 - s_nop 1 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - s_nop 1 - ds_read_b128 a[80:83], v240 offset:60192 - ds_read_b128 a[84:87], v240 offset:60208 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:60192 - s_nop 1 - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - s_nop 1 - ds_read_b128 a[56:59], v13 offset:58240 - ds_read_b128 a[60:63], v13 offset:58256 - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - v_exp_f32_e32 v50, v50 - ds_read_u8 v237, v238 offset:60192 - s_nop 1 - v_exp_f32_e32 v51, v51 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - s_nop 1 - ds_read_b128 a[88:91], v13 offset:60320 - ds_read_b128 a[92:95], v13 offset:60336 - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - s_nop 1 - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_add_f32_e32 v26, v55, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - s_nop 1 - ds_read_b128 a[64:67], v241 offset:58304 - ds_read_b128 a[68:71], v241 offset:58320 - ds_write_b32 v17, v33 offset:4096 - s_nop 1 - s_nop 1 - ds_read_b128 a[96:99], v241 offset:60384 - ds_read_b128 a[100:103], v241 offset:60400 - ds_write_b64 v20, v[48:49] - s_nop 1 - s_nop 1 - ds_read_b128 a[104:107], v16 offset:12480 - ds_read_b128 a[108:111], v16 offset:12496 - ds_write_b64 v20, v[50:51] offset:2048 - s_nop 1 - s_nop 1 - ds_read_b128 a[112:115], v16 offset:14560 - ds_read_b128 a[116:119], v16 offset:14576 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v41, 0, v41 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s58 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[50:51], v19 - v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_AEF4 - buffer_load_dword v39, v38, s[24:27], 0 offen -label_AEF4: - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_AF1C - buffer_load_dword v242, v44, s[24:27], 0 offen -label_AF1C: - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_B040 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 -label_B040: - v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] - s_cbranch_vccz label_B058 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_B058: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v64, v65, v64 - v_max3_f32 v30, v66, v67, v30 - v_max3_f32 v30, v68, v69, v30 - v_max3_f32 v30, v70, v71, v30 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v244, 0, v244 - v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s62 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_cmp_eq_u32 s69, 0 - s_cbranch_scc1 label_B26C - s_waitcnt vmcnt(9) -label_B26C: - s_waitcnt vmcnt(12) - s_barrier - ds_read_b128 v[80:83], v10 offset:8192 - ds_read_b128 v[84:87], v10 offset:8208 - s_nop 1 - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:10272 - ds_read_b128 v[92:95], v10 offset:10288 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:8192 - s_nop 1 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - s_nop 1 - ds_read_b128 a[48:51], v240 offset:8192 - ds_read_b128 a[52:55], v240 offset:8208 - ds_read_u8 v236, v238 offset:8192 - s_nop 1 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - s_nop 1 - ds_read_b128 a[80:83], v240 offset:10272 - ds_read_b128 a[84:87], v240 offset:10288 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:10272 - s_nop 1 - v_fma_f32 v64, v64, s5, -v32 - v_fma_f32 v65, v65, s5, -v32 - v_fma_f32 v66, v66, s5, -v32 - v_fma_f32 v67, v67, s5, -v32 - v_fma_f32 v68, v68, s5, -v32 - v_fma_f32 v69, v69, s5, -v32 - v_fma_f32 v70, v70, s5, -v32 - v_fma_f32 v71, v71, s5, -v32 - s_nop 1 - ds_read_b128 a[56:59], v13 offset:8320 - ds_read_b128 a[60:63], v13 offset:8336 - v_exp_f32_e32 v64, v64 - v_exp_f32_e32 v65, v65 - v_exp_f32_e32 v66, v66 - ds_read_u8 v237, v238 offset:10272 - s_nop 1 - v_exp_f32_e32 v67, v67 - v_exp_f32_e32 v68, v68 - v_exp_f32_e32 v69, v69 - v_exp_f32_e32 v70, v70 - v_exp_f32_e32 v71, v71 - v_mul_f32_e32 v46, v33, v46 - s_nop 1 - ds_read_b128 a[88:91], v13 offset:10400 - ds_read_b128 a[92:95], v13 offset:10416 - v_add_f32_e32 v26, v65, v64 - v_add_f32_e32 v26, v66, v26 - v_add_f32_e32 v26, v67, v26 - v_add_f32_e32 v26, v68, v26 - s_nop 1 - v_add_f32_e32 v26, v69, v26 - v_add_f32_e32 v26, v70, v26 - v_add_f32_e32 v26, v71, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v64, v64, v65 - v_cvt_pk_bf16_f32 v65, v66, v67 - v_cvt_pk_bf16_f32 v66, v68, v69 - v_cvt_pk_bf16_f32 v67, v70, v71 - s_nop 1 - ds_read_b128 a[64:67], v241 offset:8384 - ds_read_b128 a[68:71], v241 offset:8400 - ds_write_b32 v17, v33 offset:4096 - s_nop 1 - s_nop 1 - ds_read_b128 a[96:99], v241 offset:10464 - ds_read_b128 a[100:103], v241 offset:10480 - ds_write_b64 v20, v[64:65] - s_nop 1 - s_nop 1 - ds_read_b128 a[104:107], v16 - ds_read_b128 a[108:111], v16 offset:16 - ds_write_b64 v20, v[66:67] offset:2048 - s_nop 1 - s_nop 1 - ds_read_b128 a[112:115], v16 offset:2080 - ds_read_b128 a[116:119], v16 offset:2096 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v42, 0, v42 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s59 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[66:67], v19 - v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_B5E8 - buffer_load_dword v40, v38, s[24:27], 0 offen -label_B5E8: - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_B610 - buffer_load_dword v243, v44, s[24:27], 0 offen -label_B610: - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_B734 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 -label_B734: - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - s_cbranch_vccz label_B74C - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_B74C: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v245, 0, v245 - v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s63 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_cmp_eq_u32 s69, 0 - s_cbranch_scc1 label_B960 - s_waitcnt vmcnt(10) -label_B960: - s_waitcnt vmcnt(12) - s_barrier - ds_read_b128 v[80:83], v10 offset:24832 - ds_read_b128 v[84:87], v10 offset:24848 - s_nop 1 - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:26912 - ds_read_b128 v[92:95], v10 offset:26928 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:24832 - s_nop 1 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - s_nop 1 - ds_read_b128 a[48:51], v240 offset:24832 - ds_read_b128 a[52:55], v240 offset:24848 - ds_read_u8 v236, v238 offset:24832 - s_nop 1 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - s_nop 1 - ds_read_b128 a[80:83], v240 offset:26912 - ds_read_b128 a[84:87], v240 offset:26928 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:26912 - s_nop 1 - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - s_nop 1 - ds_read_b128 a[56:59], v13 offset:24960 - ds_read_b128 a[60:63], v13 offset:24976 - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - v_exp_f32_e32 v50, v50 - ds_read_u8 v237, v238 offset:26912 - s_nop 1 - v_exp_f32_e32 v51, v51 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - s_nop 1 - ds_read_b128 a[88:91], v13 offset:27040 - ds_read_b128 a[92:95], v13 offset:27056 - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - s_nop 1 - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_add_f32_e32 v26, v55, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - s_nop 1 - ds_read_b128 a[64:67], v241 offset:25024 - ds_read_b128 a[68:71], v241 offset:25040 - ds_write_b32 v17, v33 offset:4096 - s_nop 1 - s_nop 1 - ds_read_b128 a[96:99], v241 offset:27104 - ds_read_b128 a[100:103], v241 offset:27120 - ds_write_b64 v20, v[48:49] - s_nop 1 - s_nop 1 - ds_read_b128 a[104:107], v16 offset:4160 - ds_read_b128 a[108:111], v16 offset:4176 - ds_write_b64 v20, v[50:51] offset:2048 - s_nop 1 - s_nop 1 - ds_read_b128 a[112:115], v16 offset:6240 - ds_read_b128 a[116:119], v16 offset:6256 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_13644 - s_branch label_A02C -label_BBC0: - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v39, 0, v39 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s56 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[50:51], v19 - v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - buffer_load_dword v41, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - buffer_load_dword v244, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - s_nop 0 - s_cbranch_vccz label_BD58 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_BD58: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v242, 0, v242 - v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s60 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_waitcnt vmcnt(12) - s_barrier - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:41472 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:43552 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v40, 0, v40 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s57 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[66:67], v19 - v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - buffer_load_dword v42, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - buffer_load_dword v245, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - s_nop 0 - s_cbranch_vccz label_C170 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_C170: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v243, 0, v243 - v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s61 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_waitcnt vmcnt(11) - s_barrier - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:58112 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:60192 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v41, 0, v41 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s58 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[50:51], v19 - v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_C4B0 - buffer_load_dword v39, v38, s[24:27], 0 offen -label_C4B0: - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_C4D0 - buffer_load_dword v242, v44, s[24:27], 0 offen -label_C4D0: - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_C58C - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 -label_C58C: - s_nop 0 - s_cbranch_vccz label_C5A0 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_C5A0: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v244, 0, v244 - v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s62 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_cmp_eq_u32 s69, 0 - s_cbranch_scc1 label_C794 - s_waitcnt vmcnt(9) -label_C794: - s_waitcnt vmcnt(12) - s_barrier - ds_read_b128 v[80:83], v10 offset:8192 - ds_read_b128 v[84:87], v10 offset:8208 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:10272 - ds_read_b128 v[92:95], v10 offset:10288 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:8192 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:10272 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v42, 0, v42 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s59 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[66:67], v19 - v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_C8EC - buffer_load_dword v40, v38, s[24:27], 0 offen -label_C8EC: - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_C90C - buffer_load_dword v243, v44, s[24:27], 0 offen -label_C90C: - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_C9C8 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 -label_C9C8: - s_nop 0 - s_cbranch_vccz label_C9DC - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_C9DC: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v245, 0, v245 - v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s63 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_cmp_eq_u32 s69, 0 - s_cbranch_scc1 label_CBD0 - s_waitcnt vmcnt(10) -label_CBD0: - s_waitcnt vmcnt(12) - s_barrier - ds_read_b128 v[80:83], v10 offset:24832 - ds_read_b128 v[84:87], v10 offset:24848 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:26912 - ds_read_b128 v[92:95], v10 offset:26928 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:24832 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:26912 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_14710 - s_branch label_BBC0 -label_CC74: - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v39, 0, v39 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s56 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - buffer_load_dword v41, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - buffer_load_dword v244, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_CE0C - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_CE0C: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v242, 0, v242 - v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s60 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_waitcnt vmcnt(12) - s_barrier - s_nop 1 - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:41472 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:43552 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v40, 0, v40 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s57 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - buffer_load_dword v42, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - buffer_load_dword v245, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_D224 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_D224: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v243, 0, v243 - v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s61 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_waitcnt vmcnt(11) - s_barrier - s_nop 1 - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:58112 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:60192 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v41, 0, v41 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s58 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_D564 - buffer_load_dword v39, v38, s[24:27], 0 offen -label_D564: - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_D584 - buffer_load_dword v242, v44, s[24:27], 0 offen -label_D584: - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_D630 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 -label_D630: - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_D654 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_D654: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v244, 0, v244 - v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s62 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_cmp_eq_u32 s69, 0 - s_cbranch_scc1 label_D848 - s_waitcnt vmcnt(9) -label_D848: - s_waitcnt vmcnt(12) - s_barrier - s_nop 1 - ds_read_b128 v[80:83], v10 offset:8192 - ds_read_b128 v[84:87], v10 offset:8208 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:10272 - ds_read_b128 v[92:95], v10 offset:10288 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:8192 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:10272 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v42, 0, v42 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s59 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_D9A0 - buffer_load_dword v40, v38, s[24:27], 0 offen -label_D9A0: - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_D9C0 - buffer_load_dword v243, v44, s[24:27], 0 offen -label_D9C0: - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_DA6C - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 -label_DA6C: - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_DA90 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_DA90: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v245, 0, v245 - v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s63 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_cmp_eq_u32 s69, 0 - s_cbranch_scc1 label_DC84 - s_waitcnt vmcnt(10) -label_DC84: - s_waitcnt vmcnt(12) - s_barrier - s_nop 1 - ds_read_b128 v[80:83], v10 offset:24832 - ds_read_b128 v[84:87], v10 offset:24848 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:26912 - ds_read_b128 v[92:95], v10 offset:26928 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:24832 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:26912 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_15038 - s_branch label_CC74 -label_DD28: - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v39, 0, v39 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s56 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - buffer_load_dword v41, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 - buffer_load_dword v244, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v23 offset:128 - ds_read_b64_tr_b16 a[140:141], v24 - ds_read_b64_tr_b16 a[142:143], v24 offset:128 - v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v23 offset:512 - ds_read_b64_tr_b16 a[146:147], v23 offset:640 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v24 offset:512 - ds_read_b64_tr_b16 a[150:151], v24 offset:640 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_DEC0 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_DEC0: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v242, 0, v242 - v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s60 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_waitcnt vmcnt(12) - s_barrier - s_nop 1 - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:41472 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:43552 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v40, 0, v40 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s57 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - buffer_load_dword v42, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 offset:4160 - buffer_load_dword v245, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v23 offset:4288 - ds_read_b64_tr_b16 a[140:141], v24 offset:4160 - ds_read_b64_tr_b16 a[142:143], v24 offset:4288 - v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v23 offset:4672 - ds_read_b64_tr_b16 a[146:147], v23 offset:4800 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v24 offset:4672 - ds_read_b64_tr_b16 a[150:151], v24 offset:4800 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_E2D8 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_E2D8: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v243, 0, v243 - v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s61 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_waitcnt vmcnt(11) - s_barrier - s_nop 1 - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:58112 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:60192 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v41, 0, v41 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s58 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_E618 - buffer_load_dword v39, v38, s[24:27], 0 offen -label_E618: - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 offset:8320 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_E638 - buffer_load_dword v242, v44, s[24:27], 0 offen -label_E638: - ds_read_b64_tr_b16 a[138:139], v23 offset:8448 - ds_read_b64_tr_b16 a[140:141], v24 offset:8320 - ds_read_b64_tr_b16 a[142:143], v24 offset:8448 - v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v23 offset:8832 - ds_read_b64_tr_b16 a[146:147], v23 offset:8960 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v24 offset:8832 - ds_read_b64_tr_b16 a[150:151], v24 offset:8960 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_E6E4 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 -label_E6E4: - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_E708 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_E708: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v244, 0, v244 - v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s62 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_cmp_eq_u32 s69, 0 - s_cbranch_scc1 label_E8FC - s_waitcnt vmcnt(9) -label_E8FC: - s_waitcnt vmcnt(12) - s_barrier - s_nop 1 - ds_read_b128 v[80:83], v10 offset:8192 - ds_read_b128 v[84:87], v10 offset:8208 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:10272 - ds_read_b128 v[92:95], v10 offset:10288 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:8192 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:10272 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v42, 0, v42 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s59 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_EA54 - buffer_load_dword v40, v38, s[24:27], 0 offen -label_EA54: - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 offset:12480 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_EA74 - buffer_load_dword v243, v44, s[24:27], 0 offen -label_EA74: - ds_read_b64_tr_b16 a[138:139], v23 offset:12608 - ds_read_b64_tr_b16 a[140:141], v24 offset:12480 - ds_read_b64_tr_b16 a[142:143], v24 offset:12608 - v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v23 offset:12992 - ds_read_b64_tr_b16 a[146:147], v23 offset:13120 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v24 offset:12992 - ds_read_b64_tr_b16 a[150:151], v24 offset:13120 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_EB20 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 -label_EB20: - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_EB44 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_EB44: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(17) - v_max_i32_e32 v245, 0, v245 - v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] - s_nop 1 - s_mov_b32 m0, s63 - global_load_lds_dwordx4 v[2:3], off - s_nop 1 - s_cmp_eq_u32 s69, 0 - s_cbranch_scc1 label_ED38 - s_waitcnt vmcnt(10) -label_ED38: - s_waitcnt vmcnt(12) - s_barrier - s_nop 1 - ds_read_b128 v[80:83], v10 offset:24832 - ds_read_b128 v[84:87], v10 offset:24848 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:26912 - ds_read_b128 v[92:95], v10 offset:26928 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:24832 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:26912 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_15960 - s_branch label_DD28 -label_EDDC: - s_cmp_eq_u32 s88, 8 - s_cselect_b32 s69, 1, 0 - s_cmp_eq_u32 s70, 0 - s_cselect_b32 s69, s69, 0 - s_cmp_lt_u32 s7, 2 - s_cbranch_scc1 label_EE48 - s_cmp_eq_u32 s70, 0 - s_cbranch_scc1 label_EE3C - v_mov_b64_e32 v[48:49], 0 - v_mov_b64_e32 v[50:51], 0 - v_mov_b64_e32 v[52:53], 0 - v_mov_b64_e32 v[54:55], 0 - v_mov_b64_e32 v[56:57], 0 - v_mov_b64_e32 v[58:59], 0 - v_mov_b64_e32 v[60:61], 0 - v_mov_b64_e32 v[62:63], 0 - v_mov_b64_e32 v[64:65], 0 - v_mov_b64_e32 v[66:67], 0 - v_mov_b64_e32 v[68:69], 0 - v_mov_b64_e32 v[70:71], 0 - v_mov_b64_e32 v[72:73], 0 - v_mov_b64_e32 v[74:75], 0 - v_mov_b64_e32 v[76:77], 0 - v_mov_b64_e32 v[78:79], 0 -label_EE3C: - s_cmp_eq_u32 s7, 2 - s_cbranch_scc1 label_10ACC - s_branch label_11C70 -label_EE48: - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v39, 0, v39 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s56 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[50:51], v19 - v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] - buffer_load_dword v41, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] - buffer_load_dword v244, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[52:53], v18 offset:512 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64 v[54:55], v19 offset:512 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] - s_cbranch_vccz label_F0D8 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_F0D8: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v64, v65, v64 - v_max3_f32 v30, v66, v67, v30 - v_max3_f32 v30, v68, v69, v30 - v_max3_f32 v30, v70, v71, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v242, 0, v242 - v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - s_mov_b32 m0, s60 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_waitcnt vmcnt(12) - s_barrier - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:41472 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - s_nop 1 - ds_read_b128 a[48:51], v240 offset:41472 - ds_read_b128 a[52:55], v240 offset:41488 - ds_read_u8 v236, v238 offset:41472 - s_nop 1 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - s_nop 1 - ds_read_b128 a[80:83], v240 offset:43552 - ds_read_b128 a[84:87], v240 offset:43568 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:43552 - s_nop 1 - v_fma_f32 v64, v64, s5, -v32 - v_fma_f32 v65, v65, s5, -v32 - v_fma_f32 v66, v66, s5, -v32 - v_fma_f32 v67, v67, s5, -v32 - v_fma_f32 v68, v68, s5, -v32 - v_fma_f32 v69, v69, s5, -v32 - v_fma_f32 v70, v70, s5, -v32 - v_fma_f32 v71, v71, s5, -v32 - s_nop 1 - ds_read_b128 a[56:59], v13 offset:41600 - ds_read_b128 a[60:63], v13 offset:41616 - v_exp_f32_e32 v64, v64 - v_exp_f32_e32 v65, v65 - v_exp_f32_e32 v66, v66 - ds_read_u8 v237, v238 offset:43552 - s_nop 1 - v_exp_f32_e32 v67, v67 - v_exp_f32_e32 v68, v68 - v_exp_f32_e32 v69, v69 - v_exp_f32_e32 v70, v70 - v_exp_f32_e32 v71, v71 - v_mul_f32_e32 v46, v33, v46 - s_nop 1 - ds_read_b128 a[88:91], v13 offset:43680 - ds_read_b128 a[92:95], v13 offset:43696 - v_add_f32_e32 v26, v65, v64 - v_add_f32_e32 v26, v66, v26 - v_add_f32_e32 v26, v67, v26 - v_add_f32_e32 v26, v68, v26 - s_nop 1 - v_add_f32_e32 v26, v69, v26 - v_add_f32_e32 v26, v70, v26 - v_add_f32_e32 v26, v71, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v64, v64, v65 - v_cvt_pk_bf16_f32 v65, v66, v67 - v_cvt_pk_bf16_f32 v66, v68, v69 - v_cvt_pk_bf16_f32 v67, v70, v71 - s_nop 1 - ds_read_b128 a[64:67], v241 offset:41664 - ds_read_b128 a[68:71], v241 offset:41680 - ds_write_b32 v17, v33 offset:4096 - s_nop 1 - s_nop 1 - ds_read_b128 a[96:99], v241 offset:43744 - ds_read_b128 a[100:103], v241 offset:43760 - ds_write_b64 v20, v[64:65] - s_nop 1 - s_nop 1 - ds_read_b128 a[104:107], v16 offset:8320 - ds_read_b128 a[108:111], v16 offset:8336 - ds_write_b64 v20, v[66:67] offset:2048 - s_nop 1 - s_nop 1 - ds_read_b128 a[112:115], v16 offset:10400 - ds_read_b128 a[116:119], v16 offset:10416 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v40, 0, v40 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s57 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[66:67], v19 - v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - buffer_load_dword v42, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - buffer_load_dword v245, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[68:69], v18 offset:512 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64 v[70:71], v19 offset:512 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - s_cbranch_vccz label_F7E4 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_F7E4: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v243, 0, v243 - v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] - s_mov_b32 m0, s61 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] - s_waitcnt vmcnt(11) - s_barrier - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:58112 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - s_nop 1 - ds_read_b128 a[48:51], v240 offset:58112 - ds_read_b128 a[52:55], v240 offset:58128 - ds_read_u8 v236, v238 offset:58112 - s_nop 1 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - s_nop 1 - ds_read_b128 a[80:83], v240 offset:60192 - ds_read_b128 a[84:87], v240 offset:60208 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:60192 - s_nop 1 - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - s_nop 1 - ds_read_b128 a[56:59], v13 offset:58240 - ds_read_b128 a[60:63], v13 offset:58256 - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - v_exp_f32_e32 v50, v50 - ds_read_u8 v237, v238 offset:60192 - s_nop 1 - v_exp_f32_e32 v51, v51 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - s_nop 1 - ds_read_b128 a[88:91], v13 offset:60320 - ds_read_b128 a[92:95], v13 offset:60336 - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - s_nop 1 - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_add_f32_e32 v26, v55, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - s_nop 1 - ds_read_b128 a[64:67], v241 offset:58304 - ds_read_b128 a[68:71], v241 offset:58320 - ds_write_b32 v17, v33 offset:4096 - s_nop 1 - s_nop 1 - ds_read_b128 a[96:99], v241 offset:60384 - ds_read_b128 a[100:103], v241 offset:60400 - ds_write_b64 v20, v[48:49] - s_nop 1 - s_nop 1 - ds_read_b128 a[104:107], v16 offset:12480 - ds_read_b128 a[108:111], v16 offset:12496 - ds_write_b64 v20, v[50:51] offset:2048 - s_nop 1 - s_nop 1 - ds_read_b128 a[112:115], v16 offset:14560 - ds_read_b128 a[116:119], v16 offset:14576 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v41, 0, v41 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s58 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[50:51], v19 - v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_FD90 - buffer_load_dword v39, v38, s[24:27], 0 offen -label_FD90: - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_FDB8 - buffer_load_dword v242, v44, s[24:27], 0 offen -label_FDB8: - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[52:53], v18 offset:512 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64 v[54:55], v19 offset:512 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_FEF0 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 -label_FEF0: - v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] - s_cbranch_vccz label_FF08 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_FF08: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v64, v65, v64 - v_max3_f32 v30, v66, v67, v30 - v_max3_f32 v30, v68, v69, v30 - v_max3_f32 v30, v70, v71, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v244, 0, v244 - v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - s_mov_b32 m0, s62 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_cmp_eq_u32 s69, 0 - s_cbranch_scc1 label_10130 - s_waitcnt vmcnt(9) -label_10130: - s_waitcnt vmcnt(12) - s_barrier - ds_read_b128 v[80:83], v10 offset:8192 - ds_read_b128 v[84:87], v10 offset:8208 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - ds_read_b128 v[88:91], v10 offset:10272 - ds_read_b128 v[92:95], v10 offset:10288 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:8192 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - s_nop 1 - ds_read_b128 a[48:51], v240 offset:8192 - ds_read_b128 a[52:55], v240 offset:8208 - ds_read_u8 v236, v238 offset:8192 - s_nop 1 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - s_nop 1 - ds_read_b128 a[80:83], v240 offset:10272 - ds_read_b128 a[84:87], v240 offset:10288 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:10272 - s_nop 1 - v_fma_f32 v64, v64, s5, -v32 - v_fma_f32 v65, v65, s5, -v32 - v_fma_f32 v66, v66, s5, -v32 - v_fma_f32 v67, v67, s5, -v32 - v_fma_f32 v68, v68, s5, -v32 - v_fma_f32 v69, v69, s5, -v32 - v_fma_f32 v70, v70, s5, -v32 - v_fma_f32 v71, v71, s5, -v32 - s_nop 1 - ds_read_b128 a[56:59], v13 offset:8320 - ds_read_b128 a[60:63], v13 offset:8336 - v_exp_f32_e32 v64, v64 - v_exp_f32_e32 v65, v65 - v_exp_f32_e32 v66, v66 - ds_read_u8 v237, v238 offset:10272 - s_nop 1 - v_exp_f32_e32 v67, v67 - v_exp_f32_e32 v68, v68 - v_exp_f32_e32 v69, v69 - v_exp_f32_e32 v70, v70 - v_exp_f32_e32 v71, v71 - v_mul_f32_e32 v46, v33, v46 - s_nop 1 - ds_read_b128 a[88:91], v13 offset:10400 - ds_read_b128 a[92:95], v13 offset:10416 - v_add_f32_e32 v26, v65, v64 - v_add_f32_e32 v26, v66, v26 - v_add_f32_e32 v26, v67, v26 - v_add_f32_e32 v26, v68, v26 - s_nop 1 - v_add_f32_e32 v26, v69, v26 - v_add_f32_e32 v26, v70, v26 - v_add_f32_e32 v26, v71, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v64, v64, v65 - v_cvt_pk_bf16_f32 v65, v66, v67 - v_cvt_pk_bf16_f32 v66, v68, v69 - v_cvt_pk_bf16_f32 v67, v70, v71 - s_nop 1 - ds_read_b128 a[64:67], v241 offset:8384 - ds_read_b128 a[68:71], v241 offset:8400 - ds_write_b32 v17, v33 offset:4096 - s_nop 1 - s_nop 1 - ds_read_b128 a[96:99], v241 offset:10464 - ds_read_b128 a[100:103], v241 offset:10480 - ds_write_b64 v20, v[64:65] - s_nop 1 - s_nop 1 - ds_read_b128 a[104:107], v16 - ds_read_b128 a[108:111], v16 offset:16 - ds_write_b64 v20, v[66:67] offset:2048 - s_nop 1 - s_nop 1 - ds_read_b128 a[112:115], v16 offset:2080 - ds_read_b128 a[116:119], v16 offset:2096 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v42, 0, v42 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - s_mov_b32 m0, s59 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[66:67], v19 - v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_104C0 - buffer_load_dword v40, v38, s[24:27], 0 offen -label_104C0: - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_104E8 - buffer_load_dword v243, v44, s[24:27], 0 offen -label_104E8: - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[68:69], v18 offset:512 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64 v[70:71], v19 offset:512 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_10620 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 -label_10620: - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - s_cbranch_vccz label_10638 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_10638: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v245, 0, v245 - v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] - s_mov_b32 m0, s63 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] - s_cmp_eq_u32 s69, 0 - s_cbranch_scc1 label_10860 - s_waitcnt vmcnt(10) -label_10860: - s_waitcnt vmcnt(12) - s_barrier - ds_read_b128 v[80:83], v10 offset:24832 - ds_read_b128 v[84:87], v10 offset:24848 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] - ds_read_b128 v[88:91], v10 offset:26912 - ds_read_b128 v[92:95], v10 offset:26928 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:24832 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - s_nop 1 - ds_read_b128 a[48:51], v240 offset:24832 - ds_read_b128 a[52:55], v240 offset:24848 - ds_read_u8 v236, v238 offset:24832 - s_nop 1 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - s_nop 1 - ds_read_b128 a[80:83], v240 offset:26912 - ds_read_b128 a[84:87], v240 offset:26928 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:26912 - s_nop 1 - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - s_nop 1 - ds_read_b128 a[56:59], v13 offset:24960 - ds_read_b128 a[60:63], v13 offset:24976 - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - v_exp_f32_e32 v50, v50 - ds_read_u8 v237, v238 offset:26912 - s_nop 1 - v_exp_f32_e32 v51, v51 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - s_nop 1 - ds_read_b128 a[88:91], v13 offset:27040 - ds_read_b128 a[92:95], v13 offset:27056 - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - s_nop 1 - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_add_f32_e32 v26, v55, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - s_nop 1 - ds_read_b128 a[64:67], v241 offset:25024 - ds_read_b128 a[68:71], v241 offset:25040 - ds_write_b32 v17, v33 offset:4096 - s_nop 1 - s_nop 1 - ds_read_b128 a[96:99], v241 offset:27104 - ds_read_b128 a[100:103], v241 offset:27120 - ds_write_b64 v20, v[48:49] - s_nop 1 - s_nop 1 - ds_read_b128 a[104:107], v16 offset:4160 - ds_read_b128 a[108:111], v16 offset:4176 - ds_write_b64 v20, v[50:51] offset:2048 - s_nop 1 - s_nop 1 - ds_read_b128 a[112:115], v16 offset:6240 - ds_read_b128 a[116:119], v16 offset:6256 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_16288 - s_branch label_EE48 -label_10ACC: - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v39, 0, v39 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s56 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - buffer_load_dword v41, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - buffer_load_dword v244, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[52:53], v18 offset:512 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64 v[54:55], v19 offset:512 - v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_10C80 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_10C80: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v242, 0, v242 - v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - s_mov_b32 m0, s60 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_waitcnt vmcnt(12) - s_barrier - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:41472 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:43552 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v40, 0, v40 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s57 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - buffer_load_dword v42, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - buffer_load_dword v245, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[68:69], v18 offset:512 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64 v[70:71], v19 offset:512 - v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_110D4 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_110D4: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v243, 0, v243 - v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] - s_mov_b32 m0, s61 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] - s_waitcnt vmcnt(11) - s_barrier - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:58112 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:60192 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v41, 0, v41 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s58 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_1143C - buffer_load_dword v39, v38, s[24:27], 0 offen -label_1143C: - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_1145C - buffer_load_dword v242, v44, s[24:27], 0 offen -label_1145C: - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[52:53], v18 offset:512 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64 v[54:55], v19 offset:512 - v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_1151C - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 -label_1151C: - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_11540 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_11540: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v244, 0, v244 - v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - s_mov_b32 m0, s62 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_cmp_eq_u32 s69, 0 - s_cbranch_scc1 label_11748 - s_waitcnt vmcnt(9) -label_11748: - s_waitcnt vmcnt(12) - s_barrier - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - ds_read_b128 v[80:83], v10 offset:8192 - ds_read_b128 v[84:87], v10 offset:8208 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - ds_read_b128 v[88:91], v10 offset:10272 - ds_read_b128 v[92:95], v10 offset:10288 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:8192 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:10272 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v42, 0, v42 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s59 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_118B4 - buffer_load_dword v40, v38, s[24:27], 0 offen -label_118B4: - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_118D4 - buffer_load_dword v243, v44, s[24:27], 0 offen -label_118D4: - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[68:69], v18 offset:512 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64 v[70:71], v19 offset:512 - v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_11994 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 -label_11994: - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_119B8 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_119B8: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v245, 0, v245 - v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] - s_mov_b32 m0, s63 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] - s_cmp_eq_u32 s69, 0 - s_cbranch_scc1 label_11BC0 - s_waitcnt vmcnt(10) -label_11BC0: - s_waitcnt vmcnt(12) - s_barrier - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] - ds_read_b128 v[80:83], v10 offset:24832 - ds_read_b128 v[84:87], v10 offset:24848 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] - ds_read_b128 v[88:91], v10 offset:26912 - ds_read_b128 v[92:95], v10 offset:26928 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:24832 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:26912 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_17408 - s_branch label_10ACC -label_11C70: - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v39, 0, v39 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v39 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s56 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - buffer_load_dword v41, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 - buffer_load_dword v244, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v23 offset:128 - ds_read_b64 v[52:53], v18 offset:512 - ds_read_b64_tr_b16 a[140:141], v24 - ds_read_b64_tr_b16 a[142:143], v24 offset:128 - ds_read_b64 v[54:55], v19 offset:512 - v_mov_b32_dpp v26, v39 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v23 offset:512 - ds_read_b64_tr_b16 a[146:147], v23 offset:640 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v24 offset:512 - ds_read_b64_tr_b16 a[150:151], v24 offset:640 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v39 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v39 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_11E24 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_11E24: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v242, 0, v242 - v_mad_u64_u32 v[2:3], vcc, v242, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - s_mov_b32 m0, s60 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_waitcnt vmcnt(12) - s_barrier - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:41472 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:43552 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v40, 0, v40 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v40 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s57 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - buffer_load_dword v42, v38, s[24:27], 0 offen - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 offset:4160 - buffer_load_dword v245, v44, s[24:27], 0 offen - ds_read_b64_tr_b16 a[138:139], v23 offset:4288 - ds_read_b64 v[68:69], v18 offset:512 - ds_read_b64_tr_b16 a[140:141], v24 offset:4160 - ds_read_b64_tr_b16 a[142:143], v24 offset:4288 - ds_read_b64 v[70:71], v19 offset:512 - v_mov_b32_dpp v26, v40 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v23 offset:4672 - ds_read_b64_tr_b16 a[146:147], v23 offset:4800 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v24 offset:4672 - ds_read_b64_tr_b16 a[150:151], v24 offset:4800 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v40 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v40 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_12278 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_12278: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v243, 0, v243 - v_mad_u64_u32 v[2:3], vcc, v243, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] - s_mov_b32 m0, s61 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] - s_waitcnt vmcnt(11) - s_barrier - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:58112 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:60192 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v41, 0, v41 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s58 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_125E0 - buffer_load_dword v39, v38, s[24:27], 0 offen -label_125E0: - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 offset:8320 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_12600 - buffer_load_dword v242, v44, s[24:27], 0 offen -label_12600: - ds_read_b64_tr_b16 a[138:139], v23 offset:8448 - ds_read_b64 v[52:53], v18 offset:512 - ds_read_b64_tr_b16 a[140:141], v24 offset:8320 - ds_read_b64_tr_b16 a[142:143], v24 offset:8448 - ds_read_b64 v[54:55], v19 offset:512 - v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v23 offset:8832 - ds_read_b64_tr_b16 a[146:147], v23 offset:8960 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v24 offset:8832 - ds_read_b64_tr_b16 a[150:151], v24 offset:8960 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_126C0 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 -label_126C0: - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_126E4 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_126E4: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v244, 0, v244 - v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - s_mov_b32 m0, s62 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_cmp_eq_u32 s69, 0 - s_cbranch_scc1 label_128EC - s_waitcnt vmcnt(9) -label_128EC: - s_waitcnt vmcnt(12) - s_barrier - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - ds_read_b128 v[80:83], v10 offset:8192 - ds_read_b128 v[84:87], v10 offset:8208 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - ds_read_b128 v[88:91], v10 offset:10272 - ds_read_b128 v[92:95], v10 offset:10288 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:8192 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:10272 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - v_max_i32_e32 v42, 0, v42 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - s_mov_b32 m0, s59 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_12A58 - buffer_load_dword v40, v38, s[24:27], 0 offen -label_12A58: - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 offset:12480 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_12A78 - buffer_load_dword v243, v44, s[24:27], 0 offen -label_12A78: - ds_read_b64_tr_b16 a[138:139], v23 offset:12608 - ds_read_b64 v[68:69], v18 offset:512 - ds_read_b64_tr_b16 a[140:141], v24 offset:12480 - ds_read_b64_tr_b16 a[142:143], v24 offset:12608 - ds_read_b64 v[70:71], v19 offset:512 - v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - ds_read_b64_tr_b16 a[144:145], v23 offset:12992 - ds_read_b64_tr_b16 a[146:147], v23 offset:13120 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - ds_read_b64_tr_b16 a[148:149], v24 offset:12992 - ds_read_b64_tr_b16 a[150:151], v24 offset:13120 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_12B38 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 -label_12B38: - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - s_nop 0 - s_cbranch_vccz label_12B5C - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_12B5C: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] - s_waitcnt vmcnt(17) - v_max_i32_e32 v245, 0, v245 - v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] - s_mov_b32 m0, s63 - global_load_lds_dwordx4 v[2:3], off - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] - s_cmp_eq_u32 s69, 0 - s_cbranch_scc1 label_12D64 - s_waitcnt vmcnt(10) -label_12D64: - s_waitcnt vmcnt(12) - s_barrier - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] - ds_read_b128 v[80:83], v10 offset:24832 - ds_read_b128 v[84:87], v10 offset:24848 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] - ds_read_b128 v[88:91], v10 offset:26912 - ds_read_b128 v[92:95], v10 offset:26928 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:24832 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:26912 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_cmp_eq_u32 s69, 1 - s_cbranch_scc1 label_17DE4 - s_branch label_11C70 -label_12E14: - s_cmp_ge_u32 s7, 2 - s_cbranch_scc1 label_13230 - s_waitcnt lgkmcnt(0) - s_barrier - v_mov_b32_e32 v33, 1.0 - v_mov_b64_e32 v[48:49], 0 - v_mov_b64_e32 v[50:51], 0 - v_mov_b64_e32 v[52:53], 0 - v_mov_b64_e32 v[54:55], 0 - v_mov_b64_e32 v[56:57], 0 - v_mov_b64_e32 v[58:59], 0 - v_mov_b64_e32 v[60:61], 0 - v_mov_b64_e32 v[62:63], 0 - v_mov_b64_e32 v[64:65], 0 - v_mov_b64_e32 v[66:67], 0 - v_mov_b64_e32 v[68:69], 0 - v_mov_b64_e32 v[70:71], 0 - v_mov_b64_e32 v[72:73], 0 - v_mov_b64_e32 v[74:75], 0 - v_mov_b64_e32 v[76:77], 0 - v_mov_b64_e32 v[78:79], 0 - s_nop 1 - buffer_load_dword v39, v38, s[24:27], 0 offen - buffer_load_dword v242, v44, s[24:27], 0 offen - v_max_i32_e32 v41, 0, v41 - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - s_nop 1 - v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - s_nop 1 - s_mov_b32 m0, s58 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - s_nop 1 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_cvt_scalef32_pk_bf16_fp8 v34, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v81, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[34:37] - v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cvt_scalef32_pk_bf16_fp8 v34, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v83, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[34:37] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v34, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v85, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[34:37] offset:32 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_cvt_scalef32_pk_bf16_fp8 v34, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v87, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[34:37] offset:48 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - s_nop 1 - v_cvt_scalef32_pk_bf16_fp8 v34, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v89, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[34:37] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v34, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v91, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[34:37] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v34, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[34:37] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v34, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v95, v33 op_sel:[1,0,0] - ds_write_b128 v11, v[34:37] offset:14640 - v_max_i32_e32 v244, 0, v244 - v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] - s_mov_b32 m0, s62 - global_load_lds_dwordx4 v[2:3], off - s_waitcnt vmcnt(5) - s_barrier - ds_read_u8 v236, v238 offset:24832 - ds_read_u8 v237, v238 offset:26912 - ds_read_b128 v[80:83], v10 offset:24832 - ds_read_b128 v[84:87], v10 offset:24848 - ds_read_b128 v[88:91], v10 offset:26912 - ds_read_b128 v[92:95], v10 offset:26928 - v_max_i32_e32 v42, 0, v42 - s_nop 1 - v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - buffer_load_dword v40, v38, s[24:27], 0 offen - buffer_load_dword v243, v44, s[24:27], 0 offen - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - s_mov_b32 m0, s59 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_max_i32_e32 v245, 0, v245 - v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] - s_mov_b32 m0, s63 - global_load_lds_dwordx4 v[2:3], off - s_branch label_1F1C -label_13230: - s_waitcnt lgkmcnt(0) - s_barrier - v_mov_b32_e32 v33, 1.0 - v_mov_b64_e32 v[48:49], 0 - v_mov_b64_e32 v[50:51], 0 - v_mov_b64_e32 v[52:53], 0 - v_mov_b64_e32 v[54:55], 0 - v_mov_b64_e32 v[56:57], 0 - v_mov_b64_e32 v[58:59], 0 - v_mov_b64_e32 v[60:61], 0 - v_mov_b64_e32 v[62:63], 0 - v_mov_b64_e32 v[64:65], 0 - v_mov_b64_e32 v[66:67], 0 - v_mov_b64_e32 v[68:69], 0 - v_mov_b64_e32 v[70:71], 0 - v_mov_b64_e32 v[72:73], 0 - v_mov_b64_e32 v[74:75], 0 - v_mov_b64_e32 v[76:77], 0 - v_mov_b64_e32 v[78:79], 0 - s_nop 1 - buffer_load_dword v39, v38, s[24:27], 0 offen - buffer_load_dword v242, v44, s[24:27], 0 offen - v_max_i32_e32 v41, 0, v41 - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - s_nop 1 - v_mov_b32_dpp v26, v41 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - s_nop 1 - s_mov_b32 m0, s58 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - s_nop 1 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - v_mov_b32_dpp v26, v41 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_cvt_scalef32_pk_bf16_fp8 v34, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v81, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[34:37] - v_mov_b32_dpp v26, v41 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v41 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - v_cvt_scalef32_pk_bf16_fp8 v34, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v83, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[34:37] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v34, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v85, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[34:37] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v34, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v87, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[34:37] offset:48 - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - s_nop 1 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - v_cvt_scalef32_pk_bf16_fp8 v34, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v89, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[34:37] offset:14592 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_cvt_scalef32_pk_bf16_fp8 v34, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v91, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[34:37] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v34, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[34:37] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v34, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v35, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v36, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v37, v95, v33 op_sel:[1,0,0] - ds_write_b128 v11, v[34:37] offset:14640 - v_max_i32_e32 v244, 0, v244 - v_mad_u64_u32 v[2:3], vcc, v244, s86, v[246:247] - s_mov_b32 m0, s62 - global_load_lds_dwordx4 v[2:3], off - s_waitcnt vmcnt(5) - s_barrier - ds_read_u8 v236, v238 offset:24832 - ds_read_u8 v237, v238 offset:26912 - ds_read_b128 v[80:83], v10 offset:24832 - ds_read_b128 v[84:87], v10 offset:24848 - ds_read_b128 v[88:91], v10 offset:26912 - ds_read_b128 v[92:95], v10 offset:26928 - v_max_i32_e32 v42, 0, v42 - s_nop 1 - v_mov_b32_dpp v26, v42 row_newbcast:0 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:2 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[2:3], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v42 row_newbcast:4 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:6 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[4:5], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v42 row_newbcast:8 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:10 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[6:7], vcc, v26, s73, v[14:15] - v_mov_b32_dpp v26, v42 row_newbcast:12 row_mask:0xf bank_mask:0xf - v_mov_b32_dpp v27, v42 row_newbcast:14 row_mask:0xf bank_mask:0xf - v_cndmask_b32_e64 v26, v26, v27, s[92:93] - v_mad_u64_u32 v[8:9], vcc, v26, s73, v[14:15] - buffer_load_dword v40, v38, s[24:27], 0 offen - buffer_load_dword v243, v44, s[24:27], 0 offen - v_add_u32_e32 v38, s94, v38 - v_add_u32_e32 v44, s94, v44 - s_mov_b32 m0, s59 - global_load_lds_dwordx4 v[2:3], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[4:5], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[6:7], off - s_add_u32 m0, m0, 0x410 - global_load_lds_dwordx4 v[8:9], off - s_add_u32 m0, m0, 0x410 - v_max_i32_e32 v245, 0, v245 - v_mad_u64_u32 v[2:3], vcc, v245, s86, v[246:247] - s_mov_b32 m0, s63 - global_load_lds_dwordx4 v[2:3], off - s_branch label_1F1C -label_13644: - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[50:51], v19 - v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_cmp_neq_f32_e32 vcc, 1.0, v34 - v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] - v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] - s_nop 2 - s_cbranch_vccz label_137DC - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_137DC: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v64, v65, v64 - v_max3_f32 v30, v66, v67, v30 - v_max3_f32 v30, v68, v69, v30 - v_max3_f32 v30, v70, v71, v30 - s_nop 1 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(5) - s_barrier - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - s_nop 1 - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:41472 - s_nop 1 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - s_nop 1 - ds_read_b128 a[48:51], v240 offset:41472 - ds_read_b128 a[52:55], v240 offset:41488 - ds_read_u8 v236, v238 offset:41472 - s_nop 1 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - s_nop 1 - ds_read_b128 a[80:83], v240 offset:43552 - ds_read_b128 a[84:87], v240 offset:43568 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:43552 - s_nop 1 - v_fma_f32 v64, v64, s5, -v32 - v_fma_f32 v65, v65, s5, -v32 - v_fma_f32 v66, v66, s5, -v32 - v_fma_f32 v67, v67, s5, -v32 - v_fma_f32 v68, v68, s5, -v32 - v_fma_f32 v69, v69, s5, -v32 - v_fma_f32 v70, v70, s5, -v32 - v_fma_f32 v71, v71, s5, -v32 - s_nop 1 - ds_read_b128 a[56:59], v13 offset:41600 - ds_read_b128 a[60:63], v13 offset:41616 - v_exp_f32_e32 v64, v64 - v_exp_f32_e32 v65, v65 - v_exp_f32_e32 v66, v66 - ds_read_u8 v237, v238 offset:43552 - s_nop 1 - v_exp_f32_e32 v67, v67 - v_exp_f32_e32 v68, v68 - v_exp_f32_e32 v69, v69 - v_exp_f32_e32 v70, v70 - v_exp_f32_e32 v71, v71 - v_mul_f32_e32 v46, v33, v46 - s_nop 1 - ds_read_b128 a[88:91], v13 offset:43680 - ds_read_b128 a[92:95], v13 offset:43696 - v_add_f32_e32 v26, v65, v64 - v_add_f32_e32 v26, v66, v26 - v_add_f32_e32 v26, v67, v26 - v_add_f32_e32 v26, v68, v26 - s_nop 1 - v_add_f32_e32 v26, v69, v26 - v_add_f32_e32 v26, v70, v26 - v_add_f32_e32 v26, v71, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v64, v64, v65 - v_cvt_pk_bf16_f32 v65, v66, v67 - v_cvt_pk_bf16_f32 v66, v68, v69 - v_cvt_pk_bf16_f32 v67, v70, v71 - s_nop 1 - ds_read_b128 a[64:67], v241 offset:41664 - ds_read_b128 a[68:71], v241 offset:41680 - ds_write_b32 v17, v33 offset:4096 - s_nop 1 - s_nop 1 - ds_read_b128 a[96:99], v241 offset:43744 - ds_read_b128 a[100:103], v241 offset:43760 - ds_write_b64 v20, v[64:65] - s_nop 1 - s_nop 1 - ds_read_b128 a[104:107], v16 offset:8320 - ds_read_b128 a[108:111], v16 offset:8336 - ds_write_b64 v20, v[66:67] offset:2048 - s_nop 1 - s_nop 1 - ds_read_b128 a[112:115], v16 offset:10400 - ds_read_b128 a[116:119], v16 offset:10416 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[66:67], v19 - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_cmp_neq_f32_e32 vcc, 1.0, v34 - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - s_nop 2 - s_cbranch_vccz label_13DB4 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_13DB4: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - s_nop 1 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(0) - s_barrier - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - s_nop 1 - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:58112 - s_nop 1 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - s_nop 1 - ds_read_b128 a[48:51], v240 offset:58112 - ds_read_b128 a[52:55], v240 offset:58128 - ds_read_u8 v236, v238 offset:58112 - s_nop 1 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - s_nop 1 - ds_read_b128 a[80:83], v240 offset:60192 - ds_read_b128 a[84:87], v240 offset:60208 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:60192 - s_nop 1 - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - s_nop 1 - ds_read_b128 a[56:59], v13 offset:58240 - ds_read_b128 a[60:63], v13 offset:58256 - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - v_exp_f32_e32 v50, v50 - ds_read_u8 v237, v238 offset:60192 - s_nop 1 - v_exp_f32_e32 v51, v51 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - s_nop 1 - ds_read_b128 a[88:91], v13 offset:60320 - ds_read_b128 a[92:95], v13 offset:60336 - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - s_nop 1 - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_add_f32_e32 v26, v55, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - s_nop 1 - ds_read_b128 a[64:67], v241 offset:58304 - ds_read_b128 a[68:71], v241 offset:58320 - ds_write_b32 v17, v33 offset:4096 - s_nop 1 - s_nop 1 - ds_read_b128 a[96:99], v241 offset:60384 - ds_read_b128 a[100:103], v241 offset:60400 - ds_write_b64 v20, v[48:49] - s_nop 1 - s_nop 1 - ds_read_b128 a[104:107], v16 offset:12480 - ds_read_b128 a[108:111], v16 offset:12496 - ds_write_b64 v20, v[50:51] offset:2048 - s_nop 1 - s_nop 1 - ds_read_b128 a[112:115], v16 offset:14560 - ds_read_b128 a[116:119], v16 offset:14576 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[50:51], v19 - v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_cmp_neq_f32_e32 vcc, 1.0, v34 - v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] - v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] - s_nop 2 - s_cbranch_vccz label_1438C - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_1438C: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v64, v65, v64 - v_max3_f32 v30, v66, v67, v30 - v_max3_f32 v30, v68, v69, v30 - v_max3_f32 v30, v70, v71, v30 - s_nop 1 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(0) - s_barrier - s_nop 1 - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - s_nop 1 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - s_nop 1 - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - s_nop 1 - s_nop 1 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - s_nop 1 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - s_nop 1 - v_fma_f32 v64, v64, s5, -v32 - v_fma_f32 v65, v65, s5, -v32 - v_fma_f32 v66, v66, s5, -v32 - v_fma_f32 v67, v67, s5, -v32 - v_fma_f32 v68, v68, s5, -v32 - v_fma_f32 v69, v69, s5, -v32 - v_fma_f32 v70, v70, s5, -v32 - v_fma_f32 v71, v71, s5, -v32 - s_nop 1 - v_exp_f32_e32 v64, v64 - v_exp_f32_e32 v65, v65 - v_exp_f32_e32 v66, v66 - s_nop 1 - v_exp_f32_e32 v67, v67 - v_exp_f32_e32 v68, v68 - v_exp_f32_e32 v69, v69 - v_exp_f32_e32 v70, v70 - v_exp_f32_e32 v71, v71 - v_mul_f32_e32 v46, v33, v46 - s_nop 1 - v_add_f32_e32 v26, v65, v64 - v_add_f32_e32 v26, v66, v26 - v_add_f32_e32 v26, v67, v26 - v_add_f32_e32 v26, v68, v26 - s_nop 1 - v_add_f32_e32 v26, v69, v26 - v_add_f32_e32 v26, v70, v26 - v_add_f32_e32 v26, v71, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v64, v64, v65 - v_cvt_pk_bf16_f32 v65, v66, v67 - v_cvt_pk_bf16_f32 v66, v68, v69 - v_cvt_pk_bf16_f32 v67, v70, v71 - s_nop 1 - ds_write_b32 v17, v33 offset:4096 - s_nop 1 - s_nop 1 - ds_write_b64 v20, v[64:65] - s_nop 1 - s_nop 1 - ds_write_b64 v20, v[66:67] offset:2048 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_branch label_3D68 -label_14710: - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_nop 4 - s_cbranch_vccz label_147C8 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_147C8: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(5) - s_barrier - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:41472 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:43552 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[66:67], v19 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_nop 4 - s_cbranch_vccz label_14AE4 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_14AE4: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(0) - s_barrier - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:58112 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:60192 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_nop 4 - s_cbranch_vccz label_14E00 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_14E00: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(0) - s_barrier - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_branch label_3D68 -label_15038: - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_nop 4 - s_cbranch_vccz label_150F0 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_150F0: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(5) - s_barrier - s_nop 1 - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:41472 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:43552 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_nop 4 - s_cbranch_vccz label_1540C - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_1540C: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(0) - s_barrier - s_nop 1 - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:58112 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:60192 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_nop 4 - s_cbranch_vccz label_15728 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_15728: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(0) - s_barrier - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_branch label_3D68 -label_15960: - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 - ds_read_b64_tr_b16 a[138:139], v23 offset:128 - ds_read_b64_tr_b16 a[140:141], v24 - ds_read_b64_tr_b16 a[142:143], v24 offset:128 - ds_read_b64_tr_b16 a[144:145], v23 offset:512 - ds_read_b64_tr_b16 a[146:147], v23 offset:640 - ds_read_b64_tr_b16 a[148:149], v24 offset:512 - ds_read_b64_tr_b16 a[150:151], v24 offset:640 - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_nop 4 - s_cbranch_vccz label_15A18 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_15A18: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(5) - s_barrier - s_nop 1 - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:41472 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:43552 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 offset:4160 - ds_read_b64_tr_b16 a[138:139], v23 offset:4288 - ds_read_b64_tr_b16 a[140:141], v24 offset:4160 - ds_read_b64_tr_b16 a[142:143], v24 offset:4288 - ds_read_b64_tr_b16 a[144:145], v23 offset:4672 - ds_read_b64_tr_b16 a[146:147], v23 offset:4800 - ds_read_b64_tr_b16 a[148:149], v24 offset:4672 - ds_read_b64_tr_b16 a[150:151], v24 offset:4800 - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_nop 4 - s_cbranch_vccz label_15D34 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_15D34: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(0) - s_barrier - s_nop 1 - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - s_nop 1 - s_nop 1 - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:58112 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:60192 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 offset:8320 - ds_read_b64_tr_b16 a[138:139], v23 offset:8448 - ds_read_b64_tr_b16 a[140:141], v24 offset:8320 - ds_read_b64_tr_b16 a[142:143], v24 offset:8448 - ds_read_b64_tr_b16 a[144:145], v23 offset:8832 - ds_read_b64_tr_b16 a[146:147], v23 offset:8960 - ds_read_b64_tr_b16 a[148:149], v24 offset:8832 - ds_read_b64_tr_b16 a[150:151], v24 offset:8960 - v_cmp_neq_f32_e32 vcc, 1.0, v34 - s_nop 4 - s_cbranch_vccz label_16050 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9F34 -label_16050: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - s_nop 1 - ds_write_b128 v11, v[26:29] offset:14640 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_waitcnt vmcnt(0) - s_barrier - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_branch label_3D68 -label_16288: - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[50:51], v19 - v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[52:53], v18 offset:512 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64 v[54:55], v19 offset:512 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] - v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] - s_nop 2 - s_cbranch_vccz label_1643C - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_1643C: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v64, v65, v64 - v_max3_f32 v30, v66, v67, v30 - v_max3_f32 v30, v68, v69, v30 - v_max3_f32 v30, v70, v71, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_waitcnt vmcnt(5) - s_barrier - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:41472 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - s_nop 1 - ds_read_b128 a[48:51], v240 offset:41472 - ds_read_b128 a[52:55], v240 offset:41488 - ds_read_u8 v236, v238 offset:41472 - s_nop 1 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - s_nop 1 - ds_read_b128 a[80:83], v240 offset:43552 - ds_read_b128 a[84:87], v240 offset:43568 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:43552 - s_nop 1 - v_fma_f32 v64, v64, s5, -v32 - v_fma_f32 v65, v65, s5, -v32 - v_fma_f32 v66, v66, s5, -v32 - v_fma_f32 v67, v67, s5, -v32 - v_fma_f32 v68, v68, s5, -v32 - v_fma_f32 v69, v69, s5, -v32 - v_fma_f32 v70, v70, s5, -v32 - v_fma_f32 v71, v71, s5, -v32 - s_nop 1 - ds_read_b128 a[56:59], v13 offset:41600 - ds_read_b128 a[60:63], v13 offset:41616 - v_exp_f32_e32 v64, v64 - v_exp_f32_e32 v65, v65 - v_exp_f32_e32 v66, v66 - ds_read_u8 v237, v238 offset:43552 - s_nop 1 - v_exp_f32_e32 v67, v67 - v_exp_f32_e32 v68, v68 - v_exp_f32_e32 v69, v69 - v_exp_f32_e32 v70, v70 - v_exp_f32_e32 v71, v71 - v_mul_f32_e32 v46, v33, v46 - s_nop 1 - ds_read_b128 a[88:91], v13 offset:43680 - ds_read_b128 a[92:95], v13 offset:43696 - v_add_f32_e32 v26, v65, v64 - v_add_f32_e32 v26, v66, v26 - v_add_f32_e32 v26, v67, v26 - v_add_f32_e32 v26, v68, v26 - s_nop 1 - v_add_f32_e32 v26, v69, v26 - v_add_f32_e32 v26, v70, v26 - v_add_f32_e32 v26, v71, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v64, v64, v65 - v_cvt_pk_bf16_f32 v65, v66, v67 - v_cvt_pk_bf16_f32 v66, v68, v69 - v_cvt_pk_bf16_f32 v67, v70, v71 - s_nop 1 - ds_read_b128 a[64:67], v241 offset:41664 - ds_read_b128 a[68:71], v241 offset:41680 - ds_write_b32 v17, v33 offset:4096 - s_nop 1 - s_nop 1 - ds_read_b128 a[96:99], v241 offset:43744 - ds_read_b128 a[100:103], v241 offset:43760 - ds_write_b64 v20, v[64:65] - s_nop 1 - s_nop 1 - ds_read_b128 a[104:107], v16 offset:8320 - ds_read_b128 a[108:111], v16 offset:8336 - ds_write_b64 v20, v[66:67] offset:2048 - s_nop 1 - s_nop 1 - ds_read_b128 a[112:115], v16 offset:10400 - ds_read_b128 a[116:119], v16 offset:10416 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[48:55], a[8:15], v[48:51], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[56:63], a[16:23], v[48:51], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - v_mfma_scale_f32_16x16x128_f8f6f4 v[48:51], a[64:71], a[24:31], v[48:51], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[66:67], v19 - v_mfma_f32_16x16x32_bf16 v[48:51], a[104:107], a[32:35], v[48:51] - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[48:51], a[108:111], a[36:39], v[48:51] - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[68:69], v18 offset:512 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[80:87], a[8:15], v[52:55], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64 v[70:71], v19 offset:512 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[88:95], a[16:23], v[52:55], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - v_mfma_scale_f32_16x16x128_f8f6f4 v[52:55], a[96:103], a[24:31], v[52:55], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[52:55], a[112:115], a[32:35], v[52:55] - v_mfma_f32_16x16x32_bf16 v[52:55], a[116:119], a[36:39], v[52:55] - s_nop 2 - s_cbranch_vccz label_16A50 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_16A50: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v48, v49, v48 - v_max3_f32 v30, v50, v51, v30 - v_max3_f32 v30, v52, v53, v30 - v_max3_f32 v30, v54, v55, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] - s_waitcnt vmcnt(0) - s_barrier - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - ds_read_b128 v[228:231], v1 offset:58112 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - s_nop 1 - ds_read_b128 a[48:51], v240 offset:58112 - ds_read_b128 a[52:55], v240 offset:58128 - ds_read_u8 v236, v238 offset:58112 - s_nop 1 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - s_nop 1 - ds_read_b128 a[80:83], v240 offset:60192 - ds_read_b128 a[84:87], v240 offset:60208 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - ds_read_b128 v[232:235], v1 offset:60192 - s_nop 1 - v_fma_f32 v48, v48, s5, -v32 - v_fma_f32 v49, v49, s5, -v32 - v_fma_f32 v50, v50, s5, -v32 - v_fma_f32 v51, v51, s5, -v32 - v_fma_f32 v52, v52, s5, -v32 - v_fma_f32 v53, v53, s5, -v32 - v_fma_f32 v54, v54, s5, -v32 - v_fma_f32 v55, v55, s5, -v32 - s_nop 1 - ds_read_b128 a[56:59], v13 offset:58240 - ds_read_b128 a[60:63], v13 offset:58256 - v_exp_f32_e32 v48, v48 - v_exp_f32_e32 v49, v49 - v_exp_f32_e32 v50, v50 - ds_read_u8 v237, v238 offset:60192 - s_nop 1 - v_exp_f32_e32 v51, v51 - v_exp_f32_e32 v52, v52 - v_exp_f32_e32 v53, v53 - v_exp_f32_e32 v54, v54 - v_exp_f32_e32 v55, v55 - v_mul_f32_e32 v46, v33, v46 - s_nop 1 - ds_read_b128 a[88:91], v13 offset:60320 - ds_read_b128 a[92:95], v13 offset:60336 - v_add_f32_e32 v26, v49, v48 - v_add_f32_e32 v26, v50, v26 - v_add_f32_e32 v26, v51, v26 - v_add_f32_e32 v26, v52, v26 - s_nop 1 - v_add_f32_e32 v26, v53, v26 - v_add_f32_e32 v26, v54, v26 - v_add_f32_e32 v26, v55, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v48, v48, v49 - v_cvt_pk_bf16_f32 v49, v50, v51 - v_cvt_pk_bf16_f32 v50, v52, v53 - v_cvt_pk_bf16_f32 v51, v54, v55 - s_nop 1 - ds_read_b128 a[64:67], v241 offset:58304 - ds_read_b128 a[68:71], v241 offset:58320 - ds_write_b32 v17, v33 offset:4096 - s_nop 1 - s_nop 1 - ds_read_b128 a[96:99], v241 offset:60384 - ds_read_b128 a[100:103], v241 offset:60400 - ds_write_b64 v20, v[48:49] - s_nop 1 - s_nop 1 - ds_read_b128 a[104:107], v16 offset:12480 - ds_read_b128 a[108:111], v16 offset:12496 - ds_write_b64 v20, v[50:51] offset:2048 - s_nop 1 - s_nop 1 - ds_read_b128 a[112:115], v16 offset:14560 - ds_read_b128 a[116:119], v16 offset:14576 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(2) - s_barrier - v_perm_b32 v32, v228, v229, v12 - v_perm_b32 v228, v229, v228, v12 - v_mov_b32_e32 v229, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], v[80:87], a[0:7], 0, v228, v224 op_sel_hi:[0,0,0] - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - v_bfe_u32 v230, v230, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[48:55], a[8:15], v[64:67], v229, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - v_and_b32_e32 v231, v47, v231 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[56:63], a[16:23], v[64:67], v230, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - v_mfma_scale_f32_16x16x128_f8f6f4 v[64:67], a[64:71], a[24:31], v[64:67], v231, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64 v[50:51], v19 - v_mfma_f32_16x16x32_bf16 v[64:67], a[104:107], a[32:35], v[64:67] - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - v_mfma_f32_16x16x32_bf16 v[64:67], a[108:111], a[36:39], v[64:67] - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[52:53], v18 offset:512 - v_perm_b32 v32, v232, v233, v12 - v_perm_b32 v232, v233, v232, v12 - v_mov_b32_e32 v233, v32 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], v[88:95], a[0:7], 0, v232, v224 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - v_bfe_u32 v234, v234, v25, 8 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[80:87], a[8:15], v[68:71], v233, v225 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64 v[54:55], v19 offset:512 - v_and_b32_e32 v235, v47, v235 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[88:95], a[16:23], v[68:71], v234, v226 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - v_mfma_scale_f32_16x16x128_f8f6f4 v[68:71], a[96:103], a[24:31], v[68:71], v235, v227 op_sel_hi:[0,0,0] - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - v_mfma_f32_16x16x32_bf16 v[68:71], a[112:115], a[32:35], v[68:71] - v_mfma_f32_16x16x32_bf16 v[68:71], a[116:119], a[36:39], v[68:71] - s_nop 2 - s_cbranch_vccz label_17064 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_17064: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - ds_write_b128 v11, v[26:29] offset:14640 - v_max3_f32 v30, v64, v65, v64 - v_max3_f32 v30, v66, v67, v30 - v_max3_f32 v30, v68, v69, v30 - v_max3_f32 v30, v70, v71, v30 - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_waitcnt vmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - v_mov_b32_e32 v26, v30 - v_mov_b32_e32 v27, v30 - s_nop 1 - v_permlane16_swap_b32_e32 v26, v27 - v_mov_b32_e32 v29, v26 - v_mov_b32_e32 v28, v27 - s_nop 1 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_permlane32_swap_b32_e32 v26, v27 - v_permlane32_swap_b32_e32 v28, v29 - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - v_max3_f32 v30, v26, v27, v30 - v_max3_f32 v30, v28, v29, v30 - v_cmp_eq_u32_e64 s[64:65], s74, v45 - v_max_f32_e32 v31, v30, v45 - s_nop 1 - s_nop 1 - v_sub_f32_e32 v27, v31, v45 - v_mul_f32_e32 v27, s5, v27 - v_cmp_lt_f32_e32 vcc, 0x41000000, v27 - v_sub_f32_e32 v33, v45, v31 - v_cndmask_b32_e64 v33, v33, 0, s[64:65] - v_cndmask_b32_e32 v33, 0, v33, vcc - v_cndmask_b32_e32 v45, v45, v31, vcc - s_nop 1 - v_mul_f32_e32 v32, s5, v45 - v_mul_f32_e32 v33, s5, v33 - v_exp_f32_e32 v33, v33 - s_nop 1 - v_fma_f32 v64, v64, s5, -v32 - v_fma_f32 v65, v65, s5, -v32 - v_fma_f32 v66, v66, s5, -v32 - v_fma_f32 v67, v67, s5, -v32 - v_fma_f32 v68, v68, s5, -v32 - v_fma_f32 v69, v69, s5, -v32 - v_fma_f32 v70, v70, s5, -v32 - v_fma_f32 v71, v71, s5, -v32 - s_nop 1 - v_exp_f32_e32 v64, v64 - v_exp_f32_e32 v65, v65 - v_exp_f32_e32 v66, v66 - s_nop 1 - v_exp_f32_e32 v67, v67 - v_exp_f32_e32 v68, v68 - v_exp_f32_e32 v69, v69 - v_exp_f32_e32 v70, v70 - v_exp_f32_e32 v71, v71 - v_mul_f32_e32 v46, v33, v46 - s_nop 1 - v_add_f32_e32 v26, v65, v64 - v_add_f32_e32 v26, v66, v26 - v_add_f32_e32 v26, v67, v26 - v_add_f32_e32 v26, v68, v26 - s_nop 1 - v_add_f32_e32 v26, v69, v26 - v_add_f32_e32 v26, v70, v26 - v_add_f32_e32 v26, v71, v26 - v_add_f32_e32 v46, v26, v46 - v_cvt_pk_bf16_f32 v64, v64, v65 - v_cvt_pk_bf16_f32 v65, v66, v67 - v_cvt_pk_bf16_f32 v66, v68, v69 - v_cvt_pk_bf16_f32 v67, v70, v71 - s_nop 1 - ds_write_b32 v17, v33 offset:4096 - s_nop 1 - s_nop 1 - ds_write_b64 v20, v[64:65] - s_nop 1 - s_nop 1 - ds_write_b64 v20, v[66:67] offset:2048 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_branch label_3D68 -label_17408: - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[52:53], v18 offset:512 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64 v[54:55], v19 offset:512 - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_nop 4 - s_cbranch_vccz label_174DC - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_174DC: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_waitcnt vmcnt(5) - s_barrier - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:41472 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:43552 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[68:69], v18 offset:512 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64 v[70:71], v19 offset:512 - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_nop 4 - s_cbranch_vccz label_17834 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_17834: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] - s_waitcnt vmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:58112 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:60192 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v21 offset:128 - ds_read_b64_tr_b16 a[138:139], v21 offset:3776 - ds_read_b64 v[52:53], v18 offset:512 - ds_read_b64_tr_b16 a[140:141], v22 offset:128 - ds_read_b64_tr_b16 a[142:143], v22 offset:3776 - ds_read_b64 v[54:55], v19 offset:512 - ds_read_b64_tr_b16 a[144:145], v21 offset:192 - ds_read_b64_tr_b16 a[146:147], v21 offset:3840 - ds_read_b64_tr_b16 a[148:149], v22 offset:192 - ds_read_b64_tr_b16 a[150:151], v22 offset:3840 - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_nop 4 - s_cbranch_vccz label_17B8C - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_17B8C: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_waitcnt vmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_branch label_3D68 -label_17DE4: - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 - ds_read_b64_tr_b16 a[138:139], v23 offset:128 - ds_read_b64 v[52:53], v18 offset:512 - ds_read_b64_tr_b16 a[140:141], v24 - ds_read_b64_tr_b16 a[142:143], v24 offset:128 - ds_read_b64 v[54:55], v19 offset:512 - ds_read_b64_tr_b16 a[144:145], v23 offset:512 - ds_read_b64_tr_b16 a[146:147], v23 offset:640 - ds_read_b64_tr_b16 a[148:149], v24 offset:512 - ds_read_b64_tr_b16 a[150:151], v24 offset:640 - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_nop 4 - s_cbranch_vccz label_17EB8 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_17EB8: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_waitcnt vmcnt(5) - s_barrier - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - ds_read_b128 v[80:83], v10 offset:41472 - ds_read_b128 v[84:87], v10 offset:41488 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - ds_read_b128 v[88:91], v10 offset:43552 - ds_read_b128 v[92:95], v10 offset:43568 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:41472 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:43552 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[64:65], v18 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[66:67], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 offset:4160 - ds_read_b64_tr_b16 a[138:139], v23 offset:4288 - ds_read_b64 v[68:69], v18 offset:512 - ds_read_b64_tr_b16 a[140:141], v24 offset:4160 - ds_read_b64_tr_b16 a[142:143], v24 offset:4288 - ds_read_b64 v[70:71], v19 offset:512 - ds_read_b64_tr_b16 a[144:145], v23 offset:4672 - ds_read_b64_tr_b16 a[146:147], v23 offset:4800 - ds_read_b64_tr_b16 a[148:149], v24 offset:4672 - ds_read_b64_tr_b16 a[150:151], v24 offset:4800 - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_nop 4 - s_cbranch_vccz label_18210 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_18210: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[64:67], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[64:67], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[64:67], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[64:67], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[64:67], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[64:67], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[64:67], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[64:67], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[68:71], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[68:71], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[68:71], v[136:139] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[68:71], v[140:143] - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[68:71], v[144:147] - s_waitcnt vmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[68:71], v[148:151] - ds_read_b128 v[80:83], v10 offset:58112 - ds_read_b128 v[84:87], v10 offset:58128 - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[68:71], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[68:71], v[156:159] - ds_read_b128 v[88:91], v10 offset:60192 - ds_read_b128 v[92:95], v10 offset:60208 - s_nop 1 - s_nop 1 - ds_read_u8 v236, v238 offset:58112 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - ds_read_u8 v237, v238 offset:60192 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - ds_read_b32 v34, v43 offset:4096 - ds_read_b32 v35, v43 offset:4352 - ds_read_b64_tr_b16 a[120:121], v21 - ds_read_b64_tr_b16 a[122:123], v21 offset:3648 - ds_read_b64_tr_b16 a[124:125], v22 - ds_read_b64 v[48:49], v18 - ds_read_b64_tr_b16 a[126:127], v22 offset:3648 - ds_read_b64_tr_b16 a[128:129], v21 offset:64 - ds_read_b64 v[50:51], v19 - ds_read_b64_tr_b16 a[130:131], v21 offset:3712 - ds_read_b64_tr_b16 a[132:133], v22 offset:64 - ds_read_b64_tr_b16 a[134:135], v22 offset:3712 - ds_read_b64_tr_b16 a[136:137], v23 offset:8320 - ds_read_b64_tr_b16 a[138:139], v23 offset:8448 - ds_read_b64 v[52:53], v18 offset:512 - ds_read_b64_tr_b16 a[140:141], v24 offset:8320 - ds_read_b64_tr_b16 a[142:143], v24 offset:8448 - ds_read_b64 v[54:55], v19 offset:512 - ds_read_b64_tr_b16 a[144:145], v23 offset:8832 - ds_read_b64_tr_b16 a[146:147], v23 offset:8960 - ds_read_b64_tr_b16 a[148:149], v24 offset:8832 - ds_read_b64_tr_b16 a[150:151], v24 offset:8960 - v_min_f32_e32 v26, v34, v35 - v_cmp_neq_f32_e32 vcc, 1.0, v26 - s_nop 4 - s_cbranch_vccz label_18568 - s_getpc_b64 vcc - s_add_i32 vcc_lo, vcc_lo, 8 - s_branch label_9EB4 -label_18568: - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_waitcnt lgkmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[96:99], a[120:123], v[48:51], v[96:99] - v_lshlrev_b32_e32 v33, 23, v236 - v_cmp_eq_u32_e64 vcc, v236, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v80, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v80, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v81, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v81, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[100:103], a[124:127], v[48:51], v[100:103] - ds_write_b128 v11, v[26:29] - v_cvt_scalef32_pk_bf16_fp8 v26, v82, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v82, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v83, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v83, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[104:107], a[128:131], v[48:51], v[104:107] - ds_write_b128 v11, v[26:29] offset:16 - v_cvt_scalef32_pk_bf16_fp8 v26, v84, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v84, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v85, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v85, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[108:111], a[132:135], v[48:51], v[108:111] - ds_write_b128 v11, v[26:29] offset:32 - v_cvt_scalef32_pk_bf16_fp8 v26, v86, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v86, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v87, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v87, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[112:115], a[136:139], v[48:51], v[112:115] - ds_write_b128 v11, v[26:29] offset:48 - v_lshlrev_b32_e32 v33, 23, v237 - v_cmp_eq_u32_e64 vcc, v237, 0 - v_cndmask_b32_e64 v33, v33, 0, vcc - v_cvt_scalef32_pk_bf16_fp8 v26, v88, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v88, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v89, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v89, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[116:119], a[140:143], v[48:51], v[116:119] - ds_write_b128 v11, v[26:29] offset:14592 - v_cvt_scalef32_pk_bf16_fp8 v26, v90, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v90, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v91, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v91, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[120:123], a[144:147], v[48:51], v[120:123] - ds_write_b128 v11, v[26:29] offset:14608 - v_cvt_scalef32_pk_bf16_fp8 v26, v92, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v92, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v93, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v93, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[124:127], a[148:151], v[48:51], v[124:127] - ds_write_b128 v11, v[26:29] offset:14624 - v_cvt_scalef32_pk_bf16_fp8 v26, v94, v33 - v_cvt_scalef32_pk_bf16_fp8 v27, v94, v33 op_sel:[1,0,0] - v_cvt_scalef32_pk_bf16_fp8 v28, v95, v33 - v_cvt_scalef32_pk_bf16_fp8 v29, v95, v33 op_sel:[1,0,0] - v_mfma_f32_16x16x32_bf16 v[128:131], a[120:123], v[52:55], v[128:131] - ds_write_b128 v11, v[26:29] offset:14640 - v_mfma_f32_16x16x32_bf16 v[132:135], a[124:127], v[52:55], v[132:135] - v_mfma_f32_16x16x32_bf16 v[136:139], a[128:131], v[52:55], v[136:139] - v_mfma_f32_16x16x32_bf16 v[140:143], a[132:135], v[52:55], v[140:143] - v_mfma_f32_16x16x32_bf16 v[144:147], a[136:139], v[52:55], v[144:147] - s_waitcnt vmcnt(0) - s_barrier - v_mfma_f32_16x16x32_bf16 v[148:151], a[140:143], v[52:55], v[148:151] - v_mfma_f32_16x16x32_bf16 v[152:155], a[144:147], v[52:55], v[152:155] - v_mfma_f32_16x16x32_bf16 v[156:159], a[148:151], v[52:55], v[156:159] - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_nop 1 - s_addk_i32 s87, 0x1 - s_cmp_lt_i32 s87, s88 - s_cbranch_scc0 label_3D68 - v_permlane32_swap_b32_e32 v26, v26 - v_permlane32_swap_b32_e32 v26, v26 - s_branch label_3D68 -.rodata -.p2align 6 -.globl _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE.kd -.type _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE.kd,@object -.size _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE.kd, 64 -_ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE.kd: -.byte 0, 128, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 -.quad _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE - _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE.kd -.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 63, 3, 12, 0, 132, 3, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0 -.amdgpu_metadata ---- -amdhsa.kernels: - - .args: - - .actual_access: read_write - .address_space: global - .name: O - .offset: 0 - .size: 8 - .value_kind: global_buffer - - .name: pad - .offset: 8 - .size: 8 - .value_kind: by_value - .value_type: i32 - - .actual_access: read_write - .address_space: global - .name: Q - .offset: 16 - .size: 8 - .value_kind: global_buffer - - .name: pad - .offset: 24 - .size: 8 - .value_kind: by_value - .value_type: i32 - - .actual_access: read_only - .address_space: global - .name: K - .offset: 32 - .size: 8 - .value_kind: global_buffer - - .name: pad - .offset: 40 - .size: 8 - .value_kind: by_value - .value_type: i32 - - .actual_access: read_only - .address_space: global - .name: V - .offset: 48 - .size: 8 - .value_kind: global_buffer - - .name: pad - .offset: 56 - .size: 8 - .value_kind: by_value - .value_type: i32 - - .actual_access: read_only - .address_space: global - .name: BT - .offset: 64 - .size: 8 - .value_kind: global_buffer - - .name: pad - .offset: 72 - .size: 8 - .value_kind: by_value - .value_type: i32 - - .actual_access: read_only - .address_space: global - .name: CL - .offset: 80 - .size: 8 - .value_kind: global_buffer - - .name: pad - .offset: 88 - .size: 8 - .value_kind: by_value - .value_type: i32 - - .actual_access: read_only - .address_space: global - .name: KQ - .offset: 96 - .size: 8 - .value_kind: global_buffer - - .name: pad - .offset: 104 - .size: 8 - .value_kind: by_value - .value_type: i32 - - .name: sclg - .offset: 112 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 116 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 120 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 124 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: mblk - .offset: 128 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 132 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 136 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 140 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: batch - .offset: 144 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 148 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 152 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 156 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: Qs - .offset: 160 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 164 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 168 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 172 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: Bs - .offset: 176 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 180 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 184 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 188 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: Cs - .offset: 192 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 196 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 200 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 204 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .actual_access: read_only - .address_space: global - .name: QT - .offset: 208 - .size: 8 - .value_kind: global_buffer - - .name: pad - .offset: 216 - .size: 8 - .value_kind: by_value - .value_type: i32 - - .actual_access: read_only - .address_space: global - .name: ST - .offset: 224 - .size: 8 - .value_kind: global_buffer - - .name: pad - .offset: 232 - .size: 8 - .value_kind: by_value - .value_type: i32 - - .actual_access: read_write - .address_space: global - .name: RP - .offset: 240 - .size: 8 - .value_kind: global_buffer - - .name: pad - .offset: 248 - .size: 8 - .value_kind: by_value - .value_type: i32 - - .actual_access: read_only - .address_space: global - .name: DQ - .offset: 256 - .size: 8 - .value_kind: global_buffer - - .name: pad - .offset: 264 - .size: 8 - .value_kind: by_value - .value_type: i32 - - .actual_access: read_only - .address_space: global - .name: DK - .offset: 272 - .size: 8 - .value_kind: global_buffer - - .name: pad - .offset: 280 - .size: 8 - .value_kind: by_value - .value_type: i32 - - .name: out16 - .offset: 288 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 292 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 296 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 300 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .actual_access: read_write - .address_space: global - .name: VALID_SPLIT - .offset: 304 - .size: 8 - .value_kind: global_buffer - - .name: pad - .offset: 312 - .size: 8 - .value_kind: by_value - .value_type: i32 - - .name: use_valid_split - .offset: 320 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 324 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 328 - .size: 4 - .value_kind: by_value - .value_type: i32 - - .name: pad - .offset: 332 - .size: 4 - .value_kind: by_value - .value_type: i32 - .group_segment_fixed_size: 163840 - .kernarg_segment_align: 4 - .kernarg_segment_size: 336 - .max_flat_workgroup_size: 256 - .name: _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE - .private_segment_fixed_size: 0 - .reqd_workgroup_size: - - 256 - - 1 - - 1 - .sgpr_count: 96 - .symbol: _ZN5aiter36mla_a8w8_qh64_qseqlen1_gqaratio64_nmE.kd - .vgpr_count: 512 - .wavefront_size: 64 -amdhsa.version: - - 1 - - 2 -... -.end_amdgpu_metadata diff --git a/hsa/gfx950/mla_v4/nm_ps_src/gen.py b/hsa/gfx950/mla_v4/nm_ps_src/gen.py deleted file mode 100644 index 2fb09666a8e..00000000000 --- a/hsa/gfx950/mla_v4/nm_ps_src/gen.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. -"""Regenerate hsa/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps.co from sources (python3 stdlib only). - - python3 gen.py --out [--check ] [--keep ] [--verify-rt ] - -Steps: asm/rt.s --(asm/patch.py: persistent walk + splice/pre.hip planner + splice/fin.hip fused combine, both -compiled with hipcc -S)--> patched .s --(kernel symbol renamed to the _ps name)--> clang -x assembler (gfx950) ---> ld.lld -shared --> .co. Tools come from $ROCM_PATH (default /opt/rocm).""" - -import argparse -import hashlib -import os -import re -import shutil -import subprocess -import sys -import tempfile - -sys.dont_write_bytecode = True - -HERE = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, os.path.join(HERE, "asm")) -import patch as PATCH # noqa: E402 - -RT_SHA256 = "7fc7516ef916ba83770744ed5d55faafb57684908fd31b76727adbc6ae9bd20b" -OLD_NAME = "mla_a8w8_qh64_qseqlen1_gqaratio64_nm" -NEW_NAME = "mla_a8w8_qh64_qseqlen1_gqaratio64_nm_ps" - - -def mangle(name): - """aiter:: (Itanium mangling of a namespace-scope name, as used by the AITER ASM kernels).""" - return f"_ZN5aiter{len(name)}{name}E" - - -def tools(): - rocm = os.environ.get("ROCM_PATH") or "/opt/rocm" - t = dict( - clang=f"{rocm}/llvm/bin/clang", - lld=f"{rocm}/llvm/bin/ld.lld", - readelf=f"{rocm}/llvm/bin/llvm-readelf", - objcopy=f"{rocm}/llvm/bin/llvm-objcopy", - hipcc=f"{rocm}/bin/hipcc", - ) - for k, p in t.items(): - if not os.access(p, os.X_OK): - sys.exit(f"gen.py: {k} not found at {p} (set ROCM_PATH)") - return t - - -def sha256(path): - return hashlib.sha256(open(path, "rb").read()).hexdigest() - - -def rename(text, old, new): - """Rename the kernel symbol everywhere (symbol, .kd, metadata .name / .symbol); returns (text, occurrences).""" - mo, mn = mangle(old), mangle(new) - assert mn not in text, mn - pat = re.compile( - re.escape(mo) + r"(?![A-Za-z0-9_])" - ) # also matches the ".kd" occurrences - text, n = pat.subn(mn, text) - assert n > 0 and mo not in text, (mo, n) - return text, n - - -def assemble(t, s_path, co_path): - o_path = os.path.splitext(co_path)[0] + ".o" - subprocess.run( - [ - t["clang"], - "-x", - "assembler", - "-target", - "amdgcn-amd-amdhsa", - "-mcpu=gfx950", - "-c", - s_path, - "-o", - o_path, - ], - check=True, - ) - subprocess.run([t["lld"], "-shared", o_path, "-o", co_path], check=True) - - -def section(t, co, name, d): - out = os.path.join(d, os.path.basename(co) + name) - subprocess.run( - [t["objcopy"], "-O", "binary", f"--only-section={name}", co, out], check=True - ) - return open(out, "rb").read() - - -def main(): - ap = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - ap.add_argument("--out", required=True, help="output code object (.co)") - ap.add_argument( - "--check", - help="expected .co: fail unless the output is byte-identical (sha256)", - ) - ap.add_argument( - "--keep", - help="directory to keep the intermediates in (hipcc .s, patched .s, .o)", - ) - ap.add_argument( - "--verify-rt", - help="the #5195 .co (mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co): check that asm/rt.s " - "reassembles to its .text and kernel descriptor", - ) - a = ap.parse_args() - t = tools() - rt = os.path.join(HERE, "asm", "rt.s") - got = sha256(rt) - if got != RT_SHA256: - sys.exit(f"gen.py: asm/rt.s sha256 {got} != {RT_SHA256}") - tmp = tempfile.TemporaryDirectory() - work = a.keep or tmp.name - os.makedirs(work, exist_ok=True) - if a.verify_rt: - rco = os.path.join(work, "rt.co") - assemble(t, rt, rco) - for sec in (".text", ".rodata"): - same = section(t, rco, sec, work) == section(t, a.verify_rt, sec, work) - print(f"verify-rt {sec}: {'identical' if same else 'DIFFERENT'}") - if not same: - sys.exit(1) - text, marks, pads = PATCH.patch( - open(rt).read(), t["clang"], t["readelf"], t["hipcc"], work - ) - print("patched", marks, "pads", pads) - text, n = rename(text, OLD_NAME, NEW_NAME) - print(f"renamed {mangle(OLD_NAME)} -> {mangle(NEW_NAME)} ({n} occurrences)") - s_path = os.path.join(work, NEW_NAME + ".s") - open(s_path, "w").write(text) - co_tmp = os.path.join(work, NEW_NAME + ".co") - assemble(t, s_path, co_tmp) - out = os.path.abspath(a.out) - os.makedirs(os.path.dirname(out), exist_ok=True) - shutil.copyfile(co_tmp, out) - print(f"wrote {out} sha256 {sha256(out)}") - if a.check: - exp = sha256(a.check) - ok = exp == sha256(out) - print(f"check {a.check} sha256 {exp}: {'IDENTICAL' if ok else 'MISMATCH'}") - if not ok: - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/hsa/gfx950/mla_v4/nm_ps_src/splice/fin.hip b/hsa/gfx950/mla_v4/nm_ps_src/splice/fin.hip deleted file mode 100644 index 2f015baa888..00000000000 --- a/hsa/gfx950/mla_v4/nm_ps_src/splice/fin.hip +++ /dev/null @@ -1,273 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. -// -// Splice source: fused last-arriver combine, spliced by asm/patch.py at the end of every SPLIT piece -// (after the ASM epilogue stored the piece's partial o_acc[slot] / lse_acc[slot], before the walk's barrier). -// Only compiled to assembly (hipcc -S). ABI of the splice: kernarg pointer s[0:1], thread id v0, exec = all. -// Inputs (uniform, AGPRs written by the wrapper): a230 row r, a231 slot, a232 partition p, a233 hg, a234 tb, -// a235 te (unclamped), a236 kv_indptr[r], a237 kv_indptr[r+1]; from pre.hip: a248 per, a249 Pa, a250 org. -// Kernarg: 0x00 o_acc, 0x08 out, 0x10 lse_acc, 0x18 lse_out (0 = none), 0x98 cnt (uint32, zero at rest: -// [0, 2*65536) row counters, [2*65536, +16*512) reserved (unused), [CNTG, +4*1024) group counters), 0xb0 planner cfg, 0x120 sink. -// -// Protocol (per row r and head group hg). The pieces of r live in partitions pf..pl (pf = owner(r, 0), pl = owner(r, T-1) -// on the planner's cost line); a partition q in between holds a piece iff its tile range [ftg(q*per), ftg((q+1)*per)) is -// non-empty (partitions may be empty when per < 32); slot(q) = 2q, except the first piece: 2pf + (r is not the begin row -// of pf), whose parity travels through the counters (bit 30, added by the piece with tb == 0). -// * nq = pl - pf + 1 <= FIN_TREE_MIN ("narrow" row): every piece, after its partial is complete (s_waitcnt vmcnt(0) in -// every wave + barrier), does ONE agent-scope atomic add on cnt[2r+hg] of ntiles (+ parity). The piece that brings the -// tile sum to T(r) is the last arriver: it zeroes the counter (next launch starts from 0; ordered by the kernel -// boundary) and merges all pieces (+ sink) into out (bf16) and lse_out. -// * wider rows (two levels, fan-in FIN_G): partitions are grouped by (q - pf) / FIN_G; group counter -// cntg[4*q0 + 2*(q0 == pf) + hg] (q0 = first partition of the group; unique per (row, group)); the group's last arriver -// merges its pieces (no sink) into an fp32 partial + log2 lse stored IN PLACE in the slot of the group's first piece, -// then adds the group's tiles (+ parity for group 0) to the row counter; the row's last arriver merges the group -// partials (+ sink). The two levels spread the merge of wide rows over several CUs. -// Coherence: sc1 instruction scope instead of cache maintenance. The ASM's partial stores are built with sc1 -// (patch.py sc1: agent-scope write-through = LLVM's "store atomic monotonic agent" encoding on gfx942/950), group -// partials are stored with sc1 buffer stores, and every merge read is an sc1 load ("load atomic monotonic agent"): the -// release is s_waitcnt vmcnt(0) + barrier before the counter atomic, the acquire needs no buffer_inv; no buffer_wbl2. -// Merge math (FlashMLA combine, log2 domain): Mx = max(sk, lse_z); S = exp2(sk-Mx) + sum exp2(lse_z-Mx); -// out = sum exp2(lse_z-Mx) o_z / S; lse_out = (Mx + log2 S) ln2. Wave w merges heads hg*64 + 16w .. +15. -#include "hca_common.hip" - -typedef unsigned v4u __attribute__((ext_vector_type(4))); -struct KA { char b[336]; }; -// kernarg reads through the constant-address-space kernarg pointer -> s_load (the by-value struct compiled to vector loads) -#define KARG(T, off) (*(T const __attribute__((address_space(4)))*)((const __attribute__((address_space(4))) char*)__builtin_amdgcn_kernarg_segment_ptr() + (off))) -template __device__ __forceinline__ int rda() { - int v; - asm volatile("v_accvgpr_read_b32 %0, a%c1" : "=v"(v) : "i"(A)); - return __builtin_amdgcn_readfirstlane(v); -} -template __device__ __forceinline__ T* uni(T* p) { // make a pointer provably uniform (SGPR pair) - const unsigned long long u = (unsigned long long)p; - return (T*)(((unsigned long long)__builtin_amdgcn_readfirstlane((unsigned)(u >> 32)) << 32) | - (unsigned)__builtin_amdgcn_readfirstlane((unsigned)u)); -} -#define FIN_MAXJ 256 -#define FIN_NPF 100000 // rows spanning more than FIN_NPF partitions would be left to an external combine: none -#define FIN_TREE_MIN 8 // rows spanning more partitions than this use the two-level merge -#define FIN_G 4 // fan-in of the first level -#define FIN_LR 4 // lse values per lane held in registers (pieces 0..15); more pieces are re-read from memory -#define CNTG (2 * 65536 + 16 * 512) -#define CPOL_SC1 16 -#define OCP " sc1" - -extern "C" __global__ void __launch_bounds__(256) ps_merge(KA ka) { - __shared__ unsigned s_dec[2]; - __shared__ int s_slot[4][FIN_MAXJ]; - __shared__ float s_wt[4][FIN_MAXJ + 1][16]; - const int r = rda<230>(), slot = rda<231>(), p = rda<232>(), hg = rda<233>(), tb = rda<234>(), te = rda<235>(); - const int i0 = rda<236>(), i1 = rda<237>(); - const int per = rda<248>(), Pa = rda<249>(), org = rda<250>(); - float* o_acc = uni(KARG(float*, 0x00)); - __bf16* out = uni(KARG(__bf16*, 0x08)); - float* lse_acc = uni(KARG(float*, 0x10)); - float* lse_out = uni(KARG(float*, 0x18)); - unsigned* cnt = uni(KARG(unsigned*, 0x98)); - const int F = __builtin_amdgcn_readfirstlane(KARG(int, 0xb0)); - const float* sink = uni(KARG(const float*, 0x120)); - const int T = (i1 - i0 + TILE - 1) >> 5; - const int nt = min(te, T) - tb; - const int C0 = cfg_C0(F); - const int b = i0 - org + r * C0; - const int pf = min(Pa - 1, (b + C0) / per), pl = min(Pa - 1, (b + C0 + TILE * (T - 1)) / per); - auto ftg = [&](int pos) { const int d = pos - b - C0; const int i = d <= 0 ? 0 : (d + TILE - 1) >> 5; return min(i, T); }; - auto t0of = [&](int q) { return q == pf ? 0 : ftg(q * per); }; - auto t1of = [&](int q) { return q == pl ? T : ftg((q + 1) * per); }; - const int nq = pl - pf + 1; - if (nq > FIN_NPF) return; // never taken (FIN_NPF exceeds any partition count); pre.hip mirrors it for desc.comb - const bool tree = nq > FIN_TREE_MIN; - const int gi = tree ? (p - pf) / FIN_G : 0; - const int q0 = tree ? pf + gi * FIN_G : pf, q1 = tree ? min(pl, q0 + FIN_G - 1) : pl; - unsigned* c1 = tree ? cnt + CNTG + 4 * q0 + 2 * (q0 == pf) + hg : cnt + 2 * r + hg; - const int w = threadIdx.x >> 6, l = threadIdx.x & 63; - const int target1 = tree ? t1of(q1) - t0of(q0) : T; - const unsigned par = tb == 0 ? (unsigned)(slot & 1) << 30 : 0u; - - // one counter step: all waves' stores complete -> barrier -> thread 0 atomic (+reset if last) -> barrier -> decision - auto arrive = [&](unsigned* c, unsigned add, int target) -> unsigned { - asm volatile("s_waitcnt vmcnt(0)" ::: "memory"); - __syncthreads(); - if (threadIdx.x == 0) { - const unsigned nw = __hip_atomic_fetch_add(c, add, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT) + add; - const bool last = (int)(nw & 0x3fffffffu) == target; - if (last) __hip_atomic_store(c, 0u, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); - s_dec[0] = last ? 1u | ((nw >> 30) & 1u) << 1 : 0u; - } - __syncthreads(); - return s_dec[0]; - }; - unsigned dec = arrive(c1, (unsigned)nt | par, target1); - if (!(dec & 1u)) return; - - const int hw0 = hg * 64 + 16 * w; // first head of this wave - const int hl = l & 15, jl = l >> 4; - const __amdgpu_buffer_rsrc_t rl = __builtin_amdgcn_make_buffer_rsrc(lse_acc, 0, 0x7fffffff, 0x00020000); - const __amdgpu_buffer_rsrc_t ro = __builtin_amdgcn_make_buffer_rsrc(o_acc, 0, 0x7fffffff, 0x00020000); - auto ldl = [&](int j) { - return __builtin_bit_cast(float, __builtin_amdgcn_raw_buffer_load_b32(rl, (s_slot[w][j] * NH + hw0 + hl) * 4, 0, CPOL_SC1)); - }; - // slots of the pieces in partitions [qa, qb] (lanes over partitions) -> s_slot[w][0..np), one private copy per wave - auto slots_pieces = [&](int qa, int qb, unsigned pr) -> int { - int np = 0; - for (int j0 = qa; j0 <= qb; j0 += 64) { - const int q = j0 + l; - const bool has = q <= qb && t0of(q) < t1of(q); - const unsigned long long m = __ballot(has); - if (has) { - const int k = np + __popcll(m & ((1ull << l) - 1)); - if (k < FIN_MAXJ) s_slot[w][k] = 2 * q + (q == pf ? (int)pr : 0); - } - np += __popcll(m); - } - __builtin_amdgcn_s_waitcnt(0xc07f); // lgkmcnt(0) - __builtin_amdgcn_wave_barrier(); - return __builtin_amdgcn_readfirstlane(min(np, FIN_MAXJ)); - }; - - // ---- merge s_slot[w][0..np) for heads hw0..hw0+15. final: + sink -> out bf16 / lse_out (natural log); - // !final: no sink -> fp32 partial + log2 lse written in place into slot s_slot[w][0] (sc1 stores) ---- - auto merge = [&](int np, bool final) { - // weights: lane l = (piece j = jl + 4i, head hl); lse of pieces 0..15 in registers, one round trip - const float sk = final ? sink[hw0 + hl] * LOG2E : -__builtin_inff(); - float lz[FIN_LR]; -#pragma unroll - for (int i = 0; i < FIN_LR; ++i) lz[i] = ldl(min(jl + 4 * i, np - 1)); - // O pipeline: 2 head groups (8 heads) x np pieces, batch b = g*npp + j (npp = np rounded up to even), 16 x 1 KB - // sc1 loads per wave per batch into one of 2 register buffers; inline asm with explicit s_waitcnt vmcnt(16) before - // each consume (vmcnt completes in order; stores in between only make the wait conservative). The compiler's - // own waitcnt pass would drain vmcnt(0) at the loop head, serialising every batch. - constexpr int HB = 8; - const int npp = (np + 1) & ~1, nbt = 2 * npp; - v4f buf0[HB][2], buf1[HB][2]; - auto issue = [&](int bb, v4f (&bf)[HB][2]) { - bb = min(bb, nbt - 1); - const int g = bb >= npp ? 1 : 0, jj = min(bb - g * npp, np - 1); - const int sl = __builtin_amdgcn_readfirstlane(s_slot[w][jj]); - const float* base = uni(o_acc + ((size_t)sl * NH + hw0 + g * HB) * DV); - const int vo = 16 * l; - // the hazard recognizer does not look into inline asm: VALU (v_readfirstlane) SGPR write -> VMEM saddr read - // needs 5 wait states on gfx9 - asm volatile("s_nop 4" :: "s"(base)); -#pragma unroll - for (int k = 0; k < HB; ++k) - asm volatile("global_load_dwordx4 %0, %2, %3" OCP "\n\tglobal_load_dwordx4 %1, %2, %3 offset:1024" OCP - : "=&v"(bf[k][0]), "=&v"(bf[k][1]) : "v"(vo + k * DV * 4), "s"(base) : "memory"); - }; -#define FIN_TIE16(bf, CNT) \ - asm volatile("s_waitcnt vmcnt(" #CNT ")" : "+v"(bf[0][0]), "+v"(bf[0][1]), "+v"(bf[1][0]), "+v"(bf[1][1]), \ - "+v"(bf[2][0]), "+v"(bf[2][1]), "+v"(bf[3][0]), "+v"(bf[3][1]), "+v"(bf[4][0]), "+v"(bf[4][1]), \ - "+v"(bf[5][0]), "+v"(bf[5][1]), "+v"(bf[6][0]), "+v"(bf[6][1]), "+v"(bf[7][0]), "+v"(bf[7][1]) :: "memory") - issue(0, buf0); - float Mx = sk; -#pragma unroll - for (int i = 0; i < FIN_LR; ++i) if (jl + 4 * i < np) Mx = fmaxf(Mx, lz[i]); - for (int j = jl + 4 * FIN_LR; j < np; j += 4) Mx = fmaxf(Mx, ldl(j)); - Mx = fmaxf(Mx, __shfl_xor(Mx, 16)); Mx = fmaxf(Mx, __shfl_xor(Mx, 32)); - const bool dead = Mx == -__builtin_inff(); - float S = 0.f; -#pragma unroll - for (int i = 0; i < FIN_LR; ++i) { - if (jl + 4 * i < np) { - const float wt = dead ? 0.f : __builtin_amdgcn_exp2f(lz[i] - Mx); - s_wt[w][jl + 4 * i][hl] = wt; - S += wt; - } - } - for (int j = jl + 4 * FIN_LR; j < np; j += 4) { - const float wt = dead ? 0.f : __builtin_amdgcn_exp2f(ldl(j) - Mx); - s_wt[w][j][hl] = wt; - S += wt; - } - S += __shfl_xor(S, 16); S += __shfl_xor(S, 32); - if (final) S += dead ? 0.f : __builtin_amdgcn_exp2f(sk - Mx); - const float invS = S > 0.f ? 1.f / S : 0.f; - if (jl == 0) { - if (final) { - if (lse_out) lse_out[(size_t)r * NH + hw0 + hl] = (__builtin_amdgcn_logf(S) + Mx) * LN2; - } else { // group partial: log2-domain lse without sink (dead -> -inf); read later only by the row merge - const float lg = dead ? -__builtin_inff() : Mx + __builtin_amdgcn_logf(S); - __builtin_amdgcn_raw_buffer_store_b32(__builtin_bit_cast(unsigned, lg), rl, (s_slot[w][0] * NH + hw0 + hl) * 4, 0, - CPOL_SC1); - } - } - __builtin_amdgcn_s_waitcnt(0xc07f); - __builtin_amdgcn_wave_barrier(); - const int dslot = s_slot[w][0]; - // ---- accumulate: lane l owns dims 4l..4l+3 and 256+4l..+3 ---- - auto consume = [&](v4f (&acc)[HB][2], v4f (&bf)[HB][2], int j, int g) { -#pragma unroll - for (int k = 0; k < HB; ++k) { - const float wz = s_wt[w][j][g * HB + k]; - acc[k][0] += wz * bf[k][0]; acc[k][1] += wz * bf[k][1]; - } - }; -#pragma unroll - for (int g = 0; g < 2; ++g) { - v4f acc[HB][2]; -#pragma unroll - for (int k = 0; k < HB; ++k) acc[k][0] = acc[k][1] = (v4f){0, 0, 0, 0}; -#pragma unroll 1 - for (int j = 0; j + 1 < np; j += 2) { - issue(g * npp + j + 1, buf1); FIN_TIE16(buf0, 16); consume(acc, buf0, j, g); - issue(g * npp + j + 2, buf0); FIN_TIE16(buf1, 16); consume(acc, buf1, j + 1, g); - } - if (np & 1) { // odd np: batch np-1 is in buf0; the next group's first batch goes to buf1, moved to buf0 after it landed - issue(g * npp + npp, buf1); FIN_TIE16(buf0, 16); consume(acc, buf0, np - 1, g); - FIN_TIE16(buf1, 0); -#pragma unroll - for (int k = 0; k < HB; ++k) { buf0[k][0] = buf1[k][0]; buf0[k][1] = buf1[k][1]; } - } -#pragma unroll - for (int k = 0; k < HB; ++k) { - const float inv = __builtin_bit_cast(float, __builtin_amdgcn_readlane(__builtin_bit_cast(int, invS), g * HB + k)); - if (final) { - __bf16* dst = out + ((size_t)r * NH + hw0 + g * HB + k) * DV + 4 * l; - *(v4bf*)dst = __builtin_convertvector(acc[k][0] * inv, v4bf); - *(v4bf*)(dst + 256) = __builtin_convertvector(acc[k][1] * inv, v4bf); - } else { // in place into the group's first slot (its own reads of these addresses are complete: batch order) - const int off = ((dslot * NH + hw0 + g * HB + k) * DV + 4 * l) * 4; - __builtin_amdgcn_raw_buffer_store_b128(__builtin_bit_cast(v4u, acc[k][0] * inv), ro, off, 0, CPOL_SC1); - __builtin_amdgcn_raw_buffer_store_b128(__builtin_bit_cast(v4u, acc[k][1] * inv), ro, off + 1024, 0, CPOL_SC1); - } - } - } - // the last (clamped, unused) prefetch: all 16 destination registers stay allocated until it has landed - FIN_TIE16(buf0, 0); - }; - - if (!tree) { - merge(slots_pieces(pf, pl, (dec >> 1) & 1u), true); - } else { - // level 1: this group's pieces -> fp32 partial in the slot of the group's first piece (a 1-piece group is already there) - const int n1 = slots_pieces(q0, q1, (dec >> 1) & 1u); - if (n1 > 1) merge(n1, false); - // level 2: the group's tiles (+ the first-piece parity for group 0) go to the row counter - const unsigned par2 = gi == 0 ? (dec & 2u) << 29 : 0u; - const unsigned dec2 = arrive(cnt + 2 * r + hg, (unsigned)target1 | par2, T); - if (!(dec2 & 1u)) return; - // group partial slots: first piece of each group (lanes over groups) - const int ng = (nq + FIN_G - 1) / FIN_G; - const unsigned pr = (dec2 >> 1) & 1u; - int n2 = 0; - for (int g0 = 0; g0 < ng; g0 += 64) { - const int g = g0 + l; - int sl = -1; - if (g < ng) { - const int qa = pf + g * FIN_G, qb = min(pl, qa + FIN_G - 1); - for (int q = qa; q <= qb; ++q) - if (t0of(q) < t1of(q)) { sl = 2 * q + (q == pf ? (int)pr : 0); break; } - } - const unsigned long long m = __ballot(sl >= 0); - if (sl >= 0) { - const int k = n2 + __popcll(m & ((1ull << l) - 1)); - if (k < FIN_MAXJ) s_slot[w][k] = sl; - } - n2 += __popcll(m); - } - __builtin_amdgcn_s_waitcnt(0xc07f); - __builtin_amdgcn_wave_barrier(); - merge(__builtin_amdgcn_readfirstlane(min(n2, FIN_MAXJ)), true); - } -} diff --git a/hsa/gfx950/mla_v4/nm_ps_src/splice/hca_common.hip b/hsa/gfx950/mla_v4/nm_ps_src/splice/hca_common.hip deleted file mode 100644 index 93ef7662531..00000000000 --- a/hsa/gfx950/mla_v4/nm_ps_src/splice/hca_common.hip +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. -// -// Common definitions of the splice sources (pre.hip, fin.hip): the closed-form partition planner's cost line, -// the per-partition descriptor, the wave-cooperative 64-way boundary search and the WG -> (partition, head group) -// map. Only compiled as part of pre.hip / fin.hip (hipcc -S); nothing here is a kernel. -#include -#include - -typedef int v4i __attribute__((ext_vector_type(4))); -typedef float v4f __attribute__((ext_vector_type(4))); -typedef __bf16 v4bf __attribute__((ext_vector_type(4))); - -#define TILE 32 -#define NH 128 // heads -#define DV 512 -#define LOG2E 1.4426950408889634f -#define LN2 0.6931471805599453f - -// ---------------- closed-form partition planner: cost line ---------------- -// Planner config word Fcfg = F | (MT << 8): -// C0 = 32*F + 16 (fixed cost per row piece, token units, charged once per row on the cost line); -// MT = minimum partition size in tiles beyond one C0 (0 = off): the number of ACTIVE partitions is -// Pa = clamp(total / (C0 + 32*MT), 1, P), per = ceil(total / Pa); partitions p >= Pa are empty. -// A cut inside a row adds at most one extra C0 per partition (the fixed cost is the same for every -// partition, so balance is unaffected); Pa bounds how many pieces tiny shapes are cut into. -__device__ __forceinline__ int cfg_C0(int Fcfg) { return 32 * (Fcfg & 0xff) + 16; } -__device__ __forceinline__ int active_parts(int total, int P, int Fcfg) { - const int MT = Fcfg >> 8; - if (MT <= 0) return P; - return max(1, min(P, total / (cfg_C0(Fcfg) + 32 * MT))); -} - -struct __attribute__((aligned(32))) Desc { - // comb = 1 iff begin_row is split and its LAST piece is in this partition (begin_tile > 0 and the row ends here): - // this partition's combine WG merges that row (only used by an external combine; the fused combine clears it). - int begin_row, end_row, begin_tile, end_tile, flags, kv_begin_tok, comb, pad1; -}; - -// ---------------- wave-cooperative planner search ---------------- -// The two partition-boundary binary searches of partition p (positions p*per and (p+1)*per on the cost -// line), 64-way: each round every lane samples one row of the current interval, a ballot counts the samples -// with base() <= pos, and the interval shrinks 64x. N <= 64: 1 round, N <= 4096: 2 rounds (+ the -// ind[N], ind[N-1] loads, issued in parallel with round 0). Each lane loads ind[j-1..j+2] so the final -// step-1 round also yields every neighbour value the descriptor needs (no extra dependent round). -// Must be called by all 64 lanes of a wave with uniform arguments. -struct WRes { int r, im1, i0, i1, i2; }; // row, ind[r-1], ind[r], ind[r+1], ind[r+2] (clamped) - -__device__ __forceinline__ int rdl(int v, int lane) { return __builtin_amdgcn_readlane(v, lane); } - -__device__ __forceinline__ void wsearch2(const int* __restrict__ ind, int N, int Fcfg, int tN, int org, int P, int p, - int& total, int& per, int& posA, int& posB, WRes& A, WRes& B) { - const int C0 = cfg_C0(Fcfg); // positions are relative to org = ind[0] (tN is ind[N], absolute) - const int lane = threadIdx.x & 63; - int loA = 0, lenA = N, stA = (N + 63) >> 6; - int loB = 0, lenB = N, stB = stA; - bool doneA = false, doneB = false, first = true; - total = 0; per = 1; posA = posB = 0; - while (!(doneA && doneB)) { - const bool vA = !doneA && lane * stA < lenA, vB = !doneB && lane * stB < lenB; - const int jA = vA ? loA + lane * stA : loA, jB = vB ? loB + lane * stB : loB; - const int a_m1 = ind[max(jA - 1, 0)], a0 = ind[jA], a1 = ind[jA + 1], a2 = ind[min(jA + 2, N)]; - const int b_m1 = ind[max(jB - 1, 0)], b0 = ind[jB], b1 = ind[jB + 1], b2 = ind[min(jB + 2, N)]; - if (first) { - total = tN - org + N * C0; - const int Pa = active_parts(total, P, Fcfg); - per = max(1, (total + Pa - 1) / Pa); - posA = min(p, Pa) * per; // p >= Pa: posA = Pa*per >= total -> empty (no int overflow) - posB = (p >= Pa - 1) ? posA : (p + 1) * per; - first = false; - } - const int cA = __popcll(__ballot(vA && a0 - org + jA * C0 <= posA)); - const int cB = __popcll(__ballot(vB && b0 - org + jB * C0 <= posB)); - if (!doneA) { - if (stA == 1) { - A.r = loA + cA - 1; A.im1 = rdl(a_m1, cA - 1); A.i0 = rdl(a0, cA - 1); A.i1 = rdl(a1, cA - 1); A.i2 = rdl(a2, cA - 1); - doneA = true; - } else { - loA += (cA - 1) * stA; lenA = min(stA, lenA - (cA - 1) * stA); stA = (lenA + 63) >> 6; - } - } - if (!doneB) { - if (stB == 1) { - B.r = loB + cB - 1; B.im1 = rdl(b_m1, cB - 1); B.i0 = rdl(b0, cB - 1); B.i1 = rdl(b1, cB - 1); B.i2 = rdl(b2, cB - 1); - doneB = true; - } else { - loB += (cB - 1) * stB; lenB = min(stB, lenB - (cB - 1) * stB); stB = (lenB + 63) >> 6; - } - } - } -} - -__device__ __forceinline__ void wg_map(int wg, int P, int& p, int& hg) { - if ((P & 7) == 0) { // keep the hg pair and neighbouring partitions on one XCD (round-robin dispatch, inferred) - int xcd = wg & 7, loc = wg >> 3; - p = xcd * (P >> 3) + (loc >> 1); hg = loc & 1; - return; - } - p = wg >> 1; hg = wg & 1; -} diff --git a/hsa/gfx950/mla_v4/nm_ps_src/splice/pre.hip b/hsa/gfx950/mla_v4/nm_ps_src/splice/pre.hip deleted file mode 100644 index 9547b878fdd..00000000000 --- a/hsa/gfx950/mla_v4/nm_ps_src/splice/pre.hip +++ /dev/null @@ -1,172 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. -// -// Splice source: the in-kernel split planner, compiled for gfx950 and spliced into the prologue of the patched ASM by -// asm/patch.py. This file is only compiled to assembly (hipcc -S); the kernel is never launched. -// ABI assumed by the splice (checked by splice.py): kernarg pointer in s[0:1], workgroup id x in s2, thread id in v0. -// Outputs (uniform) in AGPRs: a243 begin_row, a244 end_row, a245 begin_tile, a246 end_tile, a247 flags, -// a248 per, a249 Pa (active partitions), a250 org = kv_indptr[0] -- the planner's cost-line constants, used by the -// fused combine (fin.hip). -// The descriptor is also published to desc[p] (hg == 0 WG, thread 0). -#include "hca_common.hip" -#define FIN_NPF 100000 // split rows spanning <= FIN_NPF partitions are merged by the fused combine (fin.hip): all rows - -struct KA { char b[336]; }; -extern "C" __device__ int __ockl_wfred_max_i32(int); -// LDS-only barrier: __syncthreads() also waits for every outstanding global load (fence), which would make the first -// barrier wait for a kv_indptr round trip -#define NS_BAR() do { __builtin_amdgcn_s_waitcnt(0xc07f); __builtin_amdgcn_s_barrier(); } while (0) -// Split / no-split choice: no-split iff Minf - (per + C0) < NS_WANY (token units; 384 = 12 tiles; an empirically -// chosen threshold). Minf = max partition load of the no-split plan, exact, per row in closed form: row r (K > 0) -// goes to partition q(r) = min(Pa-1, m_r / per), m_r = b + C0 + 32*(ceil(T/2)-1), load += C0 + 32T. -// WG-cooperative (all 256 threads; every WG computes the same result): thread t walks rows [t*c, t*c+c), c = ceil(N/256), -// adding each row's load into an LDS per-partition array (atomics); the max by an LDS atomic. -// Code size matters (straight-line instructions executed at kernel start cost instruction fetch): no unrolling, one -// division per row (float reciprocal estimate + exact correction). -#define NS_WANY 384 -#define NS_MAXP 1024 -// Split descriptor (closed-form planner, cut at p*per / (p+1)*per) + the no-split descriptor (every cut snapped to the -// nearer end of the row it falls in: tile i -> 0 if 2i < T, else T) from the same wsearch2 (one search) -__device__ __forceinline__ void plan_both(const int* __restrict__ ind, int N, int P, int Fcfg, int p, Desc& ds, Desc& dn, - int& total, int& per) { - const int C0 = cfg_C0(Fcfg); - const int tN = ind[N], tNm1 = ind[max(N - 1, 0)], org = ind[0]; - int posA, posB; - WRes A, B; - wsearch2(ind, N, Fcfg, tN, org, P, p, total, per, posA, posB, A, B); - const int Pa = active_parts(total, P, Fcfg); - auto Tof = [](int a, int b) { return (b - a + TILE - 1) >> 5; }; - auto ftg = [&](int r, int ir, int T_, int pos) { - int d = pos - (ir - org + r * C0) - C0; - int i = d <= 0 ? 0 : (d + TILE - 1) >> 5; - return min(i, T_); - }; - // ---- split ---- - int br, bt, Tb; - const int T0 = Tof(A.i0, A.i1); - const int iA = posA >= total ? 0 : ftg(A.r, A.i0, T0, posA); - if (posA >= total) { br = N; bt = 0; Tb = 0; } - else if (iA < T0) { br = A.r; bt = iA; Tb = T0; } - else { br = A.r + 1; bt = 0; Tb = (br < N) ? Tof(A.i1, A.i2) : 0; } - int er, et, Tm1, T0e; - Tm1 = Tof(B.im1, B.i0); T0e = Tof(B.i0, B.i1); - const int TN1 = Tof(tNm1, tN); - int Te_minus = 0; - const bool endN = p >= Pa - 1 || posB >= total; - const int iB = endN ? 0 : ftg(B.r, B.i0, T0e, posB); - if (endN) { er = N; et = 0; Te_minus = TN1; } - else if (iB < T0e) { er = B.r; et = iB; Te_minus = Tm1; } - else { er = B.r + 1; et = 0; Te_minus = T0e; } - ds.comb = ds.pad1 = 0; - if (!((br < er) || (br == er && bt < et))) { - ds.begin_row = N; ds.end_row = -1; ds.begin_tile = ds.end_tile = ds.flags = ds.kv_begin_tok = 0; - } else { - int Te; - if (et == 0) { er -= 1; et = Te_minus; Te = Te_minus; } - else Te = (er == B.r) ? T0e : 0; - const int f0 = (bt > 0) || (er == br && et < Tb); - const int f1 = (et < Te) || (er == br && bt > 0); - ds.begin_row = br; ds.end_row = er; ds.begin_tile = bt; ds.end_tile = et; - ds.flags = f0 | (f1 << 1); ds.kv_begin_tok = (br == A.r) ? A.i0 : A.i1; - ds.comb = bt > 0 && (er > br || et == Tb); - } - // ---- no-split ---- - const int nb = posA >= total ? N : ((2 * iA < T0 || T0 == 0) ? A.r : A.r + 1); - const int ne = endN ? N : ((2 * iB < T0e || T0e == 0) ? B.r : B.r + 1); - dn.comb = dn.pad1 = 0; dn.begin_tile = 0; dn.flags = 0; - if (nb >= ne) { - dn.begin_row = N; dn.end_row = -1; dn.end_tile = dn.kv_begin_tok = 0; - } else { - dn.begin_row = nb; dn.end_row = ne - 1; - dn.end_tile = ne == N ? TN1 : (ne == B.r ? Tm1 : T0e); - dn.kv_begin_tok = nb == A.r ? A.i0 : A.i1; - } -} -// kernarg reads through the constant-address-space kernarg pointer -> s_load (the by-value struct compiled to vector loads) -#define KARG(T, off) (*(T const __attribute__((address_space(4)))*)((const __attribute__((address_space(4))) char*)__builtin_amdgcn_kernarg_segment_ptr() + (off))) - -extern "C" __global__ void __launch_bounds__(256) ps_plan(KA ka) { - const int* ind = KARG(const int*, 0x40); - Desc* desc = KARG(Desc*, 0x60); - const int P = KARG(int, 0xa0), N = KARG(int, 0xa4), F = KARG(int, 0xb0); - int p, hg; - wg_map(blockIdx.x, P, p, hg); - int total, per; - // Wave 0 runs the planner search (plan_both: split and no-split descriptors from one search) while waves - // 1..3 compute the no-split load profile (ns rows, LDS atomics); the descriptors travel through LDS, so the decision - // overlaps the search. Every wave executes the same 4 barriers. - Desc d0; - if (N > 0) { - __shared__ int s_load[NS_MAXP]; - __shared__ int s_max; - __shared__ Desc s_dd[2]; - __shared__ int s_tp[3]; // total, per, Pa (wave 0's search) - const int C0 = cfg_C0(F), tid = threadIdx.x, wv = tid >> 6; - // LDS init before any memory dependence (the first barrier must not wait for a kv_indptr round trip) -#pragma unroll - for (int i = 0; i < NS_MAXP / 256; ++i) s_load[tid + 256 * i] = 0; - if (tid == 0) s_max = 0; - int r0 = 0, r1 = 0; - if (wv) { - const int cs = (N + 191) / 192; - r0 = min(N, (tid - 64) * cs); r1 = min(N, r0 + cs); - } - NS_BAR(); - if (wv == 0) { - Desc ds, dn; - int tt, pp; - plan_both(ind, N, P, F, p, ds, dn, tt, pp); - if (tid == 0) { s_dd[0] = ds; s_dd[1] = dn; s_tp[0] = tt; s_tp[1] = pp; s_tp[2] = active_parts(tt, P, F); } - } else { - int i0 = ind[r0], i1 = ind[min(r0 + 1, N)]; - const int org = ind[0], tot0 = ind[N] - org + N * C0; // == wsearch2's cost-line constants - const int Pa0 = active_parts(tot0, P, F), per0 = max(1, (tot0 + Pa0 - 1) / Pa0); - const float rf = __builtin_amdgcn_rcpf((float)per0); -#pragma unroll 1 - for (int r = r0; r < r1; ++r) { - if (r != r0) i1 = ind[r + 1]; - const int T = (i1 - i0 + TILE - 1) >> 5; - if (T > 0) { - const int m = i0 - org + r * C0 + C0 + TILE * (((T + 1) >> 1) - 1); - // q <= total/per <= Pa <= 1024: the float estimate has |error| < 1 (relative error ~2^-22), one correction - int q = (int)((float)m * rf); - const int rm = m - q * per0; - q += rm < 0 ? -1 : (rm >= per0 ? 1 : 0); - atomicAdd(&s_load[min(Pa0 - 1, q)], C0 + TILE * T); - } - i0 = i1; - } - } - NS_BAR(); - const int tot0 = s_tp[0], per0 = s_tp[1], Pa0 = s_tp[2]; - int mx = 0; - for (int i = tid; i < Pa0; i += 256) mx = max(mx, s_load[i]); - // wave max first: 256 lanes atomicMax-ing one LDS word would serialise - mx = __ockl_wfred_max_i32(mx); // DPP wave reduction - if ((tid & 63) == 0 && mx) atomicMax(&s_max, mx); - NS_BAR(); - d0 = s_dd[(s_max - per0 - C0 < NS_WANY) ? 1 : 0]; - NS_BAR(); // every wave has read LDS before any wave enters the ASM (which reuses LDS) - total = tot0; per = per0; - } else { // N = 0: all partitions empty (without instantiating a second search) - d0.begin_row = N; d0.end_row = -1; d0.begin_tile = d0.end_tile = d0.flags = d0.kv_begin_tok = d0.comb = d0.pad1 = 0; - total = 0; per = 1; - } - const int Pa = active_parts(total, P, F); - Desc d = d0; - // fused combine (fin.hip): a split row spanning <= FIN_NPF partitions (every row) is merged in the attention kernel - // by its last-arriving piece; desc.comb is cleared for it. Same closed form as fin.hip. - if (d.comb) { - const int br = d.begin_row, org = ind[0], C0 = cfg_C0(F); - const int b = ind[br] - org + br * C0, T = (ind[br + 1] - ind[br] + TILE - 1) >> 5; - const int pf = min(Pa - 1, (b + C0) / per), pl = min(Pa - 1, (b + C0 + TILE * (T - 1)) / per); - if (pl - pf + 1 <= FIN_NPF) d.comb = 0; - } - if (hg == 0 && threadIdx.x == 0) desc[p] = d; - asm volatile("v_accvgpr_write_b32 a243, %0\n\tv_accvgpr_write_b32 a244, %1\n\tv_accvgpr_write_b32 a245, %2\n\t" - "v_accvgpr_write_b32 a246, %3\n\tv_accvgpr_write_b32 a247, %4\n\tv_accvgpr_write_b32 a248, %5\n\t" - "v_accvgpr_write_b32 a249, %6\n\tv_accvgpr_write_b32 a250, %7" - :: "v"(d.begin_row), "v"(d.end_row), "v"(d.begin_tile), "v"(d.end_tile), "v"(d.flags), "v"(per), "v"(Pa), - "v"(N > 0 ? ind[0] : 0) - : "memory"); -} diff --git a/hsa/gfx950/mla_v4/nm_ps_src/splice/splice.py b/hsa/gfx950/mla_v4/nm_ps_src/splice/splice.py deleted file mode 100644 index 86048e5a09c..00000000000 --- a/hsa/gfx950/mla_v4/nm_ps_src/splice/splice.py +++ /dev/null @@ -1,95 +0,0 @@ -# SPDX-License-Identifier: MIT -# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved. -"""Compile a HIP splice source (splice/*.hip) to gfx950 assembly with `hipcc -S` and extract the body of one kernel -as a text block that asm/patch.py inlines into the patched ASM. - -Extraction rules (checked here): - - the kernel ABI must be: user SGPRs = kernarg pointer only (s[0:1]), workgroup id x only (s2), workitem id x (v0), - no kernarg preload, no scratch, no LDS (group segment 0) unless allow_lds; - - register budget: next_free_vgpr <= vmax, next_free_sgpr <= smax (the wrapper keeps its state above these); - - labels .LBBk_n -> _n; every s_endpgm -> s_branch _RET (the caller places _RET after the block); - - assembler directives and comments are dropped. -""" - -import os -import re -import subprocess - -HERE = os.path.dirname(os.path.abspath(__file__)) -HIP_FLAGS = [ - "--offload-arch=gfx950", - "-O3", - "--offload-device-only", - "-std=c++20", - "-w", -] - - -def compile_s(hipcc, src, out, extra=()): - subprocess.run( - [hipcc, *HIP_FLAGS, *extra, "-S", src, "-o", out], - check=True, - cwd=os.path.dirname(src), - ) - return out - - -def body( - hipcc, src, kernel, prefix, workdir, vmax=240, smax=100, allow_lds=False, extra=() -): - """-> (block text, kernel descriptor fields {name: int})""" - out_s = os.path.join(workdir, os.path.basename(src)[:-4] + ".s") - s = open(compile_s(hipcc, src, out_s, extra)).read().split("\n") - i0 = next(i for i, ln in enumerate(s) if ln.split(";")[0].strip() == f"{kernel}:") - i1 = next(i for i in range(i0, len(s)) if s[i].startswith(".Lfunc_end")) - # kernel descriptor block - k0 = next( - i for i in range(i0, len(s)) if s[i].split() == [".amdhsa_kernel", kernel] - ) - k1 = next(i for i in range(k0, len(s)) if s[i].strip() == ".end_amdhsa_kernel") - kd = {} - for ln in s[k0 + 1 : k1]: - t = ln.split() - if len(t) == 2 and t[0].startswith(".amdhsa_"): - kd[t[0][8:]] = int(t[1]) - req = dict( - user_sgpr_count=2, - user_sgpr_kernarg_segment_ptr=1, - user_sgpr_dispatch_ptr=0, - user_sgpr_queue_ptr=0, - user_sgpr_kernarg_preload_length=0, - system_sgpr_workgroup_id_y=0, - system_sgpr_workgroup_id_z=0, - system_vgpr_workitem_id=0, - private_segment_fixed_size=0, - enable_private_segment=0, - ) - for k, v in req.items(): - assert kd.get(k, v) == v, (kernel, k, kd.get(k)) - if not allow_lds: - assert kd["group_segment_fixed_size"] == 0, kd["group_segment_fixed_size"] - assert kd["next_free_vgpr"] <= vmax and kd["next_free_sgpr"] <= smax, ( - kernel, - kd["next_free_vgpr"], - kd["next_free_sgpr"], - ) - out = [] - for ln in s[i0 + 1 : min(i1, k0)]: - t = ln.split(";")[0].rstrip() - if not t.strip() or t.strip().startswith("."): - m = re.match(r"^\.LBB\d+_(\d+):", t) - if m: - out.append(f"{prefix}_{m.group(1)}:") - continue - t = re.sub(r"\.LBB\d+_(\d+)", lambda m: f"{prefix}_{m.group(1)}", t) - if t.strip() == "s_endpgm": - t = f"\ts_branch {prefix}_RET" - out.append(t) - # every register named must stay inside the budget (inline-asm AGPR outputs are allowed) - for ln in out: - for m in re.finditer(r"\bv\[?(\d+)(?::(\d+))?", ln): - assert int(m.group(2) or m.group(1)) < vmax, ln - for m in re.finditer(r"\bs\[?(\d+)(?::(\d+))?", ln): - if re.match(r"\s*s_", ln) or "," in ln: - assert int(m.group(2) or m.group(1)) < smax, ln - return "\n".join(out), kd From 5166ffbe0e01802ae359cd061562e701eab0a81a Mon Sep 17 00:00:00 2001 From: yihonglie Date: Sat, 3 Oct 2026 22:16:11 -0500 Subject: [PATCH 11/13] [MLA v4 nm] Satisfy ruff in the persistent decode changes dict literals for the workspace kwargs and the test inputs; one with-statement for the capture-time rejection check. No behaviour change. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie --- aiter/mla.py | 2 +- op_tests/test_mla_v4_nm.py | 23 +++++++++++------------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/aiter/mla.py b/aiter/mla.py index 2a8b07131c6..a94a9e8247a 100644 --- a/aiter/mla.py +++ b/aiter/mla.py @@ -493,7 +493,7 @@ def get_mla_v4_nm_ps_workspace(device="cuda", num_partitions=128) -> MlaV4NmPsWo "CUDA-graph capture" ) P = num_partitions - i32 = dict(dtype=dtypes.i32, device=device) + i32 = {"dtype": dtypes.i32, "device": device} return MlaV4NmPsWorkspace( o_acc=torch.empty(2 * P, 128, 512, dtype=dtypes.fp32, device=device), lse_acc=torch.empty(2 * P, 128, dtype=dtypes.fp32, device=device), diff --git a/op_tests/test_mla_v4_nm.py b/op_tests/test_mla_v4_nm.py index beee78b380a..f8e869e9f5d 100644 --- a/op_tests/test_mla_v4_nm.py +++ b/op_tests/test_mla_v4_nm.py @@ -2641,15 +2641,15 @@ def _ps_inputs(kv_lens, seed=0, pool=None, device="cuda"): 0, pool, (max(total, 1),), generator=g, device=device, dtype=torch.int32 ) sink = torch.randn(_PS_HEADS, generator=g, device=device) * 2.0 - return dict( - q_packed=qp, - q_rope=qr.contiguous(), - kv_packed=kp, - kv_rope=kr.contiguous(), - kv_indptr=kv_indptr, - kv_page_indices=kv_page_indices, - sink=sink, - ) + return { + "q_packed": qp, + "q_rope": qr.contiguous(), + "kv_packed": kp, + "kv_rope": kr.contiguous(), + "kv_indptr": kv_indptr, + "kv_page_indices": kv_page_indices, + "sink": sink, + } def _ps_call(inp, ws, out=None, lse=None, return_lse=True, kv_indptr=None): @@ -2837,9 +2837,8 @@ def run(): graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): run() - with pytest.raises(RuntimeError): - with torch.cuda.graph(torch.cuda.CUDAGraph()): - aiter.mla.get_mla_v4_nm_ps_workspace() + with pytest.raises(RuntimeError), torch.cuda.graph(torch.cuda.CUDAGraph()): + aiter.mla.get_mla_v4_nm_ps_workspace() g = np.random.default_rng(7) for lens in ( From ba4551d26e9b111536ace8557cc8c4d713a49c3a Mon Sep 17 00:00:00 2001 From: yihonglie Date: Sun, 4 Oct 2026 00:03:39 -0500 Subject: [PATCH 12/13] [MLA v4 nm] Expose the split count as a host-only helper get_mla_v4_nm_num_kv_splits(num_seqs, num_heads, kv_len) returns the split count the plan helper picks, with no device work. The uniform split_indptr for it is [0, s, 2s, ...], and any prefix of a longer one is valid, so a caller can keep one constant buffer per split count and slice it instead of writing a split_indptr before every call. get_mla_v4_nm_split_plan now builds on it; its behaviour is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie --- aiter/mla.py | 38 +++++++++++++++++++++++--------------- op_tests/test_mla_v4_nm.py | 12 ++++++++++++ 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/aiter/mla.py b/aiter/mla.py index a94a9e8247a..6531c7cc851 100644 --- a/aiter/mla.py +++ b/aiter/mla.py @@ -403,24 +403,17 @@ def _v4_nm_pick_num_kv_splits(cost, num_seqs, tg_factor, kv_len, cu_num): return best_splits -def get_mla_v4_nm_split_plan( - num_seqs, num_heads, kv_len, *, split_indptr=None, device="cuda" -) -> MlaV4NmSplitPlan: - """KV split plan for `mla_decode_fwd_v4_nm`. +def get_mla_v4_nm_num_kv_splits(num_seqs, num_heads, kv_len) -> int: + """`num_kv_splits` for `mla_decode_fwd_v4_nm` (host only, no device work). `num_seqs` / `num_heads` are the decode call's `qo_indptr.shape[0] - 1` and `q.size(1)`; `kv_len` is the per-seq KV length to plan for (the - longest one the call can see). The split count minimizes the latency - model above where the arch has coefficients, and is the wrapper's own - occupancy-only pick elsewhere. - - The split count is a host int that depends only on these arguments, and - `split_indptr` is the uniform `[0, s, 2s, ..., num_seqs * s]`. Pass a - persistent int32 `split_indptr` of at least `num_seqs + 1` entries to have - it filled in place (one device launch, no host sync): CUDA-graph callers - keep the buffer alive across replays and rebuild the plan with the same - arguments before each replay. Pass the plan on as - `mla_decode_fwd_v4_nm(..., **plan._asdict())`. + longest one the call can see). Minimizes the latency model above where + the arch has coefficients, and is the wrapper's own occupancy-only pick + elsewhere. The matching `split_indptr` is the uniform + `[0, s, 2s, ..., num_seqs * s]`; any prefix of a longer uniform buffer is + one, so callers can keep a constant buffer per split count and slice it + instead of writing one per call. """ tg_factor = max(1, -(-num_heads // 64)) # ceil(num_heads / 64) cost = _V4_NM_SPLIT_COST.get(get_gfx()) @@ -439,6 +432,21 @@ def get_mla_v4_nm_split_plan( tg_factor, 1, # ignore_total_kv, as in mla_decode_fwd_v4_nm ) + return num_kv_splits + + +def get_mla_v4_nm_split_plan( + num_seqs, num_heads, kv_len, *, split_indptr=None, device="cuda" +) -> MlaV4NmSplitPlan: + """KV split plan for `mla_decode_fwd_v4_nm`: the split count of + `get_mla_v4_nm_num_kv_splits` and its uniform `split_indptr`. + + Pass a persistent int32 `split_indptr` of at least `num_seqs + 1` entries + to have it filled in place (one device launch, no host sync); otherwise a + new one is allocated on `device`. Pass the plan on as + `mla_decode_fwd_v4_nm(..., **plan._asdict())`. + """ + num_kv_splits = get_mla_v4_nm_num_kv_splits(num_seqs, num_heads, kv_len) if split_indptr is None: split_indptr = torch.empty(num_seqs + 1, dtype=torch.int32, device=device) else: diff --git a/op_tests/test_mla_v4_nm.py b/op_tests/test_mla_v4_nm.py index f8e869e9f5d..6a9f157a829 100644 --- a/op_tests/test_mla_v4_nm.py +++ b/op_tests/test_mla_v4_nm.py @@ -1169,6 +1169,18 @@ def test_v4_nm_split_planner_picks(num_seqs, kv_len, expected): ) +@needs_gfx950 +@pytest.mark.parametrize( + "num_seqs,kv_len", [(7, 128), (7, 1152), (56, 1152), (14, 8320)] +) +def test_v4_nm_num_kv_splits_matches_plan(num_seqs, kv_len): + """The host-only count is the plan's split count, with no device work.""" + plan = aiter.mla.get_mla_v4_nm_split_plan(num_seqs, 128, kv_len) + assert aiter.mla.get_mla_v4_nm_num_kv_splits(num_seqs, 128, kv_len) == ( + plan.num_kv_splits + ) + + @needs_gfx950 def test_v4_nm_split_plan_fills_buffer_in_place(): buf = torch.full((10,), -1, dtype=torch.int32, device="cuda") From 7c96c0318e825539e5d62d4462db1ab4758a15d9 Mon Sep 17 00:00:00 2001 From: yihonglie Date: Sun, 4 Oct 2026 00:10:23 -0500 Subject: [PATCH 13/13] [MLA v4 nm] Drop get_mla_v4_nm_split_plan; the split count is the API Nothing calls the plan helper or MlaV4NmSplitPlan any more: callers take the count from get_mla_v4_nm_num_kv_splits and pair it with the uniform split_indptr [0, s, 2s, ...]. The tests build that pair with a local helper, and the mla_decode_fwd_v4_nm docstring points at the count helper. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: yihonglie --- aiter/mla.py | 40 +++--------------------------- op_tests/test_mla_v4_nm.py | 51 +++++++++++++++++++------------------- 2 files changed, 28 insertions(+), 63 deletions(-) diff --git a/aiter/mla.py b/aiter/mla.py index 6531c7cc851..a43084f052c 100644 --- a/aiter/mla.py +++ b/aiter/mla.py @@ -380,13 +380,6 @@ def get_meta_param( _V4_NM_MAX_SPLITS = 16 -class MlaV4NmSplitPlan(NamedTuple): - """KV split plan for `mla_decode_fwd_v4_nm`; the fields are its kwargs.""" - - num_kv_splits: int - split_indptr: torch.Tensor - - @functools.lru_cache(maxsize=1024) def _v4_nm_pick_num_kv_splits(cost, num_seqs, tg_factor, kv_len, cu_num): """Split count minimizing the latency model above with coefficients `cost`.""" @@ -435,33 +428,6 @@ def get_mla_v4_nm_num_kv_splits(num_seqs, num_heads, kv_len) -> int: return num_kv_splits -def get_mla_v4_nm_split_plan( - num_seqs, num_heads, kv_len, *, split_indptr=None, device="cuda" -) -> MlaV4NmSplitPlan: - """KV split plan for `mla_decode_fwd_v4_nm`: the split count of - `get_mla_v4_nm_num_kv_splits` and its uniform `split_indptr`. - - Pass a persistent int32 `split_indptr` of at least `num_seqs + 1` entries - to have it filled in place (one device launch, no host sync); otherwise a - new one is allocated on `device`. Pass the plan on as - `mla_decode_fwd_v4_nm(..., **plan._asdict())`. - """ - num_kv_splits = get_mla_v4_nm_num_kv_splits(num_seqs, num_heads, kv_len) - if split_indptr is None: - split_indptr = torch.empty(num_seqs + 1, dtype=torch.int32, device=device) - else: - split_indptr = split_indptr[: num_seqs + 1] - torch.arange( - 0, - (num_seqs + 1) * num_kv_splits, - num_kv_splits, - dtype=torch.int32, - device=split_indptr.device, - out=split_indptr, - ) - return MlaV4NmSplitPlan(num_kv_splits, split_indptr) - - # Workspace of the persistent v4 nm decode kernel (mla_decode_v4_ps_asm). # cnt: [0, 2*65536) row counters, [2*65536, +16*512) reserved, # [.., +4*1024) group counters. @@ -1891,9 +1857,9 @@ def mla_decode_fwd_v4_nm( path. Pass an explicit int to override. Note V4 nm is always non-persistent, so only that branch of `get_meta_param` applies. Callers that know their per-seq KV length (and CUDA-graph callers, - whose `kv_page_indices` is typically capacity-sized) should build a - plan with `get_mla_v4_nm_split_plan` and pass it as - `**plan._asdict()`. + whose `kv_page_indices` is typically capacity-sized) should pick it + with `get_mla_v4_nm_num_kv_splits` and pass it with the uniform + `split_indptr` `[0, s, 2s, ...]`. Multi-pass mode (`num_kv_splits > 1`): 1. If `split_indptr` is None, build a uniform one: diff --git a/op_tests/test_mla_v4_nm.py b/op_tests/test_mla_v4_nm.py index 6a9f157a829..32e97fa909e 100644 --- a/op_tests/test_mla_v4_nm.py +++ b/op_tests/test_mla_v4_nm.py @@ -1169,28 +1169,27 @@ def test_v4_nm_split_planner_picks(num_seqs, kv_len, expected): ) +def _uniform_split(num_seqs, num_heads, kv_len, device="cuda"): + """mla_decode_fwd_v4_nm split kwargs: aiter's split count and its uniform + split_indptr [0, s, 2s, ...].""" + s = aiter.mla.get_mla_v4_nm_num_kv_splits(num_seqs, num_heads, kv_len) + return { + "num_kv_splits": s, + "split_indptr": torch.arange( + 0, (num_seqs + 1) * s, s, dtype=torch.int32, device=device + ), + } + + @needs_gfx950 @pytest.mark.parametrize( "num_seqs,kv_len", [(7, 128), (7, 1152), (56, 1152), (14, 8320)] ) -def test_v4_nm_num_kv_splits_matches_plan(num_seqs, kv_len): - """The host-only count is the plan's split count, with no device work.""" - plan = aiter.mla.get_mla_v4_nm_split_plan(num_seqs, 128, kv_len) - assert aiter.mla.get_mla_v4_nm_num_kv_splits(num_seqs, 128, kv_len) == ( - plan.num_kv_splits - ) - - -@needs_gfx950 -def test_v4_nm_split_plan_fills_buffer_in_place(): - buf = torch.full((10,), -1, dtype=torch.int32, device="cuda") - plan = aiter.mla.get_mla_v4_nm_split_plan(7, 128, 1152, split_indptr=buf) +def test_v4_nm_num_kv_splits_uses_cost_model(num_seqs, kv_len): cost = aiter.mla._V4_NM_SPLIT_COST["gfx950"] - assert plan.num_kv_splits == aiter.mla._v4_nm_pick_num_kv_splits( - cost, 7, 2, 1152, get_cu_num() - ) - assert plan.split_indptr.data_ptr() == buf.data_ptr() - assert buf.tolist() == [i * plan.num_kv_splits for i in range(8)] + [-1, -1] + assert aiter.mla.get_mla_v4_nm_num_kv_splits( + num_seqs, 128, kv_len + ) == aiter.mla._v4_nm_pick_num_kv_splits(cost, num_seqs, 2, kv_len, get_cu_num()) @needs_gfx950 @@ -1203,8 +1202,9 @@ def test_v4_nm_split_plan_fills_buffer_in_place(): ], ) def test_v4_nm_split_plan_accuracy(kv_lens, kv_len): - plan = aiter.mla.get_mla_v4_nm_split_plan(len(kv_lens), 128, kv_len) - err, out = _run_varlen_point(kv_lens, split_kwargs=plan._asdict()) + err, out = _run_varlen_point( + kv_lens, split_kwargs=_uniform_split(len(kv_lens), 128, kv_len) + ) assert torch.isfinite(out).all() assert err < 0.02 @@ -1236,13 +1236,12 @@ def test_v4_nm_split_plan_cudagraph_replay(): kv_page_indices = torch.randperm(n * kv_cap, device=device).to(torch.int32) kv_last_page_lens = torch.ones(n, dtype=torch.int32, device=device) output = torch.empty((n, gqa, V_HEAD_DIM), dtype=dtypes.bf16, device=device) - split_buf = torch.empty(n + 1, dtype=torch.int32, device=device) def set_lens(lens): kv_indptr.copy_(torch.tensor([0] + np.cumsum(lens).tolist(), dtype=torch.int32)) - plan = aiter.mla.get_mla_v4_nm_split_plan(n, gqa, kv_cap, split_indptr=split_buf) - assert plan.num_kv_splits > 1 # exercise the stage-2 path under the graph + split = _uniform_split(n, gqa, kv_cap) + assert split["num_kv_splits"] > 1 # exercise the stage-2 path under the graph def run(): aiter.mla.mla_decode_fwd_v4_nm( @@ -1257,7 +1256,7 @@ def run(): 1, sink=sink, sm_scale=sm_scale, - **plan._asdict(), + **split, ) set_lens([kv_cap] * n) @@ -2883,7 +2882,7 @@ def test_v4_nm_ps_matches_split_kernel(ps_workspace, kv_lens): n = len(kv_lens) out = _ps_call(inp, ps_workspace, return_lse=False) ref = torch.empty(n, _PS_HEADS, V_HEAD_DIM, dtype=dtypes.bf16, device="cuda") - plan = aiter.mla.get_mla_v4_nm_split_plan(n, _PS_HEADS, max(kv_lens)) + split = _uniform_split(n, _PS_HEADS, max(kv_lens)) logits, _ = aiter.mla.mla_decode_fwd_v4_nm( inp["q_packed"], inp["q_rope"], @@ -2895,9 +2894,9 @@ def test_v4_nm_ps_matches_split_kernel(ps_workspace, kv_lens): inp["kv_page_indices"], 1, sink=inp["sink"], - **plan._asdict(), + **split, ) - if plan.num_kv_splits == 1: + if split["num_kv_splits"] == 1: ref = logits[:, 0] torch.cuda.synchronize() assert _cos_diff(out, ref) < 1e-5