diff --git a/dnn-providers/hip-kernel-provider/ROCKE_ENGINE_test_categories_external.yaml b/dnn-providers/hip-kernel-provider/ROCKE_ENGINE_test_categories_external.yaml index 3ec35f47fbc3..2b3fbe4c3602 100644 --- a/dnn-providers/hip-kernel-provider/ROCKE_ENGINE_test_categories_external.yaml +++ b/dnn-providers/hip-kernel-provider/ROCKE_ENGINE_test_categories_external.yaml @@ -74,6 +74,7 @@ test_categories: - "rocke_mx_scale_decode" - "rocke_buffer_rsrc_hip" - "rocke_tiled_attention_2d_reentrancy" + - "rocke_tiled_attention_entry_guards" - "rocke_unified_attention_scalar_lower_to_llvm" - "rocke_python_import_smoke" - "rocke_golden_static" @@ -81,6 +82,7 @@ test_categories: - "rocke_differential_golden_canonical" - "rocke_library_conv_pytest" - "rocke_library_attention_cpu_pytest" + - "rocke_library_attention_cache_key_pytest" - "rocke_library_attention_pytest" # Not registered on Windows (see rocke/platform/CMakeLists.txt); the # parser ignores a listed name that has no registered test. @@ -132,6 +134,7 @@ test_categories: - "rocke_mx_scale_decode" - "rocke_buffer_rsrc_hip" - "rocke_tiled_attention_2d_reentrancy" + - "rocke_tiled_attention_entry_guards" - "rocke_unified_attention_scalar_lower_to_llvm" - "rocke_python_import_smoke" - "rocke_golden_static" @@ -139,6 +142,7 @@ test_categories: - "rocke_differential_golden_canonical" - "rocke_library_conv_pytest" - "rocke_library_attention_cpu_pytest" + - "rocke_library_attention_cache_key_pytest" - "rocke_library_attention_pytest" # Not registered on Windows (see rocke/platform/CMakeLists.txt); the # parser ignores a listed name that has no registered test. @@ -159,6 +163,7 @@ test_categories: description: "rocKE generated-artifact reproduction checks" test_patterns: - "rocke_arch_domain_reproducibility" + - "rocke_library_attention_cache_key_sweep_pytest" labels: - "host" - "toolchain" diff --git a/dnn-providers/hip-kernel-provider/descriptor-packaging/python/hkp_pack/rocke_compile.py b/dnn-providers/hip-kernel-provider/descriptor-packaging/python/hkp_pack/rocke_compile.py index f19a0e0aebfc..87cd8fe0e524 100644 --- a/dnn-providers/hip-kernel-provider/descriptor-packaging/python/hkp_pack/rocke_compile.py +++ b/dnn-providers/hip-kernel-provider/descriptor-packaging/python/hkp_pack/rocke_compile.py @@ -323,15 +323,15 @@ def _module_from_source(source): # Builders whose validation predicate is not derivable from the builder name. -# The tiled ``supports_tiled_2d``/``3d`` are keyword-only, so these builders use the -# spec-first ``supports_tiled_*_spec`` wrappers in the gfx942/gfx950 modules. The -# fastkv-register-P tiled builders, the gfx942 reduce builder and all gfx1250 tiled -# builders have no spec-first predicate, so ``_check_support_predicate`` refuses to pack -# them. +# The tiled ``supports_tiled_2d``/``3d`` and ``supports_fastkv_register_p_2d`` are +# keyword-only, so these builders use the spec-first ``supports_*_spec`` wrappers in +# the gfx942/gfx950 modules. The gfx942 reduce builder and all gfx1250 tiled builders +# have no spec-first predicate, so ``_check_support_predicate`` refuses to pack them. _PREDICATE_ALIASES = { "build_unified_attention_2d_tiled": "supports_tiled_2d_spec", "build_gfx942_4warp_gqa": "supports_gfx942_4warp_gqa_spec", "build_unified_attention_3d_tiled": "supports_tiled_3d_spec", + "build_unified_attention_2d_fastkv_register_p": "supports_fastkv_register_p_2d_spec", "build_unified_attention_reduce_tiled": "supports_tiled_reduce_spec", } diff --git a/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/rocke/test_hkp_pack_producer_guards.py b/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/rocke/test_hkp_pack_producer_guards.py index 1720993689cf..44e620d3c5ba 100644 --- a/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/rocke/test_hkp_pack_producer_guards.py +++ b/dnn-providers/hip-kernel-provider/descriptor-packaging/tests/rocke/test_hkp_pack_producer_guards.py @@ -23,6 +23,7 @@ _module_from_source, _require_spec_arch_signature, _resolve_spec_class, + _resolve_support_predicate, build_spec, rocke_compile_arch, ) @@ -386,6 +387,36 @@ def test_every_shipped_rocke_builder_passes_the_pack_time_predicate_check(): pytest.fail(f"{source}:{builder} @ {arch} ({target}): {exc}") +@pytest.mark.skipif(not _HAS_ROCKE_LIBRARY, reason="rocke/library not in this tree") +def test_fastkv_register_p_builder_packs_through_its_spec_predicate(): + """The gfx950 fastKV register-P builder resolves its spec-first predicate by + alias; the predicate accepts the experiment's spec and refuses one the + builder would raise on.""" + module = import_module("kernels.gfx950.attention_tiled_2d_fastkv_regp") + builder = "build_unified_attention_2d_fastkv_register_p" + name, _ = _resolve_support_predicate(module, builder) + assert name == "supports_fastkv_register_p_2d_spec" + spec = module.make_fastkv_register_p_spec( + module.UnifiedAttention2DTiledSpec( + head_size=64, + block_size=32, + num_query_heads=64, + num_kv_heads=8, + dtype="bf16", + use_sinks=False, + sliding_window=0, + has_softcap=False, + ) + ) + _check_support_predicate(module, builder, spec, "gfx950") + no_fast_desc = dataclasses.replace(spec, use_fast_paged_kv_desc=False) + with pytest.raises(HkpPackError, match="use_fast_paged_kv_desc"): + _check_support_predicate(module, builder, no_fast_desc, "gfx950") + # The shared tiled gate admits gfx942 for its narrow builder; this one must not. + with pytest.raises(HkpPackError, match="gfx950 kernel"): + _check_support_predicate(module, builder, spec, "gfx942") + + # --- C. Literal validation -------------------------------------------------- @dataclasses.dataclass class _LiteralSpec: diff --git a/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/benchmark_prefill2d_live.py b/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/benchmark_prefill2d_live.py index 3fd87cdb6f50..f03ef6c5f7c9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/benchmark_prefill2d_live.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/benchmark_prefill2d_live.py @@ -464,9 +464,9 @@ def build( supports_tiled_2d, ) from kernels.common.attention_unified import ( - _attn_signature, _enable_i64_kv_addr, _tiled_2d_impl, + _tiled_2d_signature, ) from rocke.runtime import KernelLauncher @@ -525,12 +525,12 @@ def build( use_i64_kv_addr=flags["use_i64_kv_addr"] or _enable_i64_kv_addr(problem), use_register_pv=flags["use_register_pv"], ) - # kernel_name() does not encode the KV addressing width, so key on it too. + # The name carries i64 addressing; the signature keys the per-shape + # launch values the cached spec feeds. key = ( shape.signature, variant, spec.kernel_name(), - spec.use_i64_kv_addr, self.compile_backend, ) if key not in self._launchers: @@ -540,9 +540,7 @@ def build( KernelLauncher( hsaco=artifact.hsaco, kernel_name=artifact.kernel_name, - signature=_attn_signature( - dtype, include_bt_stride=True, include_qq_bias_stride=True - ), + signature=_tiled_2d_signature(ARCH, dtype), cache_key=("prefill2d_live_gfx942", key), ), spec, diff --git a/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx950/attention/prefill/benchmark_prefill2d_fastkv_regp.py b/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx950/attention/prefill/benchmark_prefill2d_fastkv_regp.py index 4c6e990e648e..646bff265ff5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx950/attention/prefill/benchmark_prefill2d_fastkv_regp.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx950/attention/prefill/benchmark_prefill2d_fastkv_regp.py @@ -8,11 +8,10 @@ ``attention_tiled_2d_fastkv_regp.py`` and optionally joins against a Triton CSV by ``shape_signature``. -The emitted 2D kernel name omits specialization constants such as ``num_seqs`` -and the derived binary-search trip count. This harness appends a short shape -hash to each built ``KernelDef.name`` before compiling, so the ROCm module/HSACO -cache never sees two different shape-specialized code objects with the same -kernel symbol. +Head counts, window and batch are kernargs, so shapes that share a kernel name +share a code object. This harness still appends a short shape hash to each built +``KernelDef.name`` before compiling, so every trace record loads its own module +and the timings stay independent of the order records run in. Example: @@ -308,7 +307,7 @@ def _variant_spec_and_builder( def _launcher(self, shape, problem, variant: str, sliding_window: int): from rocke import compile_kernel from kernels.common.attention_unified import ( - _attn_signature, + _tiled_2d_signature, _enable_i64_kv_addr, ) from rocke.runtime import KernelLauncher @@ -317,28 +316,25 @@ def _launcher(self, shape, problem, variant: str, sliding_window: int): shape, problem, variant, sliding_window ) spec = replace(spec, use_i64_kv_addr=_enable_i64_kv_addr(problem)) - # kernel_name() does not encode the KV addressing width, so key on it too. + # The name carries i64 addressing; the signature keeps one module per + # trace record (see the module docstring). key = ( variant, shape.signature, spec.kernel_name(), - spec.use_i64_kv_addr, self.compile_backend, ) if key not in self._launchers: kernel = builder(spec) - # The display name omits compile-time shape constants. Keep the - # human-readable spec name in CSV, but compile a shape-unique symbol - # to avoid ROCm HSACO/module-cache aliasing across trace records. + # Keep the human-readable spec name in CSV, but compile a + # shape-unique symbol so each trace record loads its own module. kernel.name = _shape_kernel_name(kernel.name, variant, shape.signature) artifact = compile_kernel(kernel, capture_ir_text=False) self._launchers[key] = ( KernelLauncher( hsaco=artifact.hsaco, kernel_name=artifact.kernel_name, - signature=_attn_signature( - spec.dtype, include_bt_stride=True, include_qq_bias_stride=True - ), + signature=_tiled_2d_signature("gfx950", spec.dtype), cache_key=("prefill2d_fastkv_regp", key), ), spec, diff --git a/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx950/attention/prefill/benchmark_prefill2d_live.py b/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx950/attention/prefill/benchmark_prefill2d_live.py index 934731c21c58..5373adc5f35e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx950/attention/prefill/benchmark_prefill2d_live.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx950/attention/prefill/benchmark_prefill2d_live.py @@ -436,7 +436,7 @@ def build( supports_tiled_2d, ) from kernels.common.attention_unified import ( - _attn_signature, + _tiled_2d_signature, _enable_i64_kv_addr, _select_2d_compile_backend, ) @@ -498,12 +498,12 @@ def build( # A cache past the i32 buffer range needs i64 whatever the variant asks. use_i64_kv_addr=flags["use_i64_kv_addr"] or _enable_i64_kv_addr(problem), ) - # kernel_name() does not encode the KV addressing width, so key on it too. + # The name carries i64 addressing; the signature keys the per-shape + # launch values the cached spec feeds. key = ( shape.signature, variant, spec.kernel_name(), - spec.use_i64_kv_addr, self.compile_backend, ) if key not in self._launchers: @@ -521,9 +521,7 @@ def build( KernelLauncher( hsaco=artifact.hsaco, kernel_name=artifact.kernel_name, - signature=_attn_signature( - dtype, include_bt_stride=True, include_qq_bias_stride=True - ), + signature=_tiled_2d_signature("gfx950", dtype), cache_key=("prefill2d_live", key), ), spec, diff --git a/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx950/attention/prefill/benchmark_prefill2d_traces.py b/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx950/attention/prefill/benchmark_prefill2d_traces.py index 3af6de7a0664..dffe20e7b64f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx950/attention/prefill/benchmark_prefill2d_traces.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx950/attention/prefill/benchmark_prefill2d_traces.py @@ -128,7 +128,7 @@ def _launcher(self, shape, problem, sliding_window: int): supports_tiled_2d, ) from kernels.common.attention_unified import ( - _attn_signature, + _tiled_2d_signature, _enable_i64_kv_addr, ) from rocke.runtime import KernelLauncher @@ -181,15 +181,13 @@ def _launcher(self, shape, problem, sliding_window: int): use_fast_paged_kv_desc=use_combo, use_i64_kv_addr=_enable_i64_kv_addr(problem), ) - # The display name intentionally omits several compile-time constants - # (notably num_seqs/binary_search_iters). Cache on the trace signature - # as well so shapes with the same kernel_name never reuse an HSACO that - # was specialized for a different batch geometry. - # kernel_name() does not encode the KV addressing width either. + # The kernel name identifies the binary (head counts, window and batch + # are kernargs; i64 addressing is in the name). Key on the trace + # signature as well because the cached spec also feeds per-shape launch + # values such as the K/V address limit. key = ( shape.signature, spec.kernel_name(), - spec.use_i64_kv_addr, self.compile_backend, ) if key not in self._launchers: @@ -199,9 +197,7 @@ def _launcher(self, shape, problem, sliding_window: int): KernelLauncher( hsaco=artifact.hsaco, kernel_name=artifact.kernel_name, - signature=_attn_signature( - dtype, include_bt_stride=True, include_qq_bias_stride=True - ), + signature=_tiled_2d_signature("gfx950", dtype), cache_key=("prefill2d_trace_combo", key), ), spec, diff --git a/dnn-providers/hip-kernel-provider/rocke/library/builders/common/attention_spec_builder.py b/dnn-providers/hip-kernel-provider/rocke/library/builders/common/attention_spec_builder.py index 95189bce265b..0db81453e664 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/builders/common/attention_spec_builder.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/builders/common/attention_spec_builder.py @@ -21,6 +21,7 @@ _has_score_bias, _enable_combo_2d, _enable_early_v_schedule, + _enable_fast_paged_kv_desc, _enable_fp8_mfma_qk, _enable_gfx942_3d_wide_kv_load, _enable_gfx942_bf16_flash, @@ -240,15 +241,7 @@ def _base_2d_generic_fields(problem: UnifiedAttentionProblem, arch: str) -> dict use_transposed_mask_limit=mask_opts, use_mfma32_skip_legacy_qreg=skip_legacy_qreg, use_early_v_schedule=_enable_early_v_schedule(problem, arch), - use_fast_paged_kv_desc=( - combo_no_sw - and problem.dtype - == "bf16" # fast descriptor is bf16-only (spec validator enforces) - and not problem.use_fp8 - and problem.num_query_heads == 64 - and problem.num_kv_heads == 8 - and _select_2d_tile_size(problem, arch) == 64 - ), + use_fast_paged_kv_desc=_enable_fast_paged_kv_desc(problem, arch), use_register_pv=_enable_register_pv(problem, arch), use_fp8_mfma_qk=_enable_fp8_mfma_qk(problem, arch), use_i64_kv_addr=_enable_i64_kv_addr(problem), diff --git a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/bf16_wide_check.py b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/bf16_wide_check.py index 33ab7d493f21..949562d40ba9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/bf16_wide_check.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/bf16_wide_check.py @@ -39,7 +39,7 @@ from kernels.gfx942.attention_tiled_2d import ( # noqa: E402 UnifiedAttention2DTiledSpec, ) -from kernels.common.attention_unified import _attn_signature # noqa: E402 +from kernels.common.attention_unified import _tiled_2d_signature # noqa: E402 from rocke.runtime import KernelLauncher # noqa: E402 BS = H.BLOCK_SIZE @@ -136,9 +136,7 @@ def build_launcher(s, spec, kind): launcher = KernelLauncher( hsaco=artifact.hsaco, kernel_name=artifact.kernel_name, - signature=_attn_signature( - s.dtype, include_bt_stride=True, include_qq_bias_stride=True - ), + signature=_tiled_2d_signature("gfx942", s.dtype), cache_key=("bf16wide", spec.kernel_name(), kind), ) return launcher diff --git a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/parity_unified_attention.py b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/parity_unified_attention.py index 0c5c3f1eadb0..0f957fda5697 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/parity_unified_attention.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/parity_unified_attention.py @@ -774,7 +774,7 @@ def _build_kernel(s: Shape): """ from rocke import compile_kernel from kernels import build_unified_attention_2d_tiled, supports_tiled_2d - from kernels.common.attention_unified import _attn_signature + from kernels.common.attention_unified import _tiled_2d_signature from rocke.runtime import KernelLauncher spec, config = _build_spec(s) @@ -810,9 +810,7 @@ def _build_kernel(s: Shape): launcher = KernelLauncher( hsaco=artifact.hsaco, kernel_name=artifact.kernel_name, - signature=_attn_signature( - s.dtype, include_bt_stride=True, include_qq_bias_stride=True - ), + signature=_tiled_2d_signature("gfx942", s.dtype), cache_key=("gfx942_attn_example", spec.kernel_name(), config), ) return launcher, spec, config diff --git a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/attention/prefill/parity_unified_attention.py b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/attention/prefill/parity_unified_attention.py index 21de49653ab4..99a63fda2c96 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/attention/prefill/parity_unified_attention.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/attention/prefill/parity_unified_attention.py @@ -1735,7 +1735,7 @@ def _run_rocke( supports_tiled_2d, ) from kernels.common.attention_unified import ( - _attn_signature, + _tiled_2d_signature, _attn_values, _spec_kv_addr_limit, _select_2d_compile_backend, @@ -1775,9 +1775,7 @@ def _run_rocke( launcher = KernelLauncher( hsaco=artifact.hsaco, kernel_name=artifact.kernel_name, - signature=_attn_signature( - dtype_str, include_bt_stride=True, include_qq_bias_stride=True - ), + signature=_tiled_2d_signature(_ARCH, dtype_str), cache_key=("d256_fast_direct", spec.kernel_name()), ) if probe_occupancy: @@ -1848,9 +1846,7 @@ def call_once(): launcher = KernelLauncher( hsaco=artifact.hsaco, kernel_name=artifact.kernel_name, - signature=_attn_signature( - dtype_str, include_bt_stride=True, include_qq_bias_stride=True - ), + signature=_tiled_2d_signature(_ARCH, dtype_str), cache_key=("prod_2d_direct", spec.kernel_name()), ) vals = _attn_values( diff --git a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/attention/candidate.py b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/attention/candidate.py index 3e256ae4fdaf..f4fe538c83a8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/attention/candidate.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/attention/candidate.py @@ -82,16 +82,21 @@ def build(spec, arch): return spec.build(arch) def signature(spec): - from kernels.common.attention_unified import _3d_signature, _attn_signature + from kernels.common.attention_unified import ( + _tiled_2d_signature, + _tiled_3d_signatures, + ) if spec.path == "3d": - return _3d_signature( - spec.kernel_spec.dtype, kv_dtype=spec.kernel_spec.kv_storage_dtype + segment, _ = _tiled_3d_signatures( + spec.arch, + spec.kernel_spec.dtype, + kv_dtype=spec.kernel_spec.kv_storage_dtype, ) - return _attn_signature( + return segment + return _tiled_2d_signature( + spec.arch, spec.kernel_spec.dtype, - include_bt_stride=True, - include_qq_bias_stride=True, kv_dtype=spec.kernel_spec.kv_storage_dtype, emit_lse=bool(getattr(spec.kernel_spec, "emit_lse", False)), ) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/attention/common.py b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/attention/common.py index 1f161e63b15f..4cf1011e88c0 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/attention/common.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/attention/common.py @@ -75,7 +75,9 @@ from rocke.dispatch.tuning.identity import jsonable, normalize_knobs FAMILY = "attention_unified" -ATTENTION_ABI_VERSION = "hipkg-attention-unified/v1" +# v2: the gfx942 and gfx950 tiled kernels take the head counts and the sliding +# window as kernargs. +ATTENTION_ABI_VERSION = "hipkg-attention-unified/v2" _ATTENTION_MASK_ORDINALS = tuple(mask_type.value for mask_type in AttentionMaskType) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/attention/strided_decode.py b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/attention/strided_decode.py index b3b5ce78251f..c8809c876172 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/attention/strided_decode.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/attention/strided_decode.py @@ -15,9 +15,9 @@ from kernels.common.attention_unified import ( UnifiedAttentionProblem, - _3d_signature, _explicit_path_supported, _strided_3d_specs_from_problem, + _tiled_3d_signatures, _validate_strided_3d_spec, ) from rocke.dispatch.core import Capability, KernelCandidate, ShapeRange, TorchBinding @@ -218,7 +218,8 @@ def launch(*, softmax_scale=softmax_scale, stream=stream): family=FAMILY, algorithm="strided_decode", spec_id="strided_decode", - abi_version="rocke-attention-strided-kv/v1", + # v2: the head counts and the sliding window became kernargs. + abi_version="rocke-attention-strided-kv/v2", priority=20, capability=Capability( arches=("gfx942", "gfx950"), @@ -242,7 +243,9 @@ def launch(*, softmax_scale=softmax_scale, stream=stream): select_spec=select, sweep_space=lambda req: (select(req),) if candidate.admits(req)[0] else (), build=lambda spec, arch: spec.build(arch), - signature=lambda spec: _3d_signature(spec.kernel_spec.dtype, strided_kv=True), + signature=lambda spec: _tiled_3d_signatures( + spec.arch, spec.kernel_spec.dtype, strided_kv=True + )[0], grid=lambda spec, req: spec.launch_grid(_problem(req)), block=lambda spec: spec.launch_block(), bind_torch=bind, diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/_fmha_common.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/_fmha_common.py index 4dcf33643bf6..70fa500cab53 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/_fmha_common.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/_fmha_common.py @@ -53,6 +53,33 @@ FmhaMaskMode = Literal["none", "causal", "sliding_window", "alibi", "custom"] +def check_head_counts(num_query_heads: int, num_kv_heads: int) -> Tuple[bool, str]: + """Admit a positive head pair whose query heads split evenly over the KV heads. + + The one head-count rule every attention spec validator runs first: the + kernels index heads by the GQA ratio ``num_query_heads // num_kv_heads``. + """ + if num_query_heads <= 0 or num_kv_heads <= 0: + return False, ( + f"num_query_heads / num_kv_heads must be > 0 " + f"(got {num_query_heads}, {num_kv_heads})" + ) + if num_query_heads % num_kv_heads != 0: + return False, ( + f"num_query_heads ({num_query_heads}) must be divisible by " + f"num_kv_heads ({num_kv_heads}) for GQA / MQA" + ) + return True, "" + + +def gqa_ratio(num_query_heads: int, num_kv_heads: int) -> int: + """``num_query_heads // num_kv_heads``; ``ValueError`` if :func:`check_head_counts` rejects the pair.""" + ok, reason = check_head_counts(num_query_heads, num_kv_heads) + if not ok: + raise ValueError(reason) + return num_query_heads // num_kv_heads + + @dataclass(frozen=True) class FmhaShape: """Shape bundle used by every FMHA flavour.""" @@ -65,12 +92,7 @@ class FmhaShape: @property def num_queries_per_kv(self) -> int: - if self.num_query_heads % self.num_kv_heads: - raise ValueError( - f"num_query_heads ({self.num_query_heads}) must be divisible by " - f"num_kv_heads ({self.num_kv_heads})" - ) - return self.num_query_heads // self.num_kv_heads + return gqa_ratio(self.num_query_heads, self.num_kv_heads) @dataclass(frozen=True) @@ -123,16 +145,9 @@ def validate_common_spec(spec: FmhaCommonSpec) -> Tuple[bool, str]: f"head_size {s.head_size} not in the supported set " "{32, 64, 128, 192, 256} (CK Tile's standard FMHA shapes)" ) - if s.num_query_heads <= 0 or s.num_kv_heads <= 0: - return False, ( - f"num_query_heads / num_kv_heads must be > 0 " - f"(got {s.num_query_heads}, {s.num_kv_heads})" - ) - if s.num_query_heads % s.num_kv_heads != 0: - return False, ( - f"num_query_heads ({s.num_query_heads}) must be divisible by " - f"num_kv_heads ({s.num_kv_heads}) for GQA / MQA" - ) + ok, reason = check_head_counts(s.num_query_heads, s.num_kv_heads) + if not ok: + return False, reason if s.block_size_q not in (16, 32, 64, 128): return False, f"block_size_q {s.block_size_q} not in {{16, 32, 64, 128}}" if s.block_size_k not in (16, 32, 64, 128, 256): diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_arch.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_arch.py index 394e67bcd6a8..28aa51aaa4c0 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_arch.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_arch.py @@ -37,6 +37,7 @@ from __future__ import annotations +import functools from typing import Tuple from rocke.core.arch import ArchTarget @@ -83,6 +84,9 @@ def _narrow_k_mfma_available(target: ArchTarget) -> bool: ) +# A function of the arch string alone (the catalog is static), and every tiled +# support gate calls it, so it is computed once per arch. +@functools.lru_cache(maxsize=None) def validate_tiled_attention_arch(arch: str) -> Tuple[bool, str]: """Return ``(ok, reason)`` for running a tiled attention kernel on ``arch``. diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_dense_kv_split.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_dense_kv_split.py index 8da79a427125..b1e6f2360416 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_dense_kv_split.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_dense_kv_split.py @@ -188,29 +188,49 @@ def combine_spec(spec, arch: str): ) +def combine_launch_abi(spec, arch: str): + """The combine's reduce spec, launch signature and runtime scalar values. + + The reduce reads its runtime shape fields (``num_query_heads``) as kernargs, + so the signature carries them after the pointers and the values supply them. + Host only: no torch and no compile. + """ + from kernels.common.attention_unified import ( + _reduce_signature, + _runtime_kernarg_fields, + ) + + red = combine_spec(spec, arch) + spec_type, _ = _combine_impl(arch) + runtime_fields = _runtime_kernarg_fields(spec_type) + signature = _reduce_signature(spec.dtype, runtime_fields=runtime_fields) + scalars = {name: int(getattr(red, name)) for name in runtime_fields} + return red, signature, scalars + + def launch_segment_combine(spec, arch: str, out, workspace, stream: int) -> None: """Merge the segments in ``workspace`` into ``out`` ([B, Sq, Hq, D]).""" - from kernels.common.attention_unified import _reduce_signature from rocke.helpers.compile import compile_kernel from rocke.runtime import KernelLauncher, LaunchConfig - red = combine_spec(spec, arch) + red, signature, scalars = combine_launch_abi(spec, arch) + _, build = _combine_impl(arch) key = (arch, red) launcher = _COMBINE_LAUNCHER_CACHE.get(key) if launcher is None: - _, build = _combine_impl(arch) art = compile_kernel( build(red, arch=arch), arch=arch, backend="python", capture_ir_text=False ) launcher = KernelLauncher( hsaco=art.hsaco, kernel_name=art.kernel_name, - signature=_reduce_signature(spec.dtype), + signature=signature, ) _COMBINE_LAUNCHER_CACHE[key] = launcher import torch vals = dict(workspace) + vals.update(scalars) vals["output_ptr"] = out # Unused by the combine body; any int32 buffer satisfies the ABI. vals["seq_lens_ptr"] = workspace["segm_max_ptr"].view(torch.int32) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_unified.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_unified.py index 4542d7b7b5fa..88d845002607 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_unified.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/common/attention_unified.py @@ -13,7 +13,7 @@ from collections import OrderedDict from dataclasses import dataclass, replace from threading import Lock, RLock -from typing import Any, Callable, Dict, Optional, Tuple +from typing import Any, Callable, Dict, Optional, Sequence, Tuple from rocke.core.arch import validate_arch from rocke.core.ir import ( @@ -1692,10 +1692,11 @@ def _kv_storage_dtype(problem: UnifiedAttentionProblem) -> Optional[str]: def _binary_search_iters(problem: UnifiedAttentionProblem, *, floor: int = 1) -> int: """Trip count of the seq_idx binary search the tiled kernels bake into IR. - Every tiled spec class derives ``binary_search_iters`` from ``spec.num_seqs`` - through the same ``binary_search_seq_idx_iters`` (gfx1250 3D with a floor of - 32). It is the only way ``num_seqs`` reaches emitted IR, so the tiled cache - keys carry this bucket instead of the raw batch count. + Every tiled spec class that bakes the search derives ``binary_search_iters`` + from ``spec.num_seqs`` through the same ``binary_search_seq_idx_iters`` + (gfx1250 3D with a floor of 32). It is the only way ``num_seqs`` reaches + emitted IR, so the tiled cache keys carry this bucket instead of the raw batch + count. A body that searches at runtime (``_searches_at_runtime``) bakes neither. """ return binary_search_seq_idx_iters(problem.num_seqs, floor=floor) @@ -1738,7 +1739,8 @@ def _tiled_cache_key(problem: UnifiedAttentionProblem, arch: str) -> Tuple: ``max_seqlen_k``). A raw field compiles one binary per value; a selector missing from the key lets two shapes that build different kernels share one cache slot. ``num_seqs`` reaches IR only through - ``_binary_search_iters``. Launch values that scale with the shape (the + ``_binary_search_iters``, and not at all in a body that searches at runtime + (``_searches_at_runtime``). Launch values that scale with the shape (the grid) are keyed separately through ``_shape_key``. When adding a selector to ``_tiled_spec_from_problem``, add its resolved output here; ``test_attention_tiled_cache_key.py`` sweeps the selector thresholds and @@ -1751,16 +1753,32 @@ def _tiled_cache_key(problem: UnifiedAttentionProblem, arch: str) -> Tuple: num_warps_2d = None if fp16_flash else _select_2d_num_warps(problem, arch) tile_size = _select_2d_tile_size(problem, arch) block_m_per_warp = _select_2d_block_m_per_warp(problem, arch) + # gfx950 schedule tail of `_spec_gfx950_generic`. Both flags are false + # outside the single-batch cohort (num_seqs == 1), so skip them there. Keyed + # as None when both are off, because the cohort itself does not change the + # kernel and a runtime batch search must not split ns=1 from ns>1 on it. + single_batch_sched = None + if _enable_single_batch_combo(problem, arch): + flags = ( + _enable_softmax_mfma_interleave(problem, arch), + _enable_k_single_buffer( + problem, + arch, + num_warps=num_warps_2d, + block_m_per_warp=block_m_per_warp, + tile_size=tile_size, + ), + ) + single_batch_sched = flags if any(flags) else None key = ( "tiled", arch, - _binary_search_iters(problem), - problem.num_query_heads, - problem.num_kv_heads, + None if _searches_at_runtime(arch, "2d") else _binary_search_iters(problem), + _heads_key(problem, arch, "2d"), problem.head_size, problem.block_size, problem.dtype, - problem.sliding_window, + _window_key(problem, arch, "2d"), bool(problem.use_sinks), bool(problem.softcap > 0), bool(problem.use_alibi), @@ -1796,24 +1814,12 @@ def _tiled_cache_key(problem: UnifiedAttentionProblem, arch: str) -> Tuple: # The builder derives the scalar-state / mask / skip-qreg / fast-paged-KV # spec fields from the combo predicate, which reads max_seqlen_q. _enable_combo_2d(problem, arch), - # gfx950 schedule tail of `_spec_gfx950_generic`. Both are false - # outside the single-batch cohort (num_seqs == 1), so gate on it and - # spare every other launch the selector walks. sched_barrier is implied - # by _enable_v_double_buffer, which is already in the key above. - ( - ( - _enable_softmax_mfma_interleave(problem, arch), - _enable_k_single_buffer( - problem, - arch, - num_warps=num_warps_2d, - block_m_per_warp=block_m_per_warp, - tile_size=tile_size, - ), - ) - if _enable_single_batch_combo(problem, arch) - else None - ), + # Bakes the 64/8 head geometry, which the heads entry above does not + # carry when the counts are kernargs. + _enable_fast_paged_kv_desc(problem, arch), + # sched_barrier is implied by _enable_v_double_buffer, which is already + # in the key above. + single_batch_sched, # Reads max_seqlen_q and max_seqlen_k (the 16 * T threshold). The # gfx1250 builder never sets this field, so keying it there would only # split identical kernels. @@ -2949,6 +2955,25 @@ def _enable_transposed_subflags(problem: UnifiedAttentionProblem, arch: str) -> return _enable_transposed_qk_32x32(problem, arch) +def _enable_fast_paged_kv_desc(problem: UnifiedAttentionProblem, arch: str) -> bool: + """The fast multi-block paged-KV descriptor of the no-window bf16 2D combo. + + Its loader bakes the h64kv8 bf16 T=64 geometry (the spec validator pins it), + so it is on only for exactly that head pair. The head counts are kernargs + and the cache key carries only their ratio, so the key carries this flag + too: a 32/4 problem at the same ratio must not reuse a 64/8 fast binary. + """ + return ( + _enable_combo_2d(problem, arch) + and problem.sliding_window == 0 + and problem.dtype == "bf16" + and not problem.use_fp8 + and problem.num_query_heads == 64 + and problem.num_kv_heads == 8 + and _select_2d_tile_size(problem, arch) == 64 + ) + + # Largest paged KV cache, in bytes, that i32 addressing reads correctly on the # tiled builders. They bound K/V buffer loads at num_records=0x7FFF0000, so a # load at or past that offset returns zero. @@ -4203,9 +4228,10 @@ def _tiled_3d_cache_key(problem: UnifiedAttentionProblem, arch: str) -> Tuple: with the shape are keyed separately through ``_shape_key``. """ decode_grid = _use_decode_grid(problem, arch) - if decode_grid: - # One CTA per sequence: the segment kernel indexes sequences directly - # and emits no search, so batch sizes in different buckets share it. + if decode_grid or _searches_at_runtime(arch, "3d"): + # The decode grid indexes sequences directly and emits no search; a + # runtime search reads the batch from num_seqs. Either way batch sizes + # in different buckets share the kernel. search_iters = None elif arch == "gfx1250": search_iters = _binary_search_iters(problem, floor=BINARY_SEARCH_FALLBACK_ITERS) @@ -4219,12 +4245,11 @@ def _tiled_3d_cache_key(problem: UnifiedAttentionProblem, arch: str) -> Tuple: # no way to check the key against the arch it compiles for. arch, search_iters, - problem.num_query_heads, - problem.num_kv_heads, + _heads_key(problem, arch, "3d"), problem.head_size, problem.block_size, problem.dtype, - problem.sliding_window, + _window_key(problem, arch, "3d"), bool(problem.use_sinks), bool(problem.softcap > 0), bool(problem.use_alibi), @@ -4262,12 +4287,115 @@ def _tiled_3d_cache_key(problem: UnifiedAttentionProblem, arch: str) -> Tuple: return base +def _runtime_kernarg_fields(spec_cls) -> Tuple[str, ...]: + """The problem fields a tiled spec class reads as i32 kernargs, in ABI order. + + The spec class declares them (``RUNTIME_KERNARG_FIELDS``) so its body's + params, the launch signature and the cache key read one tuple. A class + declaring none bakes those fields into its IR. + """ + return tuple(getattr(spec_cls, "RUNTIME_KERNARG_FIELDS", ())) + + +_TILED_SPEC_CLS: Dict[Tuple[str, str], type] = {} + + +def _tiled_spec_cls(arch: str, path: str) -> type: + """The tiled ``path`` ("2d" or "3d" segment) spec class for ``arch``. Memoized: + the cache keys read its class attributes on every launch.""" + key = (arch, path) + if key not in _TILED_SPEC_CLS: + impl = _tiled_2d_impl(arch) if path == "2d" else _tiled_3d_impl(arch) + _TILED_SPEC_CLS[key] = impl[0] + return _TILED_SPEC_CLS[key] + + +def _searches_at_runtime(arch: str, path: str) -> bool: + """True when the tiled ``path`` body for ``arch`` derives its cu_q search trip + count from the num_seqs kernarg (``RUNTIME_SEARCH_ITERS``), so num_seqs never + reaches its IR.""" + return bool(getattr(_tiled_spec_cls(arch, path), "RUNTIME_SEARCH_ITERS", False)) + + +def _heads_key(problem: UnifiedAttentionProblem, arch: str, path: str) -> Tuple: + """The head counts as the tiled ``path`` body for ``arch`` bakes them: both + counts, or only the GQA ratio when the counts are kernargs.""" + fields = _runtime_kernarg_fields(_tiled_spec_cls(arch, path)) + if "num_query_heads" in fields and "num_kv_heads" in fields: + return ("gqa", problem.num_queries_per_kv) + return (problem.num_query_heads, problem.num_kv_heads) + + +def _window_key(problem: UnifiedAttentionProblem, arch: str, path: str): + """The sliding window as the tiled ``path`` body for ``arch`` bakes it: the size + itself, or only whether there is one when the size is a kernarg.""" + if "sliding_window" in _runtime_kernarg_fields(_tiled_spec_cls(arch, path)): + return problem.sliding_window > 0 + return problem.sliding_window + + +def _runtime_kernarg_values(problem: UnifiedAttentionProblem) -> dict: + """Every runtime shape kernarg a tiled body can declare. The launcher packs + only the ones its signature names.""" + return { + "num_query_heads": int(problem.num_query_heads), + "num_kv_heads": int(problem.num_kv_heads), + "sliding_window": int(problem.sliding_window), + } + + +def _tiled_2d_signature( + arch: str, + dtype: str, + *, + kv_dtype: Optional[str] = None, + include_additive_bias: bool = False, + emit_lse: bool = False, +): + """Launch signature of the tiled 2D kernels built for ``arch``.""" + spec_cls, _, _ = _tiled_2d_impl(arch) + return _attn_signature( + dtype, + include_bt_stride=True, + include_qq_bias_stride=True, + include_additive_bias=include_additive_bias, + kv_dtype=kv_dtype, + runtime_fields=_runtime_kernarg_fields(spec_cls), + emit_lse=emit_lse, + ) + + +def _tiled_3d_signatures( + arch: str, + dtype: str, + *, + kv_dtype: Optional[str] = None, + strided_kv: bool = False, + include_additive_bias: bool = False, + emit_lse: bool = False, +): + """Launch signatures of the tiled 3D segment and reduce kernels for ``arch``.""" + segment_cls, reduce_cls, _, _, _ = _tiled_3d_impl(arch) + segment = _3d_signature( + dtype, + kv_dtype=kv_dtype, + strided_kv=strided_kv, + include_additive_bias=include_additive_bias, + runtime_fields=_runtime_kernarg_fields(segment_cls), + ) + reduce = _reduce_signature( + dtype, emit_lse=emit_lse, runtime_fields=_runtime_kernarg_fields(reduce_cls) + ) + return segment, reduce + + def _3d_signature( dtype: str, *, kv_dtype: Optional[str] = None, strided_kv: bool = False, include_additive_bias: bool = False, + runtime_fields: Sequence[str] = (), ): from rocke.helpers.spec import SignatureBuilder @@ -4295,6 +4423,8 @@ def _3d_signature( .scalar("block_table_stride", "i32") .scalar("qq_bias_stride_0", "i32") ) + for name in runtime_fields: + sb.scalar(name, "i32") # Conditional params, in the kernel's declaration order (bias, then strided KV). if include_additive_bias: append_additive_bias_signature(sb) @@ -4307,7 +4437,9 @@ def _3d_signature( return sb.build() -def _reduce_signature(dtype: str, *, emit_lse: bool = False): +def _reduce_signature( + dtype: str, *, emit_lse: bool = False, runtime_fields: Sequence[str] = () +): from rocke.helpers.spec import SignatureBuilder io_dtype = "f16" if dtype == "fp16" else "bf16" @@ -4319,6 +4451,8 @@ def _reduce_signature(dtype: str, *, emit_lse: bool = False): .ptr("segm_expsum_ptr", "f32") .ptr("seq_lens_ptr", "i32") ) + for name in runtime_fields: + sig = sig.scalar(name, "i32") if emit_lse: sig = sig.ptr("lse_ptr", "f32") return sig.build() @@ -4331,6 +4465,7 @@ def _attn_signature( include_qq_bias_stride: bool = False, include_additive_bias: bool = False, kv_dtype: Optional[str] = None, + runtime_fields: Sequence[str] = (), emit_lse: bool = False, ): from rocke.helpers.spec import SignatureBuilder @@ -4363,6 +4498,8 @@ def _attn_signature( sb.scalar("block_table_stride", "i32") if include_qq_bias_stride: sb.scalar("qq_bias_stride_0", "i32") + for name in runtime_fields: + sb.scalar(name, "i32") # Appended last, matching the kernel's conditional param declarations: # additive bias, then the LSE output (the tiled 2D builders declare # lse_ptr after declare_additive_bias_params). @@ -4428,6 +4565,7 @@ def _attn_values( "out_scale": float(out_scale), "softcap": float(softcap), "num_seqs": int(problem.num_seqs), + **_runtime_kernarg_values(problem), } if include_bt_stride: vals["block_table_stride"] = int(bt_stride) @@ -4776,6 +4914,7 @@ def _run_3d_tiled_on_stream( "additive_bias_batch_stride": int(additive_bias_batch_stride), "additive_bias_head_stride": int(additive_bias_head_stride), "additive_bias_sq_stride": int(additive_bias_sq_stride), + **_runtime_kernarg_values(problem), } if kv_cache_layout is not None: seg_vals.update(kv_cache_layout.arguments()) @@ -4785,6 +4924,7 @@ def _run_3d_tiled_on_stream( "segm_max_ptr": segm_max, "segm_expsum_ptr": segm_expsum, "seq_lens_ptr": seqused_k, + **_runtime_kernarg_values(problem), } if problem.emit_lse: red_vals["lse_ptr"] = lse @@ -5470,15 +5610,54 @@ def attention_3d_workspace_nbytes( def _shape_key(cache_key: tuple, problem: UnifiedAttentionProblem) -> tuple: """Extend a binary cache key with the shape values the launch derives from. - The tiled binaries are shared across batch counts and token counts, but the - grids (``total_q // block_q + num_seqs``), the 3D reduce grid and the 3D - workspace are not. Everything cached per launch geometry keys on this. + The tiled binaries are shared across batch counts, token counts and, where + the body reads them as kernargs, head counts, but the grids (``total_q // + block_q + num_seqs`` by the kv or query heads), the 3D reduce grid and the + 3D workspace are not. Everything cached per launch geometry keys on this. """ return cache_key + ( "total_q", int(problem.total_q), "num_seqs", int(problem.num_seqs), + "heads", + int(problem.num_query_heads), + int(problem.num_kv_heads), + ) + + +def _tiled_3d_kernels( + problem: UnifiedAttentionProblem, + arch: str, + num_segments: int, + *, + strided_kv: bool = False, +): + """The tiled 3D segment and reduce ``KernelDef``s the production path builds.""" + ( + _, + UnifiedAttentionReduceTiledSpec, + build_unified_attention_3d_tiled, + build_unified_attention_reduce_tiled, + _, + ) = _tiled_3d_impl(arch) + if strided_kv: + segment_spec, reduce_spec = _strided_3d_specs_from_problem(problem, arch=arch) + else: + segment_spec = _tiled_3d_spec_from_problem(problem, arch) + reduce_spec = UnifiedAttentionReduceTiledSpec( + head_size=problem.head_size, + num_query_heads=problem.num_query_heads, + num_kv_heads=problem.num_kv_heads, + dtype=problem.dtype, + num_segments=num_segments, + waves_per_eu=_select_3d_waves_per_eu(problem, arch), + # Only when on: gfx1250's reduce spec has no such field. + **({"emit_lse": True} if problem.emit_lse else {}), + ) + return ( + build_unified_attention_3d_tiled(segment_spec, arch=arch), + build_unified_attention_reduce_tiled(reduce_spec, arch=arch), ) @@ -5506,31 +5685,9 @@ def _get_3d_pipeline( if tuning_spec is not None: seg_kernel, red_kernel = tuning_spec.build(arch) else: - ( - _, - UnifiedAttentionReduceTiledSpec, - build_unified_attention_3d_tiled, - build_unified_attention_reduce_tiled, - _, - ) = _tiled_3d_impl(arch) - if strided_kv: - segment_spec, reduce_spec = _strided_3d_specs_from_problem( - problem, arch=arch - ) - else: - segment_spec = _tiled_3d_spec_from_problem(problem, arch) - reduce_spec = UnifiedAttentionReduceTiledSpec( - head_size=problem.head_size, - num_query_heads=problem.num_query_heads, - num_kv_heads=problem.num_kv_heads, - dtype=problem.dtype, - num_segments=num_segments, - waves_per_eu=_select_3d_waves_per_eu(problem, arch), - # Only when on: gfx1250's reduce spec has no such field. - **({"emit_lse": True} if problem.emit_lse else {}), - ) - seg_kernel = build_unified_attention_3d_tiled(segment_spec, arch=arch) - red_kernel = build_unified_attention_reduce_tiled(reduce_spec, arch=arch) + seg_kernel, red_kernel = _tiled_3d_kernels( + problem, arch, num_segments, strided_kv=strided_kv + ) seg_art = compile_kernel(seg_kernel, arch=arch, capture_ir_text=False) red_art = compile_kernel(red_kernel, arch=arch, capture_ir_text=False) _ATTN_3D_TILED_CACHE[cache_key] = ( @@ -5540,28 +5697,31 @@ def _get_3d_pipeline( red_art.kernel_name, ) seg_hsaco, seg_kname, red_hsaco, red_kname = _ATTN_3D_TILED_CACHE[cache_key] + # problem.emit_lse matches the built reduce: an explicit tuning spec must + # agree with it (paged: run_unified_attention_torch; strided: + # _validate_strided_3d_spec refuses an LSE reduce). + seg_signature, red_signature = _tiled_3d_signatures( + arch, + problem.dtype, + strided_kv=strided_kv, + kv_dtype=( + tuning_spec.kernel_spec.kv_storage_dtype + if tuning_spec is not None + else _kv_storage_dtype(problem) + ), + include_additive_bias=problem.use_additive_bias, + emit_lse=problem.emit_lse, + ) seg_launcher = KernelLauncher( hsaco=seg_hsaco, kernel_name=seg_kname, - signature=_3d_signature( - problem.dtype, - strided_kv=strided_kv, - kv_dtype=( - tuning_spec.kernel_spec.kv_storage_dtype - if tuning_spec is not None - else _kv_storage_dtype(problem) - ), - include_additive_bias=problem.use_additive_bias, - ), + signature=seg_signature, cache_key=("3d_seg",) + cache_key, ) red_launcher = KernelLauncher( hsaco=red_hsaco, kernel_name=red_kname, - # problem.emit_lse matches the built reduce: an explicit tuning spec - # must agree with it (paged: run_unified_attention_torch; strided: - # _validate_strided_3d_spec refuses an LSE reduce). - signature=_reduce_signature(problem.dtype, emit_lse=problem.emit_lse), + signature=red_signature, cache_key=("3d_red",) + cache_key, ) pipeline = PipelineLauncher([seg_launcher, red_launcher]) @@ -5687,6 +5847,20 @@ def _select_2d_compile_backend(problem: UnifiedAttentionProblem, arch: str) -> s return "llvm" +def _tiled_2d_kernel(problem: UnifiedAttentionProblem, arch: str): + """The tiled 2D ``KernelDef`` the production path builds for ``problem``.""" + _, build_unified_attention_2d_tiled, _ = _tiled_2d_impl(arch) + spec = _tiled_spec_from_problem(problem, arch) + route = _gfx942_4warp_route(problem, arch) + if route is not None: + # Distinct 4-warp GQA paged builder (keyed separately in + # `_tiled_cache_key`; grid in `_get_2d_launch_meta`). Same paged + # ABI as the default builder. Parity+ with AITER @Sq4096/8192 + # (vs the 1-warp std-QK's 0.55x). + return route.builder(spec, arch=arch) + return build_unified_attention_2d_tiled(spec, arch=arch) + + def _get_2d_launcher( problem: UnifiedAttentionProblem, cache_key: Tuple, @@ -5709,17 +5883,7 @@ def _get_2d_launcher( kernel = tuning_spec.build(arch) backend = tuning_spec.compile_backend else: - _, build_unified_attention_2d_tiled, _ = _tiled_2d_impl(arch) - spec = _tiled_spec_from_problem(problem, arch) - route = _gfx942_4warp_route(problem, arch) - if route is not None: - # Distinct 4-warp GQA paged builder (keyed separately in - # `_tiled_cache_key`; grid in `_get_2d_launch_meta`). Same paged - # ABI as the default builder. Parity+ with AITER @Sq4096/8192 - # (vs the 1-warp std-QK's 0.55x). - kernel = route.builder(spec, arch=arch) - else: - kernel = build_unified_attention_2d_tiled(spec, arch=arch) + kernel = _tiled_2d_kernel(problem, arch) backend = _select_2d_compile_backend(problem, arch) if backend == "hipcc": from rocke.helpers.compile import compile_kernel_via_hipcc @@ -5732,10 +5896,9 @@ def _get_2d_launcher( launcher = KernelLauncher( hsaco=hsaco, kernel_name=kname, - signature=_attn_signature( + signature=_tiled_2d_signature( + arch, problem.dtype, - include_bt_stride=True, - include_qq_bias_stride=True, include_additive_bias=problem.use_additive_bias, kv_dtype=( tuning_spec.kernel_spec.kv_storage_dtype diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_tiled_2d.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_tiled_2d.py index 5bc5e2605f9c..ebc560c85fff 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_tiled_2d.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_tiled_2d.py @@ -36,8 +36,9 @@ from dataclasses import dataclass, field import os -from typing import Optional, Tuple +from typing import ClassVar, Optional, Tuple +from kernels.common._fmha_common import check_head_counts, gqa_ratio from kernels.common._lse_store import lse_row_has_mass, lse_store from rocke.core.ir import ( @@ -60,7 +61,7 @@ from rocke.helpers.attention import ( apply_softcap_log2, binary_search_seq_idx, - binary_search_seq_idx_iters, + binary_search_seq_idx_trip_count, dequant_fp8x8_to_dtype, mfma_16x16x16_for_dtype, mfma_16x16x32_for_dtype, @@ -75,7 +76,13 @@ from rocke.helpers.distribution import make_static_tile_distribution from rocke.helpers.layouts import TransposeLdsReader from rocke.helpers.mfma_gemm_inner import decode_mfma_lanes -from rocke.helpers.transforms import TensorDescriptor, embed, indirect, unmerge +from rocke.helpers.transforms import ( + DynamicTensorDescriptor, + TensorDescriptor, + embed, + indirect, + unmerge, +) MFMA_M = 16 @@ -163,6 +170,18 @@ def _sched_group_pin_mfma_step(b, ds_reads: int, mfma_count: int, sgid: int) -> @dataclass(frozen=True) class UnifiedAttention2DTiledSpec: + # Problem fields this body reads as trailing i32 kernargs instead of baking + # them, in ABI order. The launch signature, the argument pack and the cache + # key read the same tuple (``_runtime_kernarg_fields``). + RUNTIME_KERNARG_FIELDS: ClassVar[Tuple[str, ...]] = ( + "num_query_heads", + "num_kv_heads", + "sliding_window", + ) + # The cu_q search derives its trip count from the num_seqs kernarg, so + # num_seqs never reaches the IR and one binary serves every batch. + RUNTIME_SEARCH_ITERS: ClassVar[bool] = True + head_size: int block_size: int num_query_heads: int @@ -928,7 +947,7 @@ def __post_init__(self): @property def num_queries_per_kv(self) -> int: - return self.num_query_heads // self.num_kv_heads + return gqa_ratio(self.num_query_heads, self.num_kv_heads) @property def block_m(self) -> int: @@ -970,15 +989,6 @@ def n_blocks_per_tile(self) -> int: def dtype_ir(self) -> Type: return F16 if self.dtype == "fp16" else BF16 - @property - def binary_search_iters(self) -> int: - # AITER/Triton uses a true while-loop binary search. Our IR currently - # lowers this as a fixed-trip scf.for, so specialize the trip count to - # the known problem batch size instead of always paying 32 iterations. - # Falls back to 32 for direct unit-test specs that do not provide - # `num_seqs`. - return binary_search_seq_idx_iters(self.num_seqs) - def kernel_name(self) -> str: from rocke.helpers.spec import kernel_name_join @@ -991,12 +1001,22 @@ def kernel_name(self) -> str: f"d{self.head_size}", f"b{self.block_size}", f"t{self.tile_size_eff}" if self.n_blocks_per_tile != 1 else "", - f"h{self.num_query_heads}kv{self.num_kv_heads}", + # Head counts are kernargs; only the GQA ratio is baked. + f"g{self.num_queries_per_kv}", + # Except on the fast paged-KV descriptor path, whose shifts bake them. + ( + f"h{self.num_query_heads}kv{self.num_kv_heads}" + if self.use_fast_paged_kv_desc + else "" + ), self.dtype, f"kv{self.kv_storage_dtype}" if self.kv_storage_dtype else "", "fnuz" if self.kv_storage_dtype else "", + # i64 K/V addressing changes the loaders, so it is a distinct kernel. + "i64" if self.use_i64_kv_addr else "", "" if not self.use_sinks else "sinks", - f"sw{self.sliding_window}" if self.sliding_window > 0 else "", + # The window size is a kernarg; only whether the body masks one is baked. + "sw" if self.sliding_window > 0 else "", "softcap" if self.has_softcap else "", "alibi" if self.use_alibi else "", "qqb" if self.use_qq_bias else "", @@ -1388,7 +1408,6 @@ def build_unified_attention_2d_tiled( # rows would duplicate the first tokens of the NEXT CTA and must be masked. VALID_ROWS = BLOCK_Q * NQK NUM_KV = spec.num_kv_heads - NUM_QH = spec.num_query_heads SLIDING_WINDOW = spec.sliding_window USE_SOFTCAP = spec.has_softcap USE_SINKS = spec.use_sinks @@ -1580,6 +1599,11 @@ def build_unified_attention_2d_tiled( num_seqs_p = b.param("num_seqs", I32) bt_stride_p = b.param("block_table_stride", I32) qq_bias_stride0_p = b.param("qq_bias_stride_0", I32) + # Shape values this body reads as kernargs (``RUNTIME_KERNARG_FIELDS``), + # after every fixed param so the shared launch signature can append them. + rt = {f: b.param(f, I32) for f in spec.RUNTIME_KERNARG_FIELDS} + num_qh_p = rt["num_query_heads"] + num_kv_p = rt["num_kv_heads"] # Appended LAST and only when requested, so the LSE-off ABI is unchanged. lse = ( b.param( @@ -1615,7 +1639,7 @@ def build_unified_attention_2d_tiled( q_block_global_idx, num_seqs_p, block_q=BLOCK_Q, - iterations=spec.binary_search_iters, + iterations=binary_search_seq_idx_trip_count(b, num_seqs_p), ) cu_q_start = b.global_load_i32(cu_q, seq_idx) cu_q_stop = b.global_load_i32(cu_q, b.add(seq_idx, b.const_i32(1))) @@ -2162,7 +2186,7 @@ def _v_load1(v_buf: Value, v_row: Value, v_n_col: Value) -> Value: # stays the same as the default bf16 path -- just softmax_scale / # ln(2) -- and we do NOT fold k_scale into qk_scale here. pv_fp8_scale = b.fdiv(v_scale_p, b.const_f32(240.0)) if FP8_MFMA_PV else None - sw_const = b.const_i32(int(SLIDING_WINDOW)) + sw_const = rt["sliding_window"] z8 = b.zero_vec(dtype, 8) # ---------------- Q -> LDS (cooperative vec8 chunks) ---------------- @@ -2175,15 +2199,11 @@ def _v_load1(v_buf: Value, v_row: Value, v_n_col: Value) -> Value: # Coordinate transform for Q (and the symmetric output buffer): # ``(token, head, dim)`` packed contiguously. The element-unit # descriptor is reused below for the output store too. - q_desc = TensorDescriptor.naive( + # The head count is a kernarg, so the token stride is too. + q_desc = DynamicTensorDescriptor.create( "Q", - # The runtime extents (total_q, num_query_heads, head_size) are - # only used by the validity predicate (which we don't request - # here). Use generous compile-time bounds so the descriptor's - # row-major stride product matches the kernel's layout - # assumptions exactly. - lengths=[1 << 30, NUM_QH, HD], coord_names=("token", "head", "dim"), + strides=(b.mul(num_qh_p, b.const_i32(HD)), HD, 1), ) if not Q_DIRECT_GLOBAL: for li in range(Q_VECS_PER_THREAD): @@ -2196,7 +2216,7 @@ def _v_load1(v_buf: Value, v_row: Value, v_n_col: Value) -> Value: ) _qmask_inner = b.land( b.cmp_lt(q_pos_t, cur_batch_q_len), - b.cmp_lt(qh_t, b.const_i32(NUM_QH)), + b.cmp_lt(qh_t, num_qh_p), ) qmask_t = ( b.land(_qmask_inner, b.cmp_lt(Q_row, b.const_i32(VALID_ROWS))) @@ -2378,7 +2398,7 @@ def _pack_p_a32(p_regs0_f32: list[Value], p_regs1_f32: list[Value]) -> Value: qh = b.add( b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(row, b.const_i32(NQK)) ) - qh_in = b.cmp_lt(qh, b.const_i32(NUM_QH)) + qh_in = b.cmp_lt(qh, num_qh_p) sink_h = b.global_load(sinks, qh, dtype, align=2) sink_f = b.fmul(b.cast_to_f32(sink_h), rcp_ln2) m_inits.append(b.select(qh_in, sink_f, neg_inf)) @@ -2388,7 +2408,7 @@ def _pack_p_a32(p_regs0_f32: list[Value], p_regs1_f32: list[Value]) -> Value: qh = b.add( b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(row, b.const_i32(NQK)) ) - qh_in = b.cmp_lt(qh, b.const_i32(NUM_QH)) + qh_in = b.cmp_lt(qh, num_qh_p) sink_h = b.global_load(sinks, qh, dtype, align=2) sink_f = b.fmul(b.cast_to_f32(sink_h), rcp_ln2) m_inits.append(b.select(qh_in, sink_f, neg_inf)) @@ -2482,10 +2502,11 @@ def _acc_idx(n: int, atom: int) -> int: # so for the FP8 K/V path the loader switches to a sync per-thread # dequant chain below; the byte-stride math here is shared so the # paged_kv_desc compiles consistently in both branches. - kv_stride_blk_b = BS * NUM_KV * HD * KV_BYTES - kv_stride_tok_b = NUM_KV * HD * KV_BYTES + # The token and block strides scale with the runtime kv head count. + kv_stride_tok_b = b.mul(num_kv_p, b.const_i32(HD * KV_BYTES)) + kv_stride_blk_b = b.mul(kv_stride_tok_b, b.const_i32(BS)) kv_stride_h_b = HD * KV_BYTES - kv_block_bytes_c = b.const_i32(kv_stride_blk_b) # one-block buffer bound + kv_block_bytes_c = kv_stride_blk_b # one-block buffer bound # Per-lane starting half index. With the gfx942 1-dword DMA each lane # contributes ``KV_HALVES_PER_LANE`` (= 2) contiguous halves per call, @@ -2655,13 +2676,10 @@ def _fast_paged_kv_voff(call: int, block0: Value, block1: Value): return None, b.add(block_b, within) else: - _kv_base = TensorDescriptor.naive( + _kv_base = DynamicTensorDescriptor.create( "paged_kv_bytes", - # ``lengths`` here is just informational (validity propagation - # is driven by the transforms above, not by these bounds). - lengths=[1 << 24, BS, NUM_KV, HD], - strides=[kv_stride_blk_b, kv_stride_tok_b, kv_stride_h_b, KV_BYTES], coord_names=("physical_block", "token", "kv_head", "dim"), + strides=(kv_stride_blk_b, kv_stride_tok_b, kv_stride_h_b, KV_BYTES), ) if N_BLOCKS_PER_TILE == 1: paged_kv_desc = _kv_base.transform( @@ -2730,7 +2748,7 @@ def _tile_pages( align=4, ) base = b.mul( - b.zext(b.to_sgpr_u32(physical), I64), b.const_i64(kv_stride_blk_b) + b.zext(b.to_sgpr_u32(physical), I64), b.zext(kv_stride_blk_b, I64) ) ptrs.append(b.global_ptr_add(src, base)) rsrcs = [b.buffer_rsrc(ptr, kv_block_bytes_c) for ptr in ptrs] @@ -2746,7 +2764,7 @@ def _page_voff(call: int, kv_head_b: Value) -> Value: token = b.div(half, b.const_i32(HD)) dim = b.mod(half, b.const_i32(HD)) return b.add( - b.add(b.mul(token, b.const_i32(kv_stride_tok_b)), kv_head_b), + b.add(b.mul(token, kv_stride_tok_b), kv_head_b), b.mul(dim, b.const_i32(KV_BYTES)), ) @@ -3862,7 +3880,7 @@ def _read_k8_mfma_operand( ) q32_mask = b.land( b.cmp_lt(q32_pos, cur_batch_q_len), - b.cmp_lt(q32_h, b.const_i32(NUM_QH)), + b.cmp_lt(q32_h, num_qh_p), ) q32_pos_safe = b.select(q32_mask, q32_pos, b.const_i32(0)) q32_h_safe = b.select(q32_mask, q32_h, b.const_i32(0)) @@ -3967,7 +3985,7 @@ def _read_k8_mfma_operand( qh_r = b.add(b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(row, b.const_i32(NQK))) _row_ok_inner = b.land( b.cmp_lt(qp_r, cur_batch_q_len), - b.cmp_lt(qh_r, b.const_i32(NUM_QH)), + b.cmp_lt(qh_r, num_qh_p), ) row_ok = ( b.land(_row_ok_inner, b.cmp_lt(row, b.const_i32(VALID_ROWS))) @@ -3985,7 +4003,7 @@ def _read_k8_mfma_operand( hoist_alibi = [] for reg in range(REGS_PER_LANE): qh_r = hoist_qh_r[reg] - qh_ok = b.cmp_lt(qh_r, b.const_i32(NUM_QH)) + qh_ok = b.cmp_lt(qh_r, num_qh_p) slope = b.masked_global_load( alibi_slopes_ptr, qh_r, qh_ok, b.const_f32(0.0), dtype=F32, align=4 ) @@ -4001,7 +4019,7 @@ def _read_k8_mfma_operand( ) _st_row_ok_inner = b.land( b.cmp_lt(st_qp, cur_batch_q_len), - b.cmp_lt(st_qh, b.const_i32(NUM_QH)), + b.cmp_lt(st_qh, num_qh_p), ) st_row_ok = ( b.land(_st_row_ok_inner, b.cmp_lt(st_q_row, b.const_i32(VALID_ROWS))) @@ -4028,7 +4046,7 @@ def _read_k8_mfma_operand( and TRANSPOSED_QK_32X32 and TRANSPOSED_INVARIANT_HOIST ): - st_qh_ok = b.cmp_lt(st_qh, b.const_i32(NUM_QH)) + st_qh_ok = b.cmp_lt(st_qh, num_qh_p) st_alibi_slope = b.masked_global_load( alibi_slopes_ptr, st_qh, st_qh_ok, b.const_f32(0.0), dtype=F32, align=4 ) @@ -4090,7 +4108,7 @@ def _acc_get(n: int, atom: int) -> Value: ) _st_row_ok_iter_inner = b.land( b.cmp_lt(st_qp_iter, cur_batch_q_len), - b.cmp_lt(st_qh_iter, b.const_i32(NUM_QH)), + b.cmp_lt(st_qh_iter, num_qh_p), ) st_row_ok_iter = ( b.land( @@ -4102,7 +4120,7 @@ def _acc_get(n: int, atom: int) -> Value: ) st_causal_lim_iter = b.add(context_len, st_qp_iter) if USE_ALIBI: - st_qh_iter_ok = b.cmp_lt(st_qh_iter, b.const_i32(NUM_QH)) + st_qh_iter_ok = b.cmp_lt(st_qh_iter, num_qh_p) st_alibi_slope_iter = b.masked_global_load( alibi_slopes_ptr, st_qh_iter, @@ -4425,7 +4443,7 @@ def _kslot(group_idx: int) -> int: slope_for_lane = b.masked_global_load( alibi_slopes_ptr, qh_t_alibi, - b.cmp_lt(qh_t_alibi, b.const_i32(NUM_QH)), + b.cmp_lt(qh_t_alibi, num_qh_p), b.const_f32(0.0), dtype=F32, align=4, @@ -4480,7 +4498,7 @@ def _kslot(group_idx: int) -> int: ) _row_ok_inner_t = b.land( b.cmp_lt(qp_r, cur_batch_q_len), - b.cmp_lt(qh_r, b.const_i32(NUM_QH)), + b.cmp_lt(qh_r, num_qh_p), ) row_ok = ( b.land( @@ -5556,14 +5574,14 @@ def _store_row_lse(row: Value, slot: int, writer: Value) -> None: """ pos = b.add(qb_start_pos, b.div(row, b.const_i32(NQK))) qh = b.add(b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(row, b.const_i32(NQK))) - ok = b.land(b.cmp_lt(pos, cur_batch_q_len), b.cmp_lt(qh, b.const_i32(NUM_QH))) + ok = b.land(b.cmp_lt(pos, cur_batch_q_len), b.cmp_lt(qh, num_qh_p)) if VALID_ROWS < BLOCK_M: ok = b.land(ok, b.cmp_lt(row, b.const_i32(VALID_ROWS))) with b.scf_if(b.land(writer, ok)): lse_store( b, lse, - b.add(b.mul(b.add(cu_q_start, pos), b.const_i32(NUM_QH)), qh), + b.add(b.mul(b.add(cu_q_start, pos), num_qh_p), qh), m_final[slot], l_final[slot], row_has_mass=lse_row_has_mass(b, l_final[slot]), @@ -5588,7 +5606,7 @@ def _store_row_lse(row: Value, slot: int, writer: Value) -> None: ) _op_mask_t_inner = b.land( b.cmp_lt(op_pos_t, cur_batch_q_len), - b.cmp_lt(op_qh_t, b.const_i32(NUM_QH)), + b.cmp_lt(op_qh_t, num_qh_p), ) op_mask_t = ( b.land(_op_mask_t_inner, b.cmp_lt(q_row_t, b.const_i32(VALID_ROWS))) @@ -5615,7 +5633,7 @@ def _store_row_lse(row: Value, slot: int, writer: Value) -> None: b, lse, b.add( - b.mul(b.add(cu_q_start, op_pos_t), b.const_i32(NUM_QH)), + b.mul(b.add(cu_q_start, op_pos_t), num_qh_p), op_qh_t, ), m_final[0], @@ -5660,7 +5678,7 @@ def _store_row_lse(row: Value, slot: int, writer: Value) -> None: ) _op_mask32_inner = b.land( b.cmp_lt(op_pos32_base, cur_batch_q_len), - b.cmp_lt(op_qh32_base, b.const_i32(NUM_QH)), + b.cmp_lt(op_qh32_base, num_qh_p), ) op_mask32_base = ( b.land(_op_mask32_inner, b.cmp_lt(OUT_ROW_BASE32, b.const_i32(VALID_ROWS))) @@ -5770,7 +5788,7 @@ def _store_row_lse(row: Value, slot: int, writer: Value) -> None: ) _op_mask_inner = b.land( b.cmp_lt(op_pos, cur_batch_q_len), - b.cmp_lt(op_qh, b.const_i32(NUM_QH)), + b.cmp_lt(op_qh, num_qh_p), ) op_mask = ( b.land(_op_mask_inner, b.cmp_lt(OUT_ROW_BASE, b.const_i32(VALID_ROWS))) @@ -5921,8 +5939,6 @@ def _build_gfx942_4warp_gqa_lean( raise NotImplementedError( "4-warp GQA lean body is causal-only (no sliding-window mask)" ) - H = spec.num_query_heads - HKV = spec.num_kv_heads GQAG = spec.num_queries_per_kv BS = spec.block_size BN = 64 # 4-warp: 64-key tiles @@ -5934,7 +5950,6 @@ def _build_gfx942_4warp_gqa_lean( NKpv = BN // K # V-fill: elements per thread (256 = 4 wave64 threads), 8-wide loads. v_fill_epw = (BN * HD) // 256 - ITERS = spec.binary_search_iters b = IRBuilder(spec.kernel_name() + "_4wgqa") b.kernel.attrs["max_workgroup_size"] = 256 # 4 wave64 warps @@ -5973,6 +5988,14 @@ def _build_gfx942_4warp_gqa_lean( b.param("softcap", F32) num_seqs_p = b.param("num_seqs", I32) bt_stride_p = b.param("block_table_stride", I32) + # Unused here; declared so every later param sits where the shared 2D + # launch signature packs it. + b.param("qq_bias_stride_0", I32) + # Shape values this body reads as kernargs (``RUNTIME_KERNARG_FIELDS``), + # after every fixed param so the shared launch signature can append them. + rt = {f: b.param(f, I32) for f in spec.RUNTIME_KERNARG_FIELDS} + hq_p = rt["num_query_heads"] + hkv_p = rt["num_kv_heads"] tid = b.thread_id_x() wid = b.div(tid, b.const_i32(64)) @@ -5984,7 +6007,14 @@ def _build_gfx942_4warp_gqa_lean( gqb = b.block_id_y() # grid.y = total_q_blocks (BLOCK_M=128) # in-kernel CTA->seq via binary_search on cu_seqlens_q (mirrors :5590) - sid = binary_search_seq_idx(b, CUQ, gqb, num_seqs_p, block_q=128, iterations=ITERS) + sid = binary_search_seq_idx( + b, + CUQ, + gqb, + num_seqs_p, + block_q=128, + iterations=binary_search_seq_idx_trip_count(b, num_seqs_p), + ) cu_q_start = b.global_load_i32(CUQ, sid) cu_q_stop = b.global_load_i32(CUQ, b.add(sid, b.const_i32(1))) qlen = b.sub(cu_q_stop, cu_q_start) @@ -5999,8 +6029,10 @@ def _build_gfx942_4warp_gqa_lean( b.ret() V_lds = b.smem_alloc(dtype, [BN, HD], name_hint="Vlds") - q_desc = TensorDescriptor.naive( - "query_ptr", lengths=[1 << 30, H, HD], coord_names=("token", "head", "dim") + q_desc = DynamicTensorDescriptor.create( + "query_ptr", + coord_names=("token", "head", "dim"), + strides=(b.mul(hq_p, b.const_i32(HD)), HD, 1), ) # Every K/V key is bounded by the last valid key and every Q row by the last @@ -6047,9 +6079,7 @@ def bperm(v): key = b.div(lin, b.const_i32(HD)) hd = b.mod(lin, b.const_i32(HD)) pk = phys_key(kv, key) - velem = b.add( - b.mul(b.add(b.mul(pk, b.const_i32(HKV)), kvh), b.const_i32(HD)), hd - ) + velem = b.add(b.mul(b.add(b.mul(pk, hkv_p), kvh), b.const_i32(HD)), hd) b.smem_store_vN( V_lds, [key, hd], b.global_load_vN(Vp, velem, dtype, 8, align=16), 8 ) @@ -6069,9 +6099,7 @@ def bperm(v): q = b.global_load_vN(Q, q_off, dtype, BPL, align=BPL * 2) for kt in range(NKEYT): kelem = b.add( - b.mul( - b.add(b.mul(pk_kt[kt], b.const_i32(HKV)), kvh), b.const_i32(HD) - ), + b.mul(b.add(b.mul(pk_kt[kt], hkv_p), kvh), b.const_i32(HD)), koff, ) kf = b.global_load_vN(Kp, kelem, dtype, APL, align=APL * 2) @@ -6153,7 +6181,7 @@ def bperm(v): r, c = at.lane_to_output(b, lane, i) dim = b.add(b.mul(b.const_i32(nt), b.const_i32(32)), r) q_inseq = b.add(qbase, b.add(wq, c)) - oi = b.add(b.mul(b.add(qstart, b.add(wq, c)), b.const_i32(H)), qhead) + oi = b.add(b.mul(b.add(qstart, b.add(wq, c)), hq_p), qhead) val = b.cast_f32_to(b.fmul(b.vec_extract(accs_f[nt], i), recip), dtype) with b.scf_if(b.cmp_lt(q_inseq, qlen)): b.global_store(C, b.add(b.mul(oi, b.const_i32(HD)), dim), val, align=2) @@ -6193,8 +6221,6 @@ def build_gfx942_4warp_gqa( ) if HD in _4WGQA_LEAN_HEAD_SIZES: return _build_gfx942_4warp_gqa_lean(spec, arch=arch) - H = spec.num_query_heads - HKV = spec.num_kv_heads GQAG = spec.num_queries_per_kv BS = spec.block_size if BS & (BS - 1): @@ -6224,7 +6250,6 @@ def build_gfx942_4warp_gqa( BN * HD ) // 256 # V-fill: elements per thread (256 = 4 wave64 threads) v_fill_nloads = v_fill_epw // 8 # 8-wide (dwordx4) loads per thread - ITERS = spec.binary_search_iters from ..common.attention_unified import gfx942_gqa_fold_eligible # GQA head-fold is the DEFAULT for the qualifying cohort (D128, GQA 4:1, sliding- @@ -6288,6 +6313,14 @@ def build_gfx942_4warp_gqa( b.param("softcap", F32) num_seqs_p = b.param("num_seqs", I32) bt_stride_p = b.param("block_table_stride", I32) + # Unused here; declared so every later param sits where the shared 2D + # launch signature packs it. + b.param("qq_bias_stride_0", I32) + # Shape values this body reads as kernargs (``RUNTIME_KERNARG_FIELDS``), + # after every fixed param so the shared launch signature can append them. + rt = {f: b.param(f, I32) for f in spec.RUNTIME_KERNARG_FIELDS} + hq_p = rt["num_query_heads"] + hkv_p = rt["num_kv_heads"] tid = b.thread_id_x() wid = b.div(tid, b.const_i32(64)) @@ -6306,7 +6339,12 @@ def build_gfx942_4warp_gqa( # in-kernel CTA->seq via binary_search on cu_seqlens_q (mirrors :5590) sid = binary_search_seq_idx( - b, CUQ, gqb, num_seqs_p, block_q=TOKBLK, iterations=ITERS + b, + CUQ, + gqb, + num_seqs_p, + block_q=TOKBLK, + iterations=binary_search_seq_idx_trip_count(b, num_seqs_p), ) cu_q_start = b.global_load_i32(CUQ, sid) cu_q_stop = b.global_load_i32(CUQ, b.add(sid, b.const_i32(1))) @@ -6352,8 +6390,10 @@ def build_gfx942_4warp_gqa( f"(V_lds {_v_lds_bytes} + K_lds {_k_lds_bytes}, HD={HD}, " f"HD128_PIPE={HD128_PIPE}) exceeds the {arch} {_lds_cap} B LDS capacity" ) - q_desc = TensorDescriptor.naive( - "query_ptr", lengths=[1 << 30, H, HD], coord_names=("token", "head", "dim") + q_desc = DynamicTensorDescriptor.create( + "query_ptr", + coord_names=("token", "head", "dim"), + strides=(b.mul(hq_p, b.const_i32(HD)), HD, 1), ) # Every Q row is bounded by the last token of its sequence and every K/V page by @@ -6403,6 +6443,7 @@ def bperm(v): ] context_off = b.sub(klen, qlen) # prefix in KV cache (qlen!=klen: chunked/decode) window = int(spec.sliding_window) # 0 = causal; >0 = SWA (keep dist < window) + window_p = rt["sliding_window"] # the size itself is a kernarg causal_t = b.div( b.add(b.add(context_off, qbase), b.const_i32(TOKBLK + BN - 1)), b.const_i32(BN) ) @@ -6410,7 +6451,7 @@ def bperm(v): kvend = b.select(b.cmp_lt(causal_t, klen_t), causal_t, klen_t) # E6: windowed KV-loop start - skip tiles fully before the window (bounded work) if window > 0: - _ks = b.sub(b.add(context_off, qbase), b.const_i32(window)) + _ks = b.sub(b.add(context_off, qbase), window_p) _ks = b.select(b.cmp_gt(_ks, b.const_i32(0)), _ks, b.const_i32(0)) kvstart = b.div(_ks, b.const_i32(BN)) else: @@ -6432,7 +6473,7 @@ def bperm(v): int_end = b.select(b.cmp_lt(int_end_c, int_end_r), int_end_c, int_end_r) if window > 0: _sn = b.sub( - b.add(q_blk_hi, b.const_i32(1)), b.const_i32(window) + b.add(q_blk_hi, b.const_i32(1)), window_p ) # window-interior: kv*BN>=q_blk_hi-window+1 _cs = b.div(b.add(_sn, b.const_i32(BN - 1)), b.const_i32(BN)) # ceil int_start = b.select(b.cmp_lt(_sn, b.const_i32(0)), kvstart, _cs) @@ -6491,9 +6532,7 @@ def fill_tile(tkv, buf_off): key = b.div(lin, b.const_i32(HD)) hd = b.mod(lin, b.const_i32(HD)) pk = phys_key(tkv, key) - velem = b.add( - b.mul(b.add(b.mul(pk, b.const_i32(HKV)), kvh), b.const_i32(HD)), hd - ) + velem = b.add(b.mul(b.add(b.mul(pk, hkv_p), kvh), b.const_i32(HD)), hd) row = b.add(buf_off, key) b.smem_store_vN( V_lds, @@ -6518,9 +6557,7 @@ def fill_load( key = b.div(lin, b.const_i32(HD)) hd = b.mod(lin, b.const_i32(HD)) pk = phys_key(tkv, key) - velem = b.add( - b.mul(b.add(b.mul(pk, b.const_i32(HKV)), kvh), b.const_i32(HD)), hd - ) + velem = b.add(b.mul(b.add(b.mul(pk, hkv_p), kvh), b.const_i32(HD)), hd) vr = b.global_load_vN(Vp, velem, dtype, 8, align=16) kr = b.global_load_vN(Kp, velem, dtype, 8, align=16) pf.append((vr, kr, key, hd)) @@ -6579,7 +6616,7 @@ def body(kv, carry, masked): else: kelem = b.add( b.mul( - b.add(b.mul(pk_kt[kt], b.const_i32(HKV)), kvh), + b.add(b.mul(pk_kt[kt], hkv_p), kvh), b.const_i32(HD), ), koff, @@ -6610,7 +6647,7 @@ def body(kv, carry, masked): mask_cond = b.lor(b.cmp_gt(key_g, q_g), b.cmp_ge(key_g, klen)) if window > 0: mask_cond = b.lor( - mask_cond, b.cmp_ge(b.sub(q_g, key_g), b.const_i32(window)) + mask_cond, b.cmp_ge(b.sub(q_g, key_g), window_p) ) Sm[kt][i] = b.select(mask_cond, ninf, raw) else: @@ -6734,12 +6771,12 @@ def run_loop(start, end, init_iters, masked, iv): _hl = b.mod(_m, b.const_i32(FOLD_HEADS)) q_inseq = b.add(qbase, _tl) oi = b.add( - b.mul(b.add(qstart, _tl), b.const_i32(H)), + b.mul(b.add(qstart, _tl), hq_p), b.add(b.mul(kvh, b.const_i32(GQAG)), _hl), ) else: q_inseq = b.add(qbase, b.add(wq, c)) - oi = b.add(b.mul(b.add(qstart, b.add(wq, c)), b.const_i32(H)), qhead) + oi = b.add(b.mul(b.add(qstart, b.add(wq, c)), hq_p), qhead) val = b.cast_f32_to(b.fmul(b.vec_extract(accs_f[nt], i), recip), dtype) with b.scf_if(b.cmp_lt(q_inseq, qlen)): b.global_store(C, b.add(b.mul(oi, b.const_i32(HD)), dim), val, align=2) @@ -6775,6 +6812,9 @@ def supports_gfx942_4warp_gqa_spec( def _supports_tiled_2d_spec( spec: UnifiedAttention2DTiledSpec, *, arch: str, use_d256_fast: bool ) -> Tuple[bool, str]: + ok, reason = check_head_counts(spec.num_query_heads, spec.num_kv_heads) + if not ok: + return False, reason use_fp8 = spec.kv_storage_dtype is not None return supports_tiled_2d( head_size=spec.head_size, diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_tiled_3d.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_tiled_3d.py index 3fabc7e3f761..9a905742f54d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_tiled_3d.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_tiled_3d.py @@ -23,7 +23,8 @@ gfx942 2D kernel uses (gfx942-legal), via ``dequant_fp8x8_to_dtype``. The reduce kernel is pure f32 load / exp2 / store (no MFMA, no transpose read), -so it is arch-neutral and is a byte-for-byte port of the gfx950 reduce kernel. +so it is arch-neutral. It is a port of the gfx950 reduce kernel that reads the +query head count as a kernarg instead of baking it. See the gfx950 module docstring for the split-KV grid / workspace layout. """ @@ -31,8 +32,9 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Optional, Tuple +from typing import ClassVar, Optional, Tuple +from kernels.common._fmha_common import check_head_counts, gqa_ratio from kernels.common._attention_strided_kv import ( declare_strided_kv_params, strided_kv_offset, @@ -60,14 +62,19 @@ from rocke.helpers.attention import ( apply_softcap_log2 as _apply_softcap, binary_search_seq_idx as _binary_search_seq_idx_helper, - binary_search_seq_idx_iters, + binary_search_seq_idx_trip_count, dequant_fp8x8_to_dtype, mfma_16x16x16_for_dtype as _mfma_16x16x16, wave64_reduce_max as _wave64_reduce_max, wave64_reduce_sum as _wave64_reduce_sum, ) from rocke.helpers.distribution import make_static_tile_distribution -from rocke.helpers.transforms import TensorDescriptor, embed, indirect, unmerge +from rocke.helpers.transforms import ( + DynamicTensorDescriptor, + embed, + indirect, + unmerge, +) MFMA_M = 16 MFMA_N = 16 @@ -103,6 +110,18 @@ class UnifiedAttention3DTiledSpec: dispatcher and harnesses can construct it identically. """ + # Problem fields this body reads as trailing i32 kernargs instead of baking + # them, in ABI order. The launch signature, the argument pack and the cache + # key read the same tuple (``_runtime_kernarg_fields``). + RUNTIME_KERNARG_FIELDS: ClassVar[Tuple[str, ...]] = ( + "num_query_heads", + "num_kv_heads", + "sliding_window", + ) + # The cu_q search derives its trip count from the num_seqs kernarg, so + # num_seqs never reaches the IR and one binary serves every batch. + RUNTIME_SEARCH_ITERS: ClassVar[bool] = True + head_size: int block_size: int num_query_heads: int @@ -217,7 +236,7 @@ def __post_init__(self): @property def num_queries_per_kv(self) -> int: - return self.num_query_heads // self.num_kv_heads + return gqa_ratio(self.num_query_heads, self.num_kv_heads) @property def block_m(self) -> int: @@ -239,10 +258,6 @@ def tile_size(self) -> int: def dtype_ir(self) -> Type: return F16 if self.dtype == "fp16" else BF16 - @property - def binary_search_iters(self) -> int: - return binary_search_seq_idx_iters(self.num_seqs) - def kernel_name(self) -> str: from rocke.helpers.spec import kernel_name_join @@ -250,13 +265,15 @@ def kernel_name(self) -> str: "rocke_uattn3d_tiled_gfx942", f"d{self.head_size}", f"b{self.block_size}", - f"h{self.num_query_heads}kv{self.num_kv_heads}", + # Head counts are kernargs; only the GQA ratio is baked. + f"g{self.num_queries_per_kv}", f"seg{self.num_segments}", self.dtype, f"kv{self.kv_storage_dtype}" if self.kv_storage_dtype else "", "fnuz" if self.kv_storage_dtype else "", "sinks" if self.use_sinks else "", - f"sw{self.sliding_window}" if self.sliding_window > 0 else "", + # The window size is a kernarg; only whether the body masks one is baked. + "sw" if self.sliding_window > 0 else "", "softcap" if self.has_softcap else "", "alibi" if self.use_alibi else "", "qqb" if self.use_qq_bias else "", @@ -374,8 +391,6 @@ def build_unified_attention_3d_tiled( # Rows past BLOCK_Q * NQK are q-block padding when NQK does not divide # BLOCK_M; they must not store, as they alias the next q-block's rows. VALID_ROWS = spec.block_q * NQK - NUM_KV = spec.num_kv_heads - NUM_QH = spec.num_query_heads NUM_SEG = spec.num_segments SLIDING_WINDOW = spec.sliding_window USE_SOFTCAP = spec.has_softcap @@ -459,6 +474,11 @@ def build_unified_attention_3d_tiled( num_seqs_p = b.param("num_seqs", I32) bt_stride_p = b.param("block_table_stride", I32) qq_bias_stride0_p = b.param("qq_bias_stride_0", I32) + # Shape values this body reads as kernargs (``RUNTIME_KERNARG_FIELDS``), + # after every fixed param so the shared launch signature can append them. + rt = {f: b.param(f, I32) for f in spec.RUNTIME_KERNARG_FIELDS} + num_qh_p = rt["num_query_heads"] + num_kv_p = rt["num_kv_heads"] if spec.kv_layout == "strided": k_strides = declare_strided_kv_params(b, "k") v_strides = declare_strided_kv_params(b, "v") @@ -478,7 +498,7 @@ def build_unified_attention_3d_tiled( q_block_global_idx, num_seqs_p, block_q=BLOCK_Q, - iterations=spec.binary_search_iters, + iterations=binary_search_seq_idx_trip_count(b, num_seqs_p), ) cu_q_start = b.global_load_i32(cu_q, seq_idx) cu_q_stop = b.global_load_i32(cu_q, b.add(seq_idx, b.const_i32(1))) @@ -498,20 +518,21 @@ def build_unified_attention_3d_tiled( # tiles_per_segment = cdiv(seq_len, NUM_SEG * T) tps = b.div(b.add(seq_len, b.const_i32(NUM_SEG * T - 1)), b.const_i32(NUM_SEG * T)) - ml_desc = TensorDescriptor.naive( + # The head count is a kernarg, so each token stride is too. + ml_desc = DynamicTensorDescriptor.create( "segm_ml", - lengths=[1 << 30, NUM_QH, NUM_SEG], coord_names=("token", "head", "seg"), + strides=(b.mul(num_qh_p, b.const_i32(NUM_SEG)), NUM_SEG, 1), ) - seg_acc_desc = TensorDescriptor.naive( + seg_acc_desc = DynamicTensorDescriptor.create( "segm_output", - lengths=[1 << 30, NUM_QH, NUM_SEG, HD], coord_names=("token", "head", "seg", "dim"), + strides=(b.mul(num_qh_p, b.const_i32(NUM_SEG * HD)), NUM_SEG * HD, HD, 1), ) - q_desc = TensorDescriptor.naive( + q_desc = DynamicTensorDescriptor.create( "Q", - lengths=[1 << 30, NUM_QH, HD], coord_names=("token", "head", "dim"), + strides=(b.mul(num_qh_p, b.const_i32(HD)), HD, 1), ) seg_start_tile_pos = b.mul(b.mul(seg_idx, tps), b.const_i32(T)) @@ -525,9 +546,7 @@ def build_unified_attention_3d_tiled( qh_r = b.add( b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(row, b.const_i32(NQK)) ) - row_ok = b.land( - b.cmp_lt(qp_r, cur_batch_q_len), b.cmp_lt(qh_r, b.const_i32(NUM_QH)) - ) + row_ok = b.land(b.cmp_lt(qp_r, cur_batch_q_len), b.cmp_lt(qh_r, num_qh_p)) if VALID_ROWS < spec.block_m: row_ok = b.land(row_ok, b.cmp_lt(row, b.const_i32(VALID_ROWS))) qp_r_safe = b.select(row_ok, qp_r, b.const_i32(0)) @@ -547,7 +566,7 @@ def build_unified_attention_3d_tiled( b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(row, b.const_i32(NQK)) ) row_ok = b.land( - b.cmp_lt(qp_r, cur_batch_q_len), b.cmp_lt(qh_r, b.const_i32(NUM_QH)) + b.cmp_lt(qp_r, cur_batch_q_len), b.cmp_lt(qh_r, num_qh_p) ) if VALID_ROWS < spec.block_m: row_ok = b.land(row_ok, b.cmp_lt(row, b.const_i32(VALID_ROWS))) @@ -593,7 +612,7 @@ def _row_reduce(x: Value, op) -> Value: one_f = b.const_f32(1.0) rcp_ln2 = b.const_f32(1.4426950408889634) qk_scale = b.fmul(scale_p, rcp_ln2) - sw_const = b.const_i32(int(SLIDING_WINDOW)) + sw_const = rt["sliding_window"] z8 = b.zero_vec(dtype, 8) # ---------------- Q -> LDS ---------------- @@ -608,9 +627,7 @@ def _row_reduce(x: Value, op) -> Value: b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(Q_row, b.const_i32(NQK)), ) - qmask_t = b.land( - b.cmp_lt(q_pos_t, cur_batch_q_len), b.cmp_lt(qh_t, b.const_i32(NUM_QH)) - ) + qmask_t = b.land(b.cmp_lt(q_pos_t, cur_batch_q_len), b.cmp_lt(qh_t, num_qh_p)) q_pos_safe = b.select(qmask_t, q_pos_t, b.const_i32(0)) qh_safe = b.select(qmask_t, qh_t, b.const_i32(0)) q_off_base, _ = q_desc.offset( @@ -652,7 +669,7 @@ def _row_reduce(x: Value, op) -> Value: qh = b.add( b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(lane_col, b.const_i32(NQK)) ) - qh_in = b.cmp_lt(qh, b.const_i32(NUM_QH)) + qh_in = b.cmp_lt(qh, num_qh_p) sink_h = b.global_load(sinks, qh, dtype, align=2) sink_f = b.fmul(b.cast_to_f32(sink_h), rcp_ln2) sink_with_mask = b.select(qh_in, sink_f, neg_inf) @@ -691,8 +708,9 @@ def _row_reduce(x: Value, op) -> Value: assert (T * HD) % KV_HALVES_PER_CALL == 0 kv_calls_per_tile = (T * HD) // KV_HALVES_PER_CALL bytes_per_call = KV_HALVES_PER_CALL * 2 - kv_stride_blk_b = BS * NUM_KV * HD * KV_BYTES - kv_stride_tok_b = NUM_KV * HD * KV_BYTES + # The token and page strides scale with the runtime kv head count. + kv_stride_tok_b = b.mul(num_kv_p, b.const_i32(HD * KV_BYTES)) + kv_stride_blk_b = b.mul(kv_stride_tok_b, b.const_i32(BS)) kv_stride_h_b = HD * KV_BYTES bytes_per_buf = T * HD * 2 @@ -702,11 +720,10 @@ def _row_reduce(x: Value, op) -> Value: zero_soff = b.const_i32(0) seq_base = b.mul(seq_idx, bt_stride_p) if spec.kv_layout == "paged" else None - _kv_base = TensorDescriptor.naive( + _kv_base = DynamicTensorDescriptor.create( "paged_kv_bytes", - lengths=[1 << 24, BS, NUM_KV, HD], - strides=[kv_stride_blk_b, kv_stride_tok_b, kv_stride_h_b, KV_BYTES], coord_names=("physical_block", "token", "kv_head", "dim"), + strides=(kv_stride_blk_b, kv_stride_tok_b, kv_stride_h_b, KV_BYTES), ) if T == BS: paged_kv_desc = _kv_base.transform( @@ -766,7 +783,7 @@ def _issue_k_load( b.add( b.mul( b.div(src_half, b.const_i32(HD)), - b.const_i32(kv_stride_tok_b), + kv_stride_tok_b, ), b.mul(kv_head_idx, b.const_i32(kv_stride_h_b)), ), @@ -817,7 +834,7 @@ def _issue_v_load( b.add( b.mul( b.div(src_half, b.const_i32(HD)), - b.const_i32(kv_stride_tok_b), + kv_stride_tok_b, ), b.mul(kv_head_idx, b.const_i32(kv_stride_h_b)), ), @@ -885,15 +902,15 @@ def _tile_page(kv_tile_idx: Value) -> Value: return b.div(kv_tile_idx, b.const_i32(TILES_PER_PAGE)) def _tile_base( - page_id: Value, kv_tile_idx: Value, page_stride: int, token_stride: int + page_id: Value, kv_tile_idx: Value, page_stride: Value, token_stride: Value ) -> Value: # Offset of a tile's first token: the page's, plus the tile's rows # before it in the page (sub-page tiles). - base = b.mul(page_id, b.const_i32(page_stride)) + base = b.mul(page_id, page_stride) if TILES_PER_PAGE == 1: return base in_page = b.mod(kv_tile_idx, b.const_i32(TILES_PER_PAGE)) - return b.add(base, b.mul(in_page, b.const_i32(T * token_stride))) + return b.add(base, b.mul(in_page, b.mul(token_stride, b.const_i32(T)))) def _issue_wide_load( src, lds, kv_tile_idx: Value, buf_idx: Value, page_id: Value | None @@ -906,11 +923,14 @@ def _issue_wide_load( if page_id is not None: # The tile base plus the chunk's offset within the page, which # is loop-invariant: no page-table lookup per copy. + # Element strides (the wide loader indexes halves). + tok_e = b.mul(num_kv_p, b.const_i32(HD)) + blk_e = b.mul(tok_e, b.const_i32(BS)) elem_off = b.add( - _tile_base(page_id, kv_tile_idx, BS * NUM_KV * HD, NUM_KV * HD), + _tile_base(page_id, kv_tile_idx, blk_e, tok_e), b.add( b.add( - b.mul(row, b.const_i32(NUM_KV * HD)), + b.mul(row, tok_e), b.mul(kv_head_idx, b.const_i32(HD)), ), col, @@ -1076,15 +1096,13 @@ def _refill_page_id_window(page_id_window: Value, kv_tile_idx: Value) -> Value: qh_r = b.add( b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(lane_col, b.const_i32(NQK)) ) - row_ok = b.land( - b.cmp_lt(qp_r, cur_batch_q_len), b.cmp_lt(qh_r, b.const_i32(NUM_QH)) - ) + row_ok = b.land(b.cmp_lt(qp_r, cur_batch_q_len), b.cmp_lt(qh_r, num_qh_p)) if VALID_ROWS < spec.block_m: row_ok = b.land(row_ok, b.cmp_lt(lane_col, b.const_i32(VALID_ROWS))) causal_lim = b.add(context_len, qp_r) if USE_ALIBI: - qh_ok = b.cmp_lt(qh_r, b.const_i32(NUM_QH)) + qh_ok = b.cmp_lt(qh_r, num_qh_p) alibi_slope = b.masked_global_load( alibi_slopes_ptr, qh_r, qh_ok, b.const_f32(0.0), dtype=F32, align=4 ) @@ -1294,12 +1312,17 @@ def _refill_page_id_window(page_id_window: Value, kv_tile_idx: Value) -> Value: # --------------------------------------------------------------------------- -# Reduce kernel (arch-neutral; byte-for-byte port of the gfx950 reduce kernel) +# Reduce kernel (arch-neutral; the gfx950 reduce with a runtime head count) # --------------------------------------------------------------------------- @dataclass(frozen=True) class UnifiedAttentionReduceTiledSpec: + # Problem fields this body reads as trailing i32 kernargs instead of baking + # them, in ABI order. The launch signature, the argument pack and the cache + # key read the same tuple (``_runtime_kernarg_fields``). + RUNTIME_KERNARG_FIELDS: ClassVar[Tuple[str, ...]] = ("num_query_heads",) + head_size: int num_query_heads: int num_kv_heads: int @@ -1330,7 +1353,6 @@ def kernel_name(self) -> str: parts = [ f"d{self.head_size}", - f"h{self.num_query_heads}", f"seg{self.num_segments}", self.dtype, ] @@ -1347,7 +1369,6 @@ def build_unified_attention_reduce_tiled( """Combine the per-segment partial state into the final fp16/bf16 output.""" HD = spec.head_size NUM_SEG = spec.num_segments - NUM_QH = spec.num_query_heads dtype = spec.dtype_ir THREADS = 64 @@ -1368,6 +1389,8 @@ def build_unified_attention_reduce_tiled( seg_max = b.param("segm_max_ptr", PtrType(F32, "global"), readonly=True, align=4) seg_l = b.param("segm_expsum_ptr", PtrType(F32, "global"), readonly=True, align=4) _seq_lens = b.param("seq_lens_ptr", PtrType(I32, "global"), readonly=True, align=4) + rt = {f: b.param(f, I32) for f in spec.RUNTIME_KERNARG_FIELDS} + num_qh_p = rt["num_query_heads"] lse = ( b.param( "lse_ptr", @@ -1387,20 +1410,21 @@ def build_unified_attention_reduce_tiled( neg_inf = b.const_f32(float("-inf")) zero_f = b.const_f32(0.0) - ml_desc_red = TensorDescriptor.naive( + # The head count is a kernarg, so each token stride is too. + ml_desc_red = DynamicTensorDescriptor.create( "segm_ml", - lengths=[1 << 30, NUM_QH, NUM_SEG], coord_names=("token", "head", "seg"), + strides=(b.mul(num_qh_p, b.const_i32(NUM_SEG)), NUM_SEG, 1), ) - seg_acc_desc_red = TensorDescriptor.naive( + seg_acc_desc_red = DynamicTensorDescriptor.create( "segm_output", - lengths=[1 << 30, NUM_QH, NUM_SEG, HD], coord_names=("token", "head", "seg", "dim"), + strides=(b.mul(num_qh_p, b.const_i32(NUM_SEG * HD)), NUM_SEG * HD, HD, 1), ) - out_desc_red = TensorDescriptor.naive( + out_desc_red = DynamicTensorDescriptor.create( "out", - lengths=[1 << 30, NUM_QH, HD], coord_names=("token", "head", "dim"), + strides=(b.mul(num_qh_p, b.const_i32(HD)), HD, 1), ) base_ml, _ = ml_desc_red.offset(b, token=q_token, head=q_head, seg=b.const_i32(0)) @@ -1494,7 +1518,7 @@ def build_unified_attention_reduce_tiled( # A NaN sum (NaN query row) keeps NaN, as O does: the same !(l == 0) # test the tiled 2D epilogues use, never l > 0. with b.scf_if(b.cmp_eq(tid, b.const_i32(0))): - lse_index = b.add(b.mul(q_token, b.const_i32(NUM_QH)), q_head) + lse_index = b.add(b.mul(q_token, num_qh_p), q_head) lse_store( b, lse, @@ -1517,6 +1541,9 @@ def supports_tiled_3d_spec( This wrapper reads each of them off the spec and delegates, making the 3D unified builder validatable at pack time. """ + ok, reason = check_head_counts(spec.num_query_heads, spec.num_kv_heads) + if not ok: + return False, reason use_fp8 = spec.kv_storage_dtype is not None return supports_tiled_3d( head_size=spec.head_size, diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/attention_tiled_2d.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/attention_tiled_2d.py index 04f05c1d95dd..262f0ce4c0a0 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/attention_tiled_2d.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/attention_tiled_2d.py @@ -40,8 +40,9 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Optional, Tuple +from typing import ClassVar, Optional, Tuple +from kernels.common._fmha_common import check_head_counts, gqa_ratio from kernels.common._attention_additive_bias import ( add_additive_bias, declare_additive_bias_params, @@ -67,7 +68,7 @@ from rocke.helpers.attention import ( apply_softcap_log2, binary_search_seq_idx, - binary_search_seq_idx_iters, + binary_search_seq_idx_trip_count, dequant_fp8x8_to_dtype, mfma_16x16x16_for_dtype, mfma_16x16x32_for_dtype, @@ -80,7 +81,13 @@ ) from rocke.helpers.distribution import make_static_tile_distribution from rocke.helpers.layouts import TransposeLdsReader -from rocke.helpers.transforms import TensorDescriptor, embed, indirect, unmerge +from rocke.helpers.transforms import ( + DynamicTensorDescriptor, + TensorDescriptor, + embed, + indirect, + unmerge, +) MFMA_M = 16 @@ -156,6 +163,18 @@ def _mfma_32x32_c_col(b, lane, n_tile32: int = 0): @dataclass(frozen=True) class UnifiedAttention2DTiledSpec: + # Problem fields this body reads as trailing i32 kernargs instead of baking + # them, in ABI order. The launch signature, the argument pack and the cache + # key read the same tuple (``_runtime_kernarg_fields``). + RUNTIME_KERNARG_FIELDS: ClassVar[Tuple[str, ...]] = ( + "num_query_heads", + "num_kv_heads", + "sliding_window", + ) + # The cu_q search derives its trip count from the num_seqs kernarg, so + # num_seqs never reaches the IR and one binary serves every batch. + RUNTIME_SEARCH_ITERS: ClassVar[bool] = True + head_size: int block_size: int num_query_heads: int @@ -858,7 +877,7 @@ def __post_init__(self): @property def num_queries_per_kv(self) -> int: - return self.num_query_heads // self.num_kv_heads + return gqa_ratio(self.num_query_heads, self.num_kv_heads) @property def block_m(self) -> int: @@ -916,15 +935,6 @@ def mask_type(self) -> int: causal_top_left=self.causal_top_left, ) - @property - def binary_search_iters(self) -> int: - # AITER/Triton uses a true while-loop binary search. Our IR currently - # lowers this as a fixed-trip scf.for, so specialize the trip count to - # the known problem batch size instead of always paying 32 iterations. - # Falls back to 32 for direct unit-test specs that do not provide - # `num_seqs`. - return binary_search_seq_idx_iters(self.num_seqs) - def kernel_name(self) -> str: from rocke.helpers.spec import kernel_name_join @@ -937,11 +947,21 @@ def kernel_name(self) -> str: f"d{self.head_size}", f"b{self.block_size}", f"t{self.tile_size_eff}" if self.n_blocks_per_tile != 1 else "", - f"h{self.num_query_heads}kv{self.num_kv_heads}", + # Head counts are kernargs; only the GQA ratio is baked. + f"g{self.num_queries_per_kv}", + # Except on the fast paged-KV descriptor path, whose shifts bake them. + ( + f"h{self.num_query_heads}kv{self.num_kv_heads}" + if self.use_fast_paged_kv_desc + else "" + ), self.dtype, f"kv{self.kv_storage_dtype}" if self.kv_storage_dtype else "", + # i64 K/V addressing changes the loaders, so it is a distinct kernel. + "i64" if self.use_i64_kv_addr else "", "" if not self.use_sinks else "sinks", - f"sw{self.sliding_window}" if self.sliding_window > 0 else "", + # The window size is a kernarg; only whether the body masks one is baked. + "sw" if self.sliding_window > 0 else "", "softcap" if self.has_softcap else "", "alibi" if self.use_alibi else "", "qqb" if self.use_qq_bias else "", @@ -1178,7 +1198,6 @@ def build_unified_attention_2d_tiled( NQK = spec.num_queries_per_kv VALID_ROWS = BLOCK_Q * NQK NUM_KV = spec.num_kv_heads - NUM_QH = spec.num_query_heads SLIDING_WINDOW = spec.sliding_window USE_SOFTCAP = spec.has_softcap USE_SINKS = spec.use_sinks @@ -1330,6 +1349,11 @@ def build_unified_attention_2d_tiled( num_seqs_p = b.param("num_seqs", I32) bt_stride_p = b.param("block_table_stride", I32) qq_bias_stride0_p = b.param("qq_bias_stride_0", I32) + # Shape values this body reads as kernargs (``RUNTIME_KERNARG_FIELDS``), + # after every fixed param so the shared launch signature can append them. + rt = {f: b.param(f, I32) for f in spec.RUNTIME_KERNARG_FIELDS} + num_qh_p = rt["num_query_heads"] + num_kv_p = rt["num_kv_heads"] # Additive-bias params are appended only when the feature is on, so the # non-bias kernel ABI (and its IR) is unchanged. additive_bias = declare_additive_bias_params(b) if USE_ADDITIVE_BIAS else None @@ -1374,7 +1398,7 @@ def build_unified_attention_2d_tiled( q_block_global_idx, num_seqs_p, block_q=BLOCK_Q, - iterations=spec.binary_search_iters, + iterations=binary_search_seq_idx_trip_count(b, num_seqs_p), ) cu_q_start = b.global_load_i32(cu_q, seq_idx) cu_q_stop = b.global_load_i32(cu_q, b.add(seq_idx, b.const_i32(1))) @@ -1712,7 +1736,7 @@ def _band_row_mask(row_ok, upper_ok, in_prefix): # stays the same as the default bf16 path -- just softmax_scale / # ln(2) -- and we do NOT fold k_scale into qk_scale here. pv_fp8_scale = b.fdiv(v_scale_p, b.const_f32(240.0)) if FP8_MFMA_PV else None - sw_const = b.const_i32(int(SLIDING_WINDOW)) + sw_const = rt["sliding_window"] z8 = b.zero_vec(dtype, 8) # ---------------- Q -> LDS (cooperative vec8 chunks) ---------------- @@ -1725,15 +1749,11 @@ def _band_row_mask(row_ok, upper_ok, in_prefix): # Coordinate transform for Q (and the symmetric output buffer): # ``(token, head, dim)`` packed contiguously. The element-unit # descriptor is reused below for the output store too. - q_desc = TensorDescriptor.naive( + # The head count is a kernarg, so the token stride is too. + q_desc = DynamicTensorDescriptor.create( "Q", - # The runtime extents (total_q, num_query_heads, head_size) are - # only used by the validity predicate (which we don't request - # here). Use generous compile-time bounds so the descriptor's - # row-major stride product matches the kernel's layout - # assumptions exactly. - lengths=[1 << 30, NUM_QH, HD], coord_names=("token", "head", "dim"), + strides=(b.mul(num_qh_p, b.const_i32(HD)), HD, 1), ) # direct-reg: skip the cooperative Q->LDS staging store entirely. # The per-lane Q32 MFMA operand is gathered straight from global in the @@ -1748,7 +1768,7 @@ def _band_row_mask(row_ok, upper_ok, in_prefix): ) _qmask_inner = b.land( b.cmp_lt(q_pos_t, cur_batch_q_len), - b.cmp_lt(qh_t, b.const_i32(NUM_QH)), + b.cmp_lt(qh_t, num_qh_p), ) qmask_t = ( b.land(_qmask_inner, b.cmp_lt(Q_row, b.const_i32(VALID_ROWS))) @@ -1938,7 +1958,7 @@ def _pack_p_a32(p_regs0_f32: list[Value], p_regs1_f32: list[Value]) -> Value: qh = b.add( b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(row, b.const_i32(NQK)) ) - qh_in = b.cmp_lt(qh, b.const_i32(NUM_QH)) + qh_in = b.cmp_lt(qh, num_qh_p) sink_h = b.global_load(sinks, qh, dtype, align=2) sink_f = b.fmul(b.cast_to_f32(sink_h), rcp_ln2) m_inits.append(b.select(qh_in, sink_f, neg_inf)) @@ -1948,7 +1968,7 @@ def _pack_p_a32(p_regs0_f32: list[Value], p_regs1_f32: list[Value]) -> Value: qh = b.add( b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(row, b.const_i32(NQK)) ) - qh_in = b.cmp_lt(qh, b.const_i32(NUM_QH)) + qh_in = b.cmp_lt(qh, num_qh_p) sink_h = b.global_load(sinks, qh, dtype, align=2) sink_f = b.fmul(b.cast_to_f32(sink_h), rcp_ln2) m_inits.append(b.select(qh_in, sink_f, neg_inf)) @@ -2034,10 +2054,11 @@ def _acc_idx(n: int, atom: int) -> int: # so for the FP8 K/V path the loader switches to a sync per-thread # dequant chain below; the byte-stride math here is shared so the # paged_kv_desc compiles consistently in both branches. - kv_stride_blk_b = BS * NUM_KV * HD * KV_BYTES - kv_stride_tok_b = NUM_KV * HD * KV_BYTES + # The token and block strides scale with the runtime kv head count. + kv_stride_tok_b = b.mul(num_kv_p, b.const_i32(HD * KV_BYTES)) + kv_stride_blk_b = b.mul(kv_stride_tok_b, b.const_i32(BS)) kv_stride_h_b = HD * KV_BYTES - kv_block_bytes_c = b.const_i32(kv_stride_blk_b) # one-block buffer bound + kv_block_bytes_c = kv_stride_blk_b # one-block buffer bound lane_half_base = b.mul(tid, b.const_i32(8)) @@ -2138,6 +2159,28 @@ def _acc_idx(n: int, atom: int) -> int: # return ``0`` (a guaranteed-valid block id). Downstream causal / # in_prefix masks already discard the bogus tokens. block_table_max_idx = b.to_sgpr_u32(b.mul(num_seqs_p, bt_stride_p)) + # When a copy covers COPY_TOKENS whole tokens and they fit in one page, the + # bf16/fp16 loaders build the page base, the in-page token offset and the + # lane part separately (see ``_paged_copy``). A copy that ends mid-row + # (head_size 192 with one warp: 512 halves are 2 2/3 rows) or spans pages + # keeps the paged-KV descriptor, which addresses each lane's own row. + COPY_TOKENS = KV_HALVES_PER_CALL // HD + PAGE_COPIES = ( + not FAST_PAGED_KV_DESC + and not KV_FP8 + and KV_HALVES_PER_CALL % HD == 0 + and BS % COPY_TOKENS == 0 + ) + # With page copies, the loop carries the page ids of the current and the + # next tile and reads the ids of the tile after next one iteration ahead, + # so no copy waits on a table read issued just before it. The schedules + # that load two tiles ahead (the K ring, grouped KV2) read in place, and so + # do tiles of more than 16 pages, which the C++ twin's carry arrays bound. + PAGE_PREFETCH = ( + PAGE_COPIES and not RING3 and not GROUPED_KV2 and N_BLOCKS_PER_TILE <= 16 + ) + # Page ids already read for a tile, by the tile index value. + tile_pages = {} if FAST_PAGED_KV_DESC: assert (BS, T, HD, NUM_KV, KV_BYTES, NUM_WARPS, kv_calls_per_tile) == ( 32, @@ -2191,13 +2234,10 @@ def _fast_paged_kv_voff(call: int, block0: Value, block1: Value): return None, b.add(block_b, within) else: - _kv_base = TensorDescriptor.naive( + _kv_base = DynamicTensorDescriptor.create( "paged_kv_bytes", - # ``lengths`` here is just informational (validity propagation - # is driven by the transforms above, not by these bounds). - lengths=[1 << 24, BS, NUM_KV, HD], - strides=[kv_stride_blk_b, kv_stride_tok_b, kv_stride_h_b, KV_BYTES], coord_names=("physical_block", "token", "kv_head", "dim"), + strides=(kv_stride_blk_b, kv_stride_tok_b, kv_stride_h_b, KV_BYTES), ) if N_BLOCKS_PER_TILE == 1: paged_kv_desc = _kv_base.transform( @@ -2230,6 +2270,77 @@ def _fast_paged_kv_voff(call: int, block0: Value, block1: Value): max_idx=block_table_max_idx, ), ) + if PAGE_COPIES: + # When a copy's tokens sit in one page, the bf16/fp16 loaders address + # it as its page's base, its first token's offset in the page and + # the lane's own part. The token stride is a kernarg, so build the + # lane part and the per-copy token offsets once here rather than + # multiplying the token index in every copy of every tile. + lane_tok = b.div(lane_half_base, b.const_i32(HD)) + lane_dim = b.mod(lane_half_base, b.const_i32(HD)) + lane_within_b = b.add( + b.add( + b.mul(lane_tok, kv_stride_tok_b), + b.mul(kv_head_idx, b.const_i32(kv_stride_h_b)), + ), + b.mul(lane_dim, b.const_i32(KV_BYTES)), + ) + copy_tok_b = {} + for call in range(kv_calls_per_tile): + tok0 = (call * COPY_TOKENS) % BS + if tok0 and tok0 not in copy_tok_b: + copy_tok_b[tok0] = b.mul(kv_stride_tok_b, b.const_i32(tok0)) + if I64_KV_ADDR: + kv_stride_blk_i64 = b.zext(kv_stride_blk_b, I64) + + def _tile_page_id(kv_tile_idx: Value, blk: int) -> Value: + """The paged-KV descriptor's indirect lookup for page ``blk`` of a tile: + one guarded block-table read, wave-uniform.""" + idx = b.add( + seq_base, + ( + kv_tile_idx + if N_BLOCKS_PER_TILE == 1 + else b.add( + b.mul(kv_tile_idx, b.const_i32(N_BLOCKS_PER_TILE)), + b.const_i32(blk), + ) + ), + ) + page = b.masked_global_load( + block_tables, + idx, + b.cmp_lt(idx, block_table_max_idx), + b.const_i32(0), + dtype=I32, + align=4, + ) + return b.to_sgpr_u32(page) + + def _tile_page_ids(kv_tile_idx: Value) -> list: + return [_tile_page_id(kv_tile_idx, j) for j in range(N_BLOCKS_PER_TILE)] + + if PAGE_COPIES: + + def _paged_copy(src: Value, src_rsrc: Value, kv_tile_idx: Value, call, pages): + """(rsrc, voffset, soffset) of one bf16/fp16 K/V copy. ``pages`` + caches this tile's page ids by block within the tile when the + loop does not carry them.""" + blk = (call * COPY_TOKENS) // BS + tok0 = (call * COPY_TOKENS) % BS + carried = tile_pages.get(id(kv_tile_idx)) + if carried is not None: + pages = carried + elif blk not in pages: + pages[blk] = _tile_page_id(kv_tile_idx, blk) + voff = lane_within_b + if tok0: + voff = b.add(voff, copy_tok_b[tok0]) + if I64_KV_ADDR: + base_i64 = b.mul(b.zext(pages[blk], I64), kv_stride_blk_i64) + rsrc = b.buffer_rsrc(b.global_ptr_add(src, base_i64), kv_block_bytes_c) + return rsrc, voff, zero_soff + return src_rsrc, voff, b.mul(pages[blk], kv_stride_blk_b) def _issue_k_load_runtime(kv_tile_idx: Value, buf_idx: Value) -> None: """Issue async K loads for one tile into K_lds[buf_idx]. @@ -2252,14 +2363,20 @@ def _issue_k_load_runtime(kv_tile_idx: Value, buf_idx: Value) -> None: K_wave_base = b.smem_ptr_add(K_buf_base, wave_lds_offset_K_i64) if FAST_PAGED_KV_DESC: fast_block0, fast_block1 = _fast_paged_kv_blocks(kv_tile_idx) + pages = {} for call in range(kv_calls_per_tile): k_rsrc = key_rsrc + soff = zero_soff if FAST_PAGED_KV_DESC: base_i64, voff = _fast_paged_kv_voff(call, fast_block0, fast_block1) if base_i64 is not None: # I64_KV_ADDR: per-block 64-bit base k_rsrc = b.buffer_rsrc( b.global_ptr_add(key, base_i64), kv_block_bytes_c ) + elif PAGE_COPIES: + k_rsrc, voff, soff = _paged_copy( + key, key_rsrc, kv_tile_idx, call, pages + ) else: linear_half = b.add( b.const_i32(call * KV_HALVES_PER_CALL), lane_half_base @@ -2288,7 +2405,7 @@ def _issue_k_load_runtime(kv_tile_idx: Value, buf_idx: Value) -> None: # ``dsl_docs/primitives/intrinsics_and_primitives.md`` as the # right hint for K-loop streaming tile loads. b.async_buffer_load_lds_addr( - k_rsrc, k_dst, voff, zero_soff, 4, coherency=CACHE_STREAM + k_rsrc, k_dst, voff, soff, 4, coherency=CACHE_STREAM ) def _issue_v_load_runtime(kv_tile_idx: Value, buf_idx: Value) -> None: @@ -2310,14 +2427,20 @@ def _issue_v_load_runtime(kv_tile_idx: Value, buf_idx: Value) -> None: V_wave_base = b.smem_ptr_add(V_lds_addr, wave_lds_offset_i64) if FAST_PAGED_KV_DESC: fast_block0, fast_block1 = _fast_paged_kv_blocks(kv_tile_idx) + pages = {} for call in range(kv_calls_per_tile): v_rsrc = value_rsrc + soff = zero_soff if FAST_PAGED_KV_DESC: base_i64, voff = _fast_paged_kv_voff(call, fast_block0, fast_block1) if base_i64 is not None: v_rsrc = b.buffer_rsrc( b.global_ptr_add(value, base_i64), kv_block_bytes_c ) + elif PAGE_COPIES: + v_rsrc, voff, soff = _paged_copy( + value, value_rsrc, kv_tile_idx, call, pages + ) else: linear_half = b.add( b.const_i32(call * KV_HALVES_PER_CALL), lane_half_base @@ -2345,7 +2468,7 @@ def _issue_v_load_runtime(kv_tile_idx: Value, buf_idx: Value) -> None: # re-read within this kernel; see _issue_k_load_runtime for # the rationale. b.async_buffer_load_lds_addr( - v_rsrc, v_dst, voff, zero_soff, 4, coherency=CACHE_STREAM + v_rsrc, v_dst, voff, soff, 4, coherency=CACHE_STREAM ) # ---------------- FP8 K/V cache: async DMA loader (round 2) ---------------- @@ -2981,7 +3104,7 @@ def _read_q32(k: int) -> Value: # so the <8 x dtype> operand is bit-identical to the staged path. # row -> q_pos = qb_start_pos + row // NQK ; qh = kv_head*NQK + row%NQK # col -> dim - # Out-of-range (token >= q_len or head >= NUM_QH) lanes read 0 (masked), + # Out-of-range (token >= q_len or head >= num_query_heads) lanes read 0 (masked), # matching the staging store's zero-fill. def _read_q32_global(k: int) -> Value: q32_col = b.add(b.const_i32(k * 16), b.mul(lane_half, b.const_i32(8))) @@ -2992,7 +3115,7 @@ def _read_q32_global(k: int) -> Value: ) qg_mask = b.land( b.cmp_lt(qg_pos, cur_batch_q_len), - b.cmp_lt(qg_h, b.const_i32(NUM_QH)), + b.cmp_lt(qg_h, num_qh_p), ) qg_pos_safe = b.select(qg_mask, qg_pos, b.const_i32(0)) qg_h_safe = b.select(qg_mask, qg_h, b.const_i32(0)) @@ -3054,6 +3177,16 @@ def _read_q32_global(k: int) -> Value: # the next thing scheduled, so this ordering is naturally # correct in the IR. + if PAGE_PREFETCH: + # Page ids of the first tile and of the one after it, carried into the + # loop; iteration i then reads the ids of tile i + 2. + start_next_raw = b.add(tile_start, b.const_i32(1)) + start_next = b.select( + b.cmp_lt(start_next_raw, tile_end), start_next_raw, tile_start + ) + start_pages = _tile_page_ids(tile_start) + start_next_pages = _tile_page_ids(start_next) + tile_pages[id(tile_start)] = start_pages # Prefetch tile_start's K into buffer 0 BEFORE the loop. _issue_k(tile_start, b.const_i32(0)) if RING3: @@ -3089,12 +3222,17 @@ def _read_q32_global(k: int) -> Value: # 6. Yield ``(m, l, acc, nxt_buf)`` so the next iter consumes nxt_buf. cur_buf_init = b.const_i32(0) iter_args.append(("cur_buf", cur_buf_init)) + if PAGE_PREFETCH: + for j in range(N_BLOCKS_PER_TILE): + iter_args.append((f"page_cur{j}", start_pages[j])) + for j in range(N_BLOCKS_PER_TILE): + iter_args.append((f"page_next{j}", start_next_pages[j])) # ---------------- LICM hoist: per-reg invariants ---------------- # ``qp_r``, ``qh_r``, ``row_ok``, ``causal_lim``, and ``alibi_per_row`` # depend only on CTA-level constants (``wave_row_base``, ``lane_rg``, # ``qb_start_pos``, ``kv_head_idx``, ``cur_batch_q_len``, ``context_len``, - # ``NUM_QH``, ``NQK``). Computing them BEFORE the kvloop avoids paying + # ``num_qh_p``, ``NQK``). Computing them BEFORE the kvloop avoids paying # them per kv tile. LLVM LICM should hoist them automatically, but # explicit hoisting makes the IR's hot path leaner and eliminates a # source of compiler-scheduling variability. The per-reg arrays @@ -3114,7 +3252,7 @@ def _read_q32_global(k: int) -> Value: qh_r = b.add(b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(row, b.const_i32(NQK))) _row_ok_inner = b.land( b.cmp_lt(qp_r, cur_batch_q_len), - b.cmp_lt(qh_r, b.const_i32(NUM_QH)), + b.cmp_lt(qh_r, num_qh_p), ) row_ok = ( b.land(_row_ok_inner, b.cmp_lt(row, b.const_i32(VALID_ROWS))) @@ -3132,7 +3270,7 @@ def _read_q32_global(k: int) -> Value: hoist_alibi = [] for reg in range(REGS_PER_LANE): qh_r = hoist_qh_r[reg] - qh_ok = b.cmp_lt(qh_r, b.const_i32(NUM_QH)) + qh_ok = b.cmp_lt(qh_r, num_qh_p) slope = b.masked_global_load( alibi_slopes_ptr, qh_r, qh_ok, b.const_f32(0.0), dtype=F32, align=4 ) @@ -3148,7 +3286,7 @@ def _read_q32_global(k: int) -> Value: ) _st_row_ok_inner = b.land( b.cmp_lt(st_qp, cur_batch_q_len), - b.cmp_lt(st_qh, b.const_i32(NUM_QH)), + b.cmp_lt(st_qh, num_qh_p), ) st_row_ok = ( b.land(_st_row_ok_inner, b.cmp_lt(st_q_row, b.const_i32(VALID_ROWS))) @@ -3175,7 +3313,7 @@ def _read_q32_global(k: int) -> Value: and TRANSPOSED_QK_32X32 and TRANSPOSED_INVARIANT_HOIST ): - st_qh_ok = b.cmp_lt(st_qh, b.const_i32(NUM_QH)) + st_qh_ok = b.cmp_lt(st_qh, num_qh_p) st_alibi_slope = b.masked_global_load( alibi_slopes_ptr, st_qh, st_qh_ok, b.const_f32(0.0), dtype=F32, align=4 ) @@ -3209,6 +3347,13 @@ def _acc_get(n: int, atom: int) -> Value: return acc_vals[n * ACC_M_ATOMS + atom] cur_buf = carry[ml_count + ACC_N_TILES * ACC_M_ATOMS] + if PAGE_PREFETCH: + pages_at = ml_count + ACC_N_TILES * ACC_M_ATOMS + 1 + cur_pages = list(carry[pages_at : pages_at + N_BLOCKS_PER_TILE]) + next_pages = list( + carry[pages_at + N_BLOCKS_PER_TILE : pages_at + 2 * N_BLOCKS_PER_TILE] + ) + tile_pages[id(kv_tile_iv)] = cur_pages if RING3: # 3-slot ring: cur_buf holds K[i]; (cur_buf+1)%3 holds K[i+1] # (prefetched last iter / prologue); the prefetch target for K[i+2] @@ -3254,7 +3399,7 @@ def _mod3_add(x, c): ) _st_row_ok_iter_inner = b.land( b.cmp_lt(st_qp_iter, cur_batch_q_len), - b.cmp_lt(st_qh_iter, b.const_i32(NUM_QH)), + b.cmp_lt(st_qh_iter, num_qh_p), ) st_row_ok_iter = ( b.land( @@ -3266,7 +3411,7 @@ def _mod3_add(x, c): ) st_causal_lim_iter = b.add(context_len, st_qp_iter) if USE_ALIBI: - st_qh_iter_ok = b.cmp_lt(st_qh_iter, b.const_i32(NUM_QH)) + st_qh_iter_ok = b.cmp_lt(st_qh_iter, num_qh_p) st_alibi_slope_iter = b.masked_global_load( alibi_slopes_ptr, st_qh_iter, @@ -3292,6 +3437,8 @@ def _mod3_add(x, c): next_tile_iv_raw = b.add(kv_tile_iv, b.const_i32(1)) in_range_next = b.cmp_lt(next_tile_iv_raw, tile_end) safe_next_tile = b.select(in_range_next, next_tile_iv_raw, kv_tile_iv) + if PAGE_PREFETCH: + tile_pages[id(safe_next_tile)] = next_pages if RING3: # Ring depth 3 prefetches K[i+2] (two tiles ahead) into ring_pf_buf. ring_pf_tile_raw = b.add(kv_tile_iv, b.const_i32(2)) @@ -3334,6 +3481,15 @@ def _mod3_add(x, c): # GEMM MFMA-interleave, level 1 = attention-style interleave. if USE_SOFTMAX_INTERLEAVE and SOFTMAX_INTERLEAVE_MODE in (0, 1): b.iglp_opt(SOFTMAX_INTERLEAVE_MODE) + if PAGE_PREFETCH: + # Read the page ids of the tile after next now, right after the + # iter-start drain, so they arrive long before next iteration's + # copies need them. That tile is next iteration's safe_next_tile. + after_next_raw = b.add(kv_tile_iv, b.const_i32(2)) + after_next = b.select( + b.cmp_lt(after_next_raw, tile_end), after_next_raw, next_tile_iv_raw + ) + after_next_pages = _tile_page_ids(after_next) if EARLY_V_SCHEDULE: if V_DOUBLE_BUF: # V[i] (cur_buf) was prefetched in the previous iter / prologue @@ -3537,7 +3693,7 @@ def _mod3_add(x, c): slope_for_lane = b.masked_global_load( alibi_slopes_ptr, qh_t_alibi, - b.cmp_lt(qh_t_alibi, b.const_i32(NUM_QH)), + b.cmp_lt(qh_t_alibi, num_qh_p), b.const_f32(0.0), dtype=F32, align=4, @@ -3592,7 +3748,7 @@ def _mod3_add(x, c): ) _row_ok_inner_t = b.land( b.cmp_lt(qp_r, cur_batch_q_len), - b.cmp_lt(qh_r, b.const_i32(NUM_QH)), + b.cmp_lt(qh_r, num_qh_p), ) row_ok = ( b.land( @@ -4498,6 +4654,9 @@ def _apply_transposed_pv_regs(acc32: Value, n: int, p_regs) -> Value: yields.append(ring_next_cur) else: yields.append(cur_buf if GROUPED_KV2 else nxt_buf) + if PAGE_PREFETCH: + yields.extend(next_pages) + yields.extend(after_next_pages) b.scf_yield(*yields) # ---- drive the (one or two) KV-loop phases ---- @@ -4611,7 +4770,7 @@ def _acc_final_get(n: int, atom: int) -> Value: ) _op_mask_t_inner = b.land( b.cmp_lt(op_pos_t, cur_batch_q_len), - b.cmp_lt(op_qh_t, b.const_i32(NUM_QH)), + b.cmp_lt(op_qh_t, num_qh_p), ) op_mask_t = ( b.land(_op_mask_t_inner, b.cmp_lt(q_row_t, b.const_i32(VALID_ROWS))) @@ -4634,7 +4793,7 @@ def _acc_final_get(n: int, atom: int) -> Value: b, lse, b.add( - b.mul(b.add(cu_q_start, op_pos_t), b.const_i32(NUM_QH)), + b.mul(b.add(cu_q_start, op_pos_t), num_qh_p), op_qh_t, ), m_final[0], @@ -4679,7 +4838,7 @@ def _acc_final_get(n: int, atom: int) -> Value: ) _op_mask32_inner = b.land( b.cmp_lt(op_pos32_base, cur_batch_q_len), - b.cmp_lt(op_qh32_base, b.const_i32(NUM_QH)), + b.cmp_lt(op_qh32_base, num_qh_p), ) op_mask32_base = ( b.land(_op_mask32_inner, b.cmp_lt(OUT_ROW_BASE32, b.const_i32(VALID_ROWS))) @@ -4705,7 +4864,7 @@ def _acc_final_get(n: int, atom: int) -> Value: b.add( b.mul( b.add(cu_q_start, hoist_qp_r[reg]), - b.const_i32(NUM_QH), + num_qh_p, ), hoist_qh_r[reg], ), @@ -4799,7 +4958,7 @@ def _acc_final_get(n: int, atom: int) -> Value: ) _op_mask_inner = b.land( b.cmp_lt(op_pos, cur_batch_q_len), - b.cmp_lt(op_qh, b.const_i32(NUM_QH)), + b.cmp_lt(op_qh, num_qh_p), ) op_mask = ( b.land(_op_mask_inner, b.cmp_lt(OUT_ROW_BASE, b.const_i32(VALID_ROWS))) @@ -4865,7 +5024,7 @@ def _acc_final_get(n: int, atom: int) -> Value: b.add( b.mul( b.add(cu_q_start, hoist_qp_r[reg]), - b.const_i32(NUM_QH), + num_qh_p, ), hoist_qh_r[reg], ), @@ -4892,6 +5051,9 @@ def supports_tiled_2d_spec( ``use_conflict_free_v_store``, ``use_k_sliced_ring`` and ``use_d256_fast`` are not gfx950 spec fields. """ + ok, reason = check_head_counts(spec.num_query_heads, spec.num_kv_heads) + if not ok: + return False, reason use_fp8 = spec.kv_storage_dtype is not None return supports_tiled_2d( head_size=spec.head_size, diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/attention_tiled_2d_fastkv_regp.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/attention_tiled_2d_fastkv_regp.py index dfabc19a5e89..4a1b73eb18e5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/attention_tiled_2d_fastkv_regp.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/attention_tiled_2d_fastkv_regp.py @@ -18,6 +18,7 @@ from dataclasses import replace from typing import Any +from kernels.common._fmha_common import check_head_counts from rocke.core.ir import KernelDef from .attention_tiled_2d import ( UnifiedAttention2DTiledSpec, @@ -94,6 +95,10 @@ def make_fastkv_register_p_spec( ) +# The body uses gfx950-only wide MFMA atoms and transposed LDS reads. +_NOT_GFX950 = "fastKV register-P is a gfx950 kernel; {arch} is not supported" + + def supports_fastkv_register_p_2d( *, head_size: int, @@ -109,8 +114,13 @@ def supports_fastkv_register_p_2d( tile_size: int | None = None, arch: str = "gfx950", ) -> tuple[bool, str]: - """Check support for the experimental fastKV + register-P kernel.""" + """Check support for the experimental fastKV + register-P kernel. + ``supports_tiled_2d`` also admits gfx942 for its narrow-atom variant; this + kernel's body is gfx950-only, so every other target is refused first. + """ + if arch != "gfx950": + return False, _NOT_GFX950.format(arch=arch) ok, reason = supports_tiled_2d( head_size=head_size, block_size=block_size, @@ -141,6 +151,52 @@ def supports_fastkv_register_p_2d( return True, "supported" +def _fastkv_register_p_spec_error(spec: UnifiedAttention2DTiledSpec) -> str | None: + """Why the fastKV register-P builder refuses ``spec``, or None if it builds.""" + if not spec.use_fast_paged_kv_desc: + return "fastKV register-P experiment requires use_fast_paged_kv_desc" + if not (spec.use_mfma_32x32 and spec.use_transposed_qk_32x32): + return "fastKV register-P experiment requires transposed R4" + if spec.kv_storage_dtype is not None: + return "fastKV register-P experiment does not support FP8 KV cache" + return None + + +def supports_fastkv_register_p_2d_spec( + spec: UnifiedAttention2DTiledSpec, *, arch: str = "gfx950" +) -> tuple[bool, str]: + """Spec-shaped wrapper for ``supports_fastkv_register_p_2d``. + + ``supports_fastkv_register_p_2d`` takes its parameters as keyword-only + arguments, so ``hkp_pack``'s ``_check_support_predicate`` cannot call it with a + spec instance. This wrapper reads them off the spec and delegates, after + refusing what :func:`build_unified_attention_2d_fastkv_register_p` raises on, + so the builder is validatable at pack time. + """ + ok, reason = check_head_counts(spec.num_query_heads, spec.num_kv_heads) + if not ok: + return False, reason + if arch != "gfx950": + return False, _NOT_GFX950.format(arch=arch) + error = _fastkv_register_p_spec_error(spec) + if error is not None: + return False, error + return supports_fastkv_register_p_2d( + head_size=spec.head_size, + block_size=spec.block_size, + dtype=spec.dtype, + num_queries_per_kv=spec.num_queries_per_kv, + use_alibi=spec.use_alibi, + use_qq_bias=spec.use_qq_bias, + use_fp8=False, + q_dtype=spec.dtype, + num_query_heads=spec.num_query_heads, + num_kv_heads=spec.num_kv_heads, + tile_size=spec.tile_size, + arch=arch, + ) + + def build_unified_attention_2d_fastkv_register_p( spec: UnifiedAttention2DTiledSpec, *, @@ -148,15 +204,14 @@ def build_unified_attention_2d_fastkv_register_p( ) -> KernelDef: """Build the experimental fastKV + register-P 2D attention kernel. - Thin wrapper over :func:`build_unified_attention_2d_tiled`; the ``arch`` - target (gfx950-only, see that builder) is threaded straight through, so - a gfx942 request fails with the same clean structured error. + Thin wrapper over :func:`build_unified_attention_2d_tiled`. That builder + also admits gfx942 for its narrow-atom variant, so this one refuses every + target but gfx950 itself. """ - if not spec.use_fast_paged_kv_desc: - raise ValueError("fastKV register-P experiment requires use_fast_paged_kv_desc") - if not (spec.use_mfma_32x32 and spec.use_transposed_qk_32x32): - raise ValueError("fastKV register-P experiment requires transposed R4") - if spec.kv_storage_dtype is not None: - raise ValueError("fastKV register-P experiment does not support FP8 KV cache") + if arch != "gfx950": + raise ValueError(_NOT_GFX950.format(arch=arch)) + error = _fastkv_register_p_spec_error(spec) + if error is not None: + raise ValueError(error) return build_unified_attention_2d_tiled(_FastKvRegisterPProxy(spec), arch=arch) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/attention_tiled_3d.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/attention_tiled_3d.py index 83e49482c65e..7552e6ba8437 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/attention_tiled_3d.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/attention_tiled_3d.py @@ -28,8 +28,9 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Optional, Tuple +from typing import ClassVar, Optional, Tuple +from kernels.common._fmha_common import check_head_counts, gqa_ratio from kernels.common._attention_additive_bias import ( add_additive_bias, declare_additive_bias_params, @@ -66,7 +67,7 @@ from rocke.helpers.attention import ( apply_softcap_log2 as _apply_softcap, binary_search_seq_idx as _binary_search_seq_idx_helper, - binary_search_seq_idx_iters, + binary_search_seq_idx_trip_count, dequant_fp8x8_to_dtype, mfma_16x16x16_for_dtype as _mfma_16x16x16, mfma_16x16x32_for_dtype as _mfma_16x16x32, @@ -75,7 +76,7 @@ ) from rocke.helpers.distribution import make_static_tile_distribution from rocke.helpers.layouts import TransposeLdsReader -from rocke.helpers.transforms import TensorDescriptor, indirect, unmerge +from rocke.helpers.transforms import DynamicTensorDescriptor, indirect, unmerge MFMA_M = 16 MFMA_N = 16 @@ -131,6 +132,18 @@ class UnifiedAttention3DTiledSpec: knob exactly as AITER's ``select_3d_config`` derives it. """ + # Problem fields this body reads as trailing i32 kernargs instead of baking + # them, in ABI order. The launch signature, the argument pack and the cache + # key read the same tuple (``_runtime_kernarg_fields``). + RUNTIME_KERNARG_FIELDS: ClassVar[Tuple[str, ...]] = ( + "num_query_heads", + "num_kv_heads", + "sliding_window", + ) + # The cu_q search derives its trip count from the num_seqs kernarg, so + # num_seqs never reaches the IR and one binary serves every batch. + RUNTIME_SEARCH_ITERS: ClassVar[bool] = True + head_size: int block_size: int num_query_heads: int @@ -303,7 +316,7 @@ def __post_init__(self): @property def num_queries_per_kv(self) -> int: - return self.num_query_heads // self.num_kv_heads + return gqa_ratio(self.num_query_heads, self.num_kv_heads) @property def block_m(self) -> int: @@ -327,10 +340,6 @@ def n_blocks_per_tile(self) -> int: def dtype_ir(self) -> Type: return F16 if self.dtype == "fp16" else BF16 - @property - def binary_search_iters(self) -> int: - return binary_search_seq_idx_iters(self.num_seqs) - def mask_type(self) -> int: """The band as one ``AttentionMaskType`` value, by the mapping ``UnifiedAttention2DTiledSpec.mask_type`` also delegates to @@ -356,13 +365,15 @@ def kernel_name(self) -> str: f"d{self.head_size}", f"b{self.block_size}", f"t{self.tile_size}" if self.n_blocks_per_tile != 1 else "", - f"h{self.num_query_heads}kv{self.num_kv_heads}", + # Head counts are kernargs; only the GQA ratio is baked. + f"g{self.num_queries_per_kv}", f"seg{self.num_segments}", self.dtype, f"kv{self.kv_storage_dtype}" if self.kv_storage_dtype else "", "i64kv" if self.use_i64_kv_addr else "", "sinks" if self.use_sinks else "", - f"sw{self.sliding_window}" if self.sliding_window > 0 else "", + # The window size is a kernarg; only whether the body masks one is baked. + "sw" if self.sliding_window > 0 else "", "softcap" if self.has_softcap else "", "alibi" if self.use_alibi else "", "qqb" if self.use_qq_bias else "", @@ -478,8 +489,6 @@ def build_unified_attention_3d_tiled( # Rows past BLOCK_Q * NQK are q-block padding when NQK does not divide # BLOCK_M; they must not store, as they alias the next q-block's rows. VALID_ROWS = spec.block_q * NQK - NUM_KV = spec.num_kv_heads - NUM_QH = spec.num_query_heads NUM_SEG = spec.num_segments SLIDING_WINDOW = spec.sliding_window USE_SOFTCAP = spec.has_softcap @@ -561,6 +570,11 @@ def build_unified_attention_3d_tiled( num_seqs_p = b.param("num_seqs", I32) bt_stride_p = b.param("block_table_stride", I32) qq_bias_stride0_p = b.param("qq_bias_stride_0", I32) + # Shape values this body reads as kernargs (``RUNTIME_KERNARG_FIELDS``), + # after every fixed param so the shared launch signature can append them. + rt = {f: b.param(f, I32) for f in spec.RUNTIME_KERNARG_FIELDS} + num_qh_p = rt["num_query_heads"] + num_kv_p = rt["num_kv_heads"] # Additive-bias params are appended only when the feature is on, so the # non-bias kernel ABI (and its IR) is unchanged. additive_bias = declare_additive_bias_params(b) if USE_ADDITIVE_BIAS else None @@ -583,7 +597,7 @@ def build_unified_attention_3d_tiled( q_block_global_idx, num_seqs_p, block_q=BLOCK_Q, - iterations=spec.binary_search_iters, + iterations=binary_search_seq_idx_trip_count(b, num_seqs_p), ) cu_q_start = b.global_load_i32(cu_q, seq_idx) cu_q_stop = b.global_load_i32(cu_q, b.add(seq_idx, b.const_i32(1))) @@ -628,23 +642,22 @@ def _band_row_mask(row_ok, upper_ok, in_prefix): # segm_max [total_q, num_qh, num_segments] # segm_expsum [total_q, num_qh, num_segments] # segm_output [total_q, num_qh, num_segments, head_size] - # plus Q / output which both share ``(token, head, dim)``. ``1<<30`` - # is a compile-time upper bound on ``total_q`` so the row-major - # stride product matches the kernel's layout assumptions exactly. - ml_desc = TensorDescriptor.naive( + # plus Q / output which both share ``(token, head, dim)``. The head count + # is a kernarg, so each token stride is too. + ml_desc = DynamicTensorDescriptor.create( "segm_ml", - lengths=[1 << 30, NUM_QH, NUM_SEG], coord_names=("token", "head", "seg"), + strides=(b.mul(num_qh_p, b.const_i32(NUM_SEG)), NUM_SEG, 1), ) - seg_acc_desc = TensorDescriptor.naive( + seg_acc_desc = DynamicTensorDescriptor.create( "segm_output", - lengths=[1 << 30, NUM_QH, NUM_SEG, HD], coord_names=("token", "head", "seg", "dim"), + strides=(b.mul(num_qh_p, b.const_i32(NUM_SEG * HD)), NUM_SEG * HD, HD, 1), ) - q_desc = TensorDescriptor.naive( + q_desc = DynamicTensorDescriptor.create( "Q", - lengths=[1 << 30, NUM_QH, HD], coord_names=("token", "head", "dim"), + strides=(b.mul(num_qh_p, b.const_i32(HD)), HD, 1), ) seg_start_tile_pos = b.mul(b.mul(seg_idx, tps), b.const_i32(T)) @@ -658,9 +671,7 @@ def _band_row_mask(row_ok, upper_ok, in_prefix): qh_r = b.add( b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(row, b.const_i32(NQK)) ) - row_ok = b.land( - b.cmp_lt(qp_r, cur_batch_q_len), b.cmp_lt(qh_r, b.const_i32(NUM_QH)) - ) + row_ok = b.land(b.cmp_lt(qp_r, cur_batch_q_len), b.cmp_lt(qh_r, num_qh_p)) if VALID_ROWS < spec.block_m: row_ok = b.land(row_ok, b.cmp_lt(row, b.const_i32(VALID_ROWS))) qp_r_safe = b.select(row_ok, qp_r, b.const_i32(0)) @@ -682,7 +693,7 @@ def _band_row_mask(row_ok, upper_ok, in_prefix): b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(row, b.const_i32(NQK)) ) row_ok = b.land( - b.cmp_lt(qp_r, cur_batch_q_len), b.cmp_lt(qh_r, b.const_i32(NUM_QH)) + b.cmp_lt(qp_r, cur_batch_q_len), b.cmp_lt(qh_r, num_qh_p) ) if VALID_ROWS < spec.block_m: row_ok = b.land(row_ok, b.cmp_lt(row, b.const_i32(VALID_ROWS))) @@ -726,7 +737,7 @@ def _row_reduce(x: Value, op) -> Value: one_f = b.const_f32(1.0) rcp_ln2 = b.const_f32(1.4426950408889634) qk_scale = b.fmul(scale_p, rcp_ln2) - sw_const = b.const_i32(int(SLIDING_WINDOW)) + sw_const = rt["sliding_window"] z8 = b.zero_vec(dtype, 8) # ---------------- Q -> LDS ---------------- @@ -741,9 +752,7 @@ def _row_reduce(x: Value, op) -> Value: b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(Q_row, b.const_i32(NQK)), ) - qmask_t = b.land( - b.cmp_lt(q_pos_t, cur_batch_q_len), b.cmp_lt(qh_t, b.const_i32(NUM_QH)) - ) + qmask_t = b.land(b.cmp_lt(q_pos_t, cur_batch_q_len), b.cmp_lt(qh_t, num_qh_p)) q_pos_safe = b.select(qmask_t, q_pos_t, b.const_i32(0)) qh_safe = b.select(qmask_t, qh_t, b.const_i32(0)) q_off_base, _ = q_desc.offset( @@ -797,7 +806,7 @@ def _row_reduce(x: Value, op) -> Value: qh = b.add( b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(lane_col, b.const_i32(NQK)) ) - qh_in = b.cmp_lt(qh, b.const_i32(NUM_QH)) + qh_in = b.cmp_lt(qh, num_qh_p) sink_h = b.global_load(sinks, qh, dtype, align=2) sink_f = b.fmul(b.cast_to_f32(sink_h), rcp_ln2) sink_with_mask = b.select(qh_in, sink_f, neg_inf) @@ -828,14 +837,15 @@ def _row_reduce(x: Value, op) -> Value: # fp8e4m3. The FP8 sync loader path below dequantises and stores to # LDS in the working dtype, so ``bytes_per_buf`` stays at the working # dtype slab size. - kv_stride_blk_b = BS * NUM_KV * HD * KV_BYTES - kv_stride_tok_b = NUM_KV * HD * KV_BYTES + # The token and page strides scale with the runtime kv head count. + kv_stride_tok_b = b.mul(num_kv_p, b.const_i32(HD * KV_BYTES)) + kv_stride_blk_b = b.mul(kv_stride_tok_b, b.const_i32(BS)) kv_stride_h_b = HD * KV_BYTES bytes_per_buf = T * HD * 2 # One-block buffer bound for the i64-addressing path: each per-block # buffer descriptor only spans a single physical KV block, so the # within-block i32 voffset can never overflow. - kv_block_bytes_c = b.const_i32(kv_stride_blk_b) + kv_block_bytes_c = kv_stride_blk_b lane_half_base = b.mul(tid, b.const_i32(8)) K_lds_addr = b.smem_addr_of(K_lds) @@ -860,11 +870,10 @@ def _row_reduce(x: Value, op) -> Value: PAGE_IDS = not KV_FP8 and spec.kv_layout == "paged" WAVE_PAGE_ID_LOAD = spec.use_wave_page_id_load if not PAGE_IDS: - paged_kv_desc = TensorDescriptor.naive( + paged_kv_desc = DynamicTensorDescriptor.create( "paged_kv_bytes", - lengths=[1 << 24, T, NUM_KV, HD], - strides=[kv_stride_blk_b, kv_stride_tok_b, kv_stride_h_b, KV_BYTES], coord_names=("physical_block", "token", "kv_head", "dim"), + strides=(kv_stride_blk_b, kv_stride_tok_b, kv_stride_h_b, KV_BYTES), ).transform( indirect( "tile_idx", into="physical_block", table=block_tables, base=seq_base @@ -886,7 +895,7 @@ def _row_reduce(x: Value, op) -> Value: lane_tok = b.div(lane_half_base, b.const_i32(HD)) lane_dim = b.mod(lane_half_base, b.const_i32(HD)) lane_tok_head = b.add( - b.mul(lane_tok, b.const_i32(kv_stride_tok_b)), + b.mul(lane_tok, kv_stride_tok_b), b.mul(kv_head_idx, b.const_i32(kv_stride_h_b)), ) # Per-lane byte offset within a page, keyed by the call's swizzle @@ -910,6 +919,17 @@ def _row_reduce(x: Value, op) -> Value: lane_within_b[0] = b.add( lane_tok_head, b.mul(lane_dim, b.const_i32(KV_BYTES)) ) + # The token stride is a kernarg, so each call's token offset within the + # page is a multiply. They are wave-uniform and loop-invariant: build + # them once here rather than in every tile's copies. Calls that cover a + # partial row (head_size 192) compute their own offsets instead. + call_tok_b = {} + for call in range(kv_calls_per_tile if KV_HALVES_PER_CALL % HD == 0 else 0): + tok0 = (call % calls_per_block) * tokens_per_call + if tok0 and tok0 not in call_tok_b: + call_tok_b[tok0] = b.mul(kv_stride_tok_b, b.const_i32(tok0)) + if I64_KV_ADDR: + kv_stride_blk_i64 = b.zext(kv_stride_blk_b, I64) def _kv_col(row: Value, col: Value) -> Value: # LDS column of K/V element (row, col): its 16-byte chunk is permuted @@ -964,23 +984,23 @@ def _issue_paged_tile( # The call covers a partial row (head_size 192), so each lane # takes its own (row, dim); the tile is one page (spec check). lin = b.add(b.const_i32(call * KV_HALVES_PER_CALL), lane_half_base) - tok_b = b.mul(b.div(lin, b.const_i32(HD)), b.const_i32(kv_stride_tok_b)) + tok_b = b.mul(b.div(lin, b.const_i32(HD)), kv_stride_tok_b) head_b = b.mul(kv_head_idx, b.const_i32(kv_stride_h_b)) dim_b = b.mul(b.mod(lin, b.const_i32(HD)), b.const_i32(KV_BYTES)) voff = b.add(b.add(tok_b, head_b), dim_b) else: - voff = b.add(lane_within_b[ph], b.const_i32(tok0 * kv_stride_tok_b)) + voff = lane_within_b[ph] + if tok0: + voff = b.add(voff, call_tok_b[tok0]) if I64_KV_ADDR: - base_i64 = b.mul( - b.zext(page_ids[blk], I64), b.const_i64(kv_stride_blk_b) - ) + base_i64 = b.mul(b.zext(page_ids[blk], I64), kv_stride_blk_i64) call_rsrc = b.buffer_rsrc( b.global_ptr_add(src, base_i64), kv_block_bytes_c ) soff = zero_soff else: call_rsrc = src_rsrc - soff = b.mul(page_ids[blk], b.const_i32(kv_stride_blk_b)) + soff = b.mul(page_ids[blk], kv_stride_blk_b) dst = b.smem_ptr_add(buf_base, b.const_i64(call * bytes_per_call)) b.async_buffer_load_lds_addr( call_rsrc, dst, voff, soff, 4, coherency=KV_CACHE_BITS @@ -1181,15 +1201,13 @@ def _refill_page_id_window(page_id_window: Value, kv_tile_idx: Value) -> Value: qh_r = b.add( b.mul(kv_head_idx, b.const_i32(NQK)), b.mod(lane_col, b.const_i32(NQK)) ) - row_ok = b.land( - b.cmp_lt(qp_r, cur_batch_q_len), b.cmp_lt(qh_r, b.const_i32(NUM_QH)) - ) + row_ok = b.land(b.cmp_lt(qp_r, cur_batch_q_len), b.cmp_lt(qh_r, num_qh_p)) if VALID_ROWS < spec.block_m: row_ok = b.land(row_ok, b.cmp_lt(lane_col, b.const_i32(VALID_ROWS))) causal_lim = b.add(context_len, qp_r) if USE_ALIBI: - qh_ok = b.cmp_lt(qh_r, b.const_i32(NUM_QH)) + qh_ok = b.cmp_lt(qh_r, num_qh_p) alibi_slope = b.masked_global_load( alibi_slopes_ptr, qh_r, qh_ok, b.const_f32(0.0), dtype=F32, align=4 ) @@ -1434,6 +1452,11 @@ def _refill_page_id_window(page_id_window: Value, kv_tile_idx: Value) -> Value: @dataclass(frozen=True) class UnifiedAttentionReduceTiledSpec: + # Problem fields this body reads as trailing i32 kernargs instead of baking + # them, in ABI order. The launch signature, the argument pack and the cache + # key read the same tuple (``_runtime_kernarg_fields``). + RUNTIME_KERNARG_FIELDS: ClassVar[Tuple[str, ...]] = ("num_query_heads",) + head_size: int num_query_heads: int num_kv_heads: int @@ -1466,7 +1489,6 @@ def kernel_name(self) -> str: parts = [ f"d{self.head_size}", - f"h{self.num_query_heads}", f"seg{self.num_segments}", self.dtype, ] @@ -1498,7 +1520,6 @@ def build_unified_attention_reduce_tiled( """ HD = spec.head_size NUM_SEG = spec.num_segments - NUM_QH = spec.num_query_heads dtype = spec.dtype_ir THREADS = 64 @@ -1519,6 +1540,8 @@ def build_unified_attention_reduce_tiled( seg_max = b.param("segm_max_ptr", PtrType(F32, "global"), readonly=True, align=4) seg_l = b.param("segm_expsum_ptr", PtrType(F32, "global"), readonly=True, align=4) _seq_lens = b.param("seq_lens_ptr", PtrType(I32, "global"), readonly=True, align=4) + rt = {f: b.param(f, I32) for f in spec.RUNTIME_KERNARG_FIELDS} + num_qh_p = rt["num_query_heads"] lse = ( b.param( "lse_ptr", @@ -1543,21 +1566,22 @@ def build_unified_attention_reduce_tiled( # segm_output : [total_q, num_qh, num_segments, head_size] # And the final output uses ``(token, head, dim)``. Encoding them # as descriptors makes the per-pass offset math one call instead of - # an `add(mul(qt,...), mul(qh,...))` ladder. - ml_desc_red = TensorDescriptor.naive( + # an `add(mul(qt,...), mul(qh,...))` ladder. The head count is a kernarg, + # so each token stride is too. + ml_desc_red = DynamicTensorDescriptor.create( "segm_ml", - lengths=[1 << 30, NUM_QH, NUM_SEG], coord_names=("token", "head", "seg"), + strides=(b.mul(num_qh_p, b.const_i32(NUM_SEG)), NUM_SEG, 1), ) - seg_acc_desc_red = TensorDescriptor.naive( + seg_acc_desc_red = DynamicTensorDescriptor.create( "segm_output", - lengths=[1 << 30, NUM_QH, NUM_SEG, HD], coord_names=("token", "head", "seg", "dim"), + strides=(b.mul(num_qh_p, b.const_i32(NUM_SEG * HD)), NUM_SEG * HD, HD, 1), ) - out_desc_red = TensorDescriptor.naive( + out_desc_red = DynamicTensorDescriptor.create( "out", - lengths=[1 << 30, NUM_QH, HD], coord_names=("token", "head", "dim"), + strides=(b.mul(num_qh_p, b.const_i32(HD)), HD, 1), ) # Per-segment ml base for this (q_token, q_head); the @@ -1679,7 +1703,7 @@ def build_unified_attention_reduce_tiled( # A NaN sum (NaN query row) keeps NaN, as O does: the same !(l == 0) # test the tiled 2D epilogues use, never l > 0. with b.scf_if(b.cmp_eq(tid, b.const_i32(0))): - lse_index = b.add(b.mul(q_token, b.const_i32(NUM_QH)), q_head) + lse_index = b.add(b.mul(q_token, num_qh_p), q_head) lse_store( b, lse, @@ -1702,6 +1726,9 @@ def supports_tiled_3d_spec( This wrapper reads each of them off the spec and delegates, making the 3D unified builder validatable at pack time. """ + ok, reason = check_head_counts(spec.num_query_heads, spec.num_kv_heads) + if not ok: + return False, reason use_fp8 = spec.kv_storage_dtype is not None return supports_tiled_3d( head_size=spec.head_size, @@ -1762,16 +1789,9 @@ def supports_tiled_reduce_spec( "tiled reduce kernel needs head_size divisible by " f"{_REDUCE_THREADS} (got {spec.head_size})", ) - if ( - spec.num_kv_heads < 1 - or spec.num_query_heads < 1 - or spec.num_query_heads % spec.num_kv_heads != 0 - ): - return ( - False, - "tiled reduce kernel needs a positive num_query_heads divisible by " - f"num_kv_heads (got {spec.num_query_heads} / {spec.num_kv_heads})", - ) + ok, reason = check_head_counts(spec.num_query_heads, spec.num_kv_heads) + if not ok: + return False, f"tiled reduce kernel: {reason}" # ``arch`` passed validate_tiled_attention_arch, so it has a catalog row. target = ArchTarget.from_gfx(arch) lds_bytes = split_kv_reduce_lds_bytes(spec.num_segments) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/attention/test_tiled_support_spec_wrappers.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/attention/test_tiled_support_spec_wrappers.py index 2f950bb88f52..0d628eea54fb 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/attention/test_tiled_support_spec_wrappers.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/attention/test_tiled_support_spec_wrappers.py @@ -10,6 +10,7 @@ import kernels.gfx942.attention_tiled_2d as gfx942_2d import kernels.gfx942.attention_tiled_3d as gfx942_3d import kernels.gfx950.attention_tiled_2d as gfx950_2d +import kernels.gfx950.attention_tiled_2d_fastkv_regp as fastkv import kernels.gfx950.attention_tiled_3d as gfx950_3d _PROBLEM = dict( @@ -89,6 +90,65 @@ def test_every_wrapper_judges_head_size(self): self.assertFalse(accepted) self.assertIn("head_size", reason) + def test_every_wrapper_refuses_bad_head_counts_first(self): + """The name carries the GQA ratio, so a zero or non-dividing head pair is + refused by the wrapper (not a ZeroDivisionError) and by kernel_name().""" + seg = {"num_segments": 16} + cases = ( + (gfx942_2d, "2d", {}), + (gfx950_2d, "2d", {}), + (gfx942_3d, "3d", seg), + (gfx950_3d, "3d", seg), + ) + heads = ( + ((8, 0), "must be > 0"), + ((0, 1), "must be > 0"), + ((8, 3), "divisible"), + ) + for module, dim, extra in cases: + spec_cls = getattr(module, f"UnifiedAttention{dim.upper()}TiledSpec") + wrapper = getattr(module, f"supports_tiled_{dim}_spec") + for (hq, hkv), reason in heads: + with self.subTest(module=module.__name__, hq=hq, hkv=hkv): + problem = {**_PROBLEM, "num_query_heads": hq, "num_kv_heads": hkv} + spec = spec_cls(head_size=128, **problem, **extra) + accepted, why = wrapper(spec) + self.assertFalse(accepted) + self.assertIn(reason, why) + with self.assertRaisesRegex(ValueError, reason): + spec.kernel_name() + + def test_fastkv_register_p_refuses_bad_head_counts(self): + spec = gfx950_2d.UnifiedAttention2DTiledSpec( + head_size=64, **{**_PROBLEM, "num_query_heads": 64, "num_kv_heads": 0} + ) + accepted, why = fastkv.supports_fastkv_register_p_2d_spec(spec) + self.assertFalse(accepted) + self.assertIn("must be > 0", why) + + def test_fastkv_register_p_kwargs_predicate_refuses_gfx942(self): + """supports_tiled_2d admits gfx942, but the fastKV register-P body is + gfx950-only; the kwargs predicate must agree with the spec wrapper.""" + kwargs = { + "head_size": 64, + "block_size": 32, + "dtype": "bf16", + "num_queries_per_kv": 8, + "use_alibi": False, + "use_qq_bias": False, + "use_fp8": False, + "q_dtype": "bf16", + "num_query_heads": 64, + "num_kv_heads": 8, + } + self.assertEqual( + fastkv.supports_fastkv_register_p_2d(**kwargs, arch="gfx950"), + (True, "supported"), + ) + accepted, why = fastkv.supports_fastkv_register_p_2d(**kwargs, arch="gfx942") + self.assertFalse(accepted) + self.assertIn("gfx950 kernel", why) + if __name__ == "__main__": unittest.main() diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/attention/test_unified_lse_dispatch.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/attention/test_unified_lse_dispatch.py index 05ad82a158b2..f51864a80479 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/attention/test_unified_lse_dispatch.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/attention/test_unified_lse_dispatch.py @@ -99,12 +99,13 @@ def test_pinned_split_kv_spec_follows_the_request(arch): # develop's own values (gfx950's split-KV config_key moved with the # additive-bias field); emit_lse must leave them exactly as develop computes them. # Both split-KV config_keys moved again once decode kept K/V cacheable when it -# fits the last-level cache: this shape's 64 MiB of K/V now records -# kv_cache_policy="all", not "stream". +# fits the last-level cache (this shape's 64 MiB of K/V now records +# kv_cache_policy="all", not "stream"), and both pins with the attention ABI's +# move to v2 when the head counts and window became kernargs. _PRE_LSE_UNPINNED_SPEC_HASH = "65416b2a8b9ffa90" _PRE_LSE_PIN = { - "gfx942": ("splitkv_seg16_t1xb_wpenone@4bcc4e4cecb32443", "7fed9a709925a807"), - "gfx950": ("splitkv_seg16_t1xb_wpenone@2e6392907c305942", "2aa486d787146a27"), + "gfx942": ("splitkv_seg16_t1xb_wpenone@97be63b645f95797", "7fed9a709925a807"), + "gfx950": ("splitkv_seg16_t1xb_wpenone@08f6c1555f9431c2", "2aa486d787146a27"), } @@ -126,14 +127,15 @@ def test_pre_lse_cache_entries_still_resolve_with_lse_off(arch): # What a long-lived cache stored for ``_prefill_request`` before the 2D specs # had emit_lse: unpinned spec_hash, narrow pin + hash. Develop's own values # (gfx950's pin moved with the additive-bias field, then with use_qblock_reverse). +# The pinned tuning_ids moved with the attention ABI v2; the spec hashes did not. _PRE_LSE_2D_UNPINNED_SPEC_HASH = "c7dd16a4a52afb9d" _PRE_LSE_2D_PIN = { "gfx942": ( - "narrow_nw1_mw16_t1xb_llvm_wpenone@98141a2e646271d9", + "narrow_nw1_mw16_t1xb_llvm_wpenone@23742b663ef81a19", "738d09721910c8ad", ), "gfx950": ( - "narrow_nw1_mw16_t1xb_llvm_wpenone@7a7cfcc8a5d56e43", + "narrow_nw1_mw16_t1xb_llvm_wpenone@71b379b26143236e", "c97e33ec12b3b1a4", ), } diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/golden/attention_strided_kv_ir_sha256.json b/dnn-providers/hip-kernel-provider/rocke/library/tests/golden/attention_strided_kv_ir_sha256.json index a896280896bf..e46608f6623b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/golden/attention_strided_kv_ir_sha256.json +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/golden/attention_strided_kv_ir_sha256.json @@ -1,50 +1,50 @@ { "llvm20": { - "0": "1626048eeaface4bf6eb73c5bc0c81a653da20fc18ca502898a55e6111d13fb2", - "1": "a9d06552869ca1485cb3a7b8a362d2a448fb8d55220f222d50d4a54ded0a2488", - "2": "8c0cb312ca581170ce8e387a94a1b77ee52b4c7fc233aa5b15f65f0c707eea8b", - "3": "272f3227c873b525dc8e48c4f024f7885de375178e20f4c4d6686b4724d82c0b", - "4": "b25f6bfc84f5db8903883a85671f13b109256697e33c67877686b4d90ac07606", - "5": "bde0dfb3994fb2a89081fa05809323059111a5e7d4df55ed8859ec2f6b0cd930", - "6": "db68f7a59b6a018e310c3d86d065432fbc4a7201f3d914114dec2d58d3200a19", - "7": "e15ef2766af1f7f39f1e2e1e6f1602e17ed832ed960fee0fc87c54447742c45f", - "8": "15c5cc154e40e4e5f099d1bf6e476e283a70c450824db3ebb9df927b2f7aa2e0", - "9": "7866c9eb6d414e0edf3678a477c387e2742057eed6a96616544f29057d609d0a", - "10": "05d6f2c670f9c66f59974f7f2715eed844799ac869e2765d3d1f691bc417d2da", - "11": "68530d1c8c9c9a12300743809054235d88e2a1698f811bbd86a531eb47113d40", - "12": "fa6ae13282f4e5748eae40227d1355529e793d2306f210136a498b6621fcff96", - "13": "d7982e56673824863076878c3461209f8557cfa8507b102071eae22d712b6414" + "0": "8cfb54512441a845a2ae325065291e39e5367be96f320596e8622d7e308ec610", + "1": "b1cee01d468e92ecb5b8571323ffd897bff01c7d2e42dc8ba50814387f908a63", + "2": "a80d03691e079aacfb8653f7054a836d2ecd57256285274fa51c52a8a398c727", + "3": "e9f5684da874f4c1bedfccc3a38750263a8d9364803d660871cba0b1176be7f4", + "4": "67c89f7a4550323dd055e30f2fdf5c12a7708b41f2af7e9cfdf1dc5faaa090e7", + "5": "77d12b3c9210688cfa94e005484498b7c285e7c173f48b46d2b87cbf9a3c3ff2", + "6": "23c2881b225837148b997823910e598ad844d781441a1f122a5d370b4b97900c", + "7": "966f73aef9172bbe60348f970553576d609299131e029e6842e1f4dc49c96da1", + "8": "c5f4baa4b7f0237ce60bf63f6694fcca24bdd440c0fbc34a1dc0a6c8947b791c", + "9": "977a23a9b768a42645ef5dd393278c4d577de5f3cb0c5b7d847260c1bf0b4462", + "10": "1238b48be61bb4a6c83e3581ed269a5de59b717901772d81d17577988a788e9e", + "11": "c996154b2f38e896037f4e14e8574c7c85d938d297ec93068407f0cca78cbde8", + "12": "6adc12acb6dd75f05927ef1e4f6683b85c780abae25d76588bf57ae25f8dfd88", + "13": "5266e99d4e82df6558795288cbdaef86099a84229f002945d227977834b89ffe" }, "llvm22": { - "0": "bcef03e3b5555c8bbaaa03188901c1dfc8bee0cdfa80da32026367e82e7c727b", - "1": "9becf0d4f1b8eedc388e818afb01a40607a27efe2d415772ba1f6352c2ba35d7", - "2": "7fc95348ccfd43e6afef1b5bc83b0e17b55c0879d5d8c752d40e6f631d0b9c77", - "3": "c88f5e1109c70e0c4f132b05578c6abf440d04bccb52bd774b492d67d3acc9bb", - "4": "3e5522113635114b994bb40cf19bc51a417765ac827cbfa879acb45cb0e62796", - "5": "3c03d60f49863bc856426e1431cb98e8e682da4b6818f1df320536fab12b1314", - "6": "2437aa43c1700bf2af3731b08d30869d68cf9110b2fbed6114fae27b78a4e0f0", - "7": "b1503cc02eed262c4d08b02d3802a16f3ee74ff0700f756e79e9a1c17580ff71", - "8": "8942a331f8dd8285230d4a53c02bca59ee5158284edaf12d7cbbd50900a5bfe5", - "9": "43cbf9fc03696e3fcf0a132e31a769d92cd48267eaea3526e2cc7c46c35225da", - "10": "35992a4d15f09b54bd668d4b21556a2c871fa8c6f8aa39a89169736e303476ea", - "11": "ec52f6ee37cff049c8b81eb579b71c32be965bb4730fc7b30bd477196499727f", - "12": "002dfd5d5ce7d67b6992178da77d629eb26e80c5343c95a16a0e2f89462cf752", - "13": "f71e2da33c81aacfd3a6690c0fa2c540aa3a83586e0aa8ed0f1ed638023677db" + "0": "14ccc0e86cb9a96163a8c32812cd718dc454f726dacced64fa3cf07db5c70c90", + "1": "62086a1cb544ba32772dc112e252813ffff1961bc2db23401bf0003bce718b73", + "2": "f2eb0fd45eb3693671f6d11bdce0eac92295e784e74efffe21351150e2807039", + "3": "3a8fe959ee0737f990adf184e812dfe983b24e0f50cfdbcad667aeda89eade72", + "4": "2f21d041985082346f34105cef0ed4b58df655eed4c600a176deaaec0350cfc9", + "5": "d2770b39913bb089531c1db4d9fc31db4114053922f0bdbca3b34cdcad122a69", + "6": "b4720021fe24f67cfdd7b6854a25f4b371cd5c086d7d804e4a1a451d8f885c82", + "7": "281cf49847fec9bf2f0133d0b0f83c9ff72e99e3f9bfd846d2d056289604609b", + "8": "8e1697deca33346efb263397db405b14b835a617e0fb8a9d6e5dd19958d767e4", + "9": "5f423f4b85911405a7548c7641e955fcdb485e040872744970c16c92326ae3e1", + "10": "0c4719c51595a01254f8a7c7c26bcf7cf10f7164fe54dc0c697fe36b938bc1b9", + "11": "c8bdc92a7fafee7c1187375671ea8caab00edceaf7b5f82c844d8343767f22b3", + "12": "b543ef77d6fd6bcdeab9b00f9c7dd8d9823777987f2e9e09b79195f3d230be37", + "13": "a2a213edfcbe00373fe8d3962f4b20a7216c9489a0c5c039536d63483f5bae57" }, "llvm23": { - "0": "67e7f5bd0d8861dc16e6cff53316d780d40f6232716a5a3ea25f06055ac48442", - "1": "a0b5229967c3892d492cbacb9db2d15f06ce919b1f15c8de8d298e0d74306e46", - "2": "4cded6921b9951b733d1dbfdfe93015aa7b78c5bd5def3dfbe503175949af968", - "3": "393df7f6bb892e49b3c3031ff696d11addb1da00ad2088aa881ea37074c7db77", - "4": "db62e65f3aa74588eb95d2bf69935aa6331d0a57d42996bbe81c7510e146f119", - "5": "ee8248f21f5631e005750601f275daf2fd57d87f102a241c2fee76caa4567d33", - "6": "92c9560cfd6ae383a059972d0f4863e3699b56beba16aab1736d06bb41946143", - "7": "1b3cb6e2c817fce85c1bd189f7a7a01eb7e089c37ef1824cabdfbe5bd6915b47", - "8": "4b0b553b289d0de60dd9401f61f97d2f36cb201b6343437e28e5fbdc370cf529", - "9": "cb2426e5ff5ce00a3094ea4217df160031006f22cfe058568e4c959f1ecff086", - "10": "400f75d835f4bb4d37dfe0039c609deb5e172c77e5a83b87b89071ed3b0017bf", - "11": "0e441b0ae432165d00da483191b9470dcfc043c2bacd386b86fe6bef8f05266f", - "12": "03eb993466369d2ca4aabaff7880f23138bf08a0f71154159a0c9e4627e17df3", - "13": "74d14799531a8e079e822a46af3a22f1875a85a46c6c561ef26427fd44ffd748" + "0": "7ce6e33c41e2752f1863c4ef79eef519c4748cc9f1c3035cab5cab030f386d55", + "1": "38fa323beea8da252380810dbbe76686f3cff17bc15aca23c32a73c11f6a70e4", + "2": "5e39245a8752ce0229c9229aba2b11fc8cb07241e93ebaafa051cae24aab4514", + "3": "76bca1175f651389730a2856cf8bb9cf9d79998445425cde57ac37700dd1d2ef", + "4": "0ea423ae0c4cdd6d4b72ed754b1a18dab37fb3f6b9fbad492b1bcd036529c823", + "5": "29304ac08dddbce980440041340d863e3996b72fa786fecd5f739ca8c14bd150", + "6": "b3add7e87f7a74a750718e45156d80bc9a5725c33a8a25fadc55b71bb600af89", + "7": "94ea80d639a380a3cf0069ff8679e6f197fa1c40c9a4885413d747341964ef9c", + "8": "d8a78f5237a1c466494c27808c41590153f9ae47d899a5d6ad240510e680d89f", + "9": "67bfeb3341425741eb76f57e12f56c47a87bbdb73b1e5641c4a4f256f2647591", + "10": "59ba76e31fd186ba00c06d841b8a038c8941a8419e56b8fa46daf2ea9eb61028", + "11": "a30d1148dddca335071fee50a5cedd26bb5323693c278b14d22580d6cc5df811", + "12": "0ec51a3f92220ca3dad8979ef3dbe976f89ab95e28de5a178eaee7477661b8d4", + "13": "9bb3761cb7e928a0efc7789cda47a011260003ba29f7db4cdde89af0a0bda125" } } diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx942_attention_tiled_2d_stress_emit.c b/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx942_attention_tiled_2d_stress_emit.c index 50212bbfbd5e..58d24f5e9670 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx942_attention_tiled_2d_stress_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx942_attention_tiled_2d_stress_emit.c @@ -231,6 +231,8 @@ static int make_spec(int idx, rocke_attention_tiled_2d_spec_t* s) s->sliding_window = 0; s->has_softcap = false; break; + /* 32/5 does not divide: both engines refuse it (check_head_counts), so it + * is a both-reject parity case rather than an emitted kernel. */ case 21: s->head_size = 64; s->block_size = 32; diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx942_attention_tiled_2d_stress_emit.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx942_attention_tiled_2d_stress_emit.py index ae69cac3b1a7..bb6948fa4521 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx942_attention_tiled_2d_stress_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx942_attention_tiled_2d_stress_emit.py @@ -228,6 +228,8 @@ sliding_window=0, has_softcap=False, ), + # 32/5 does not divide: both engines refuse it (check_head_counts), so it + # is a both-reject parity case rather than an emitted kernel. 21: dict( head_size=64, block_size=32, diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_emit.c b/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_emit.c index 06a858db0096..82705f94e692 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_emit.c @@ -1123,6 +1123,87 @@ static int make_spec(int idx, rocke_attention_tiled_2d_spec_t* s) s->use_alibi = true; s->emit_lse = true; break; + /* cfg68: bf16 D64/b16 GQA-4 multi-batch: the shipping spec, whose one K/V copy per + * tile (32 tokens) spans two pages, so the bf16 loaders keep the paged-KV + * descriptor. Every other bf16/fp16 case takes the per-page copy offsets. */ + case 68: + s->head_size = 64; + s->block_size = 16; + s->num_query_heads = 32; + s->num_kv_heads = 8; + s->dtype = "bf16"; + s->use_sinks = false; + s->sliding_window = 0; + s->has_softcap = false; + s->num_seqs = 2; + s->num_warps = 4; + s->has_waves_per_eu = true; + s->waves_per_eu = 2; + s->has_tile_size = true; + s->tile_size = 32; + s->block_m_per_warp = 32; + s->use_mfma_32x32 = true; + s->use_transposed_qk_32x32 = true; + s->use_transposed_scalar_state = true; + s->use_transposed_mask_once = true; + s->use_transposed_half_local_pv = true; + s->use_mfma32_skip_legacy_qreg = true; + s->use_transposed_mask_limit = true; + break; + /* cfg69/70: page copies at the 16-page prefetch bound (page ids carried) + * and past it (page ids read in place). */ + case 69: + s->head_size = 64; + s->block_size = 16; + s->num_query_heads = 32; + s->num_kv_heads = 8; + s->dtype = "bf16"; + s->use_sinks = false; + s->sliding_window = 0; + s->has_softcap = false; + s->num_seqs = 2; + s->has_tile_size = true; + s->tile_size = 256; + s->block_m_per_warp = 32; + s->use_mfma_32x32 = true; + s->use_transposed_qk_32x32 = true; + break; + case 70: + s->head_size = 64; + s->block_size = 16; + s->num_query_heads = 32; + s->num_kv_heads = 8; + s->dtype = "bf16"; + s->use_sinks = false; + s->sliding_window = 0; + s->has_softcap = false; + s->num_seqs = 2; + s->has_tile_size = true; + s->tile_size = 288; + s->block_m_per_warp = 32; + s->use_mfma_32x32 = true; + s->use_transposed_qk_32x32 = true; + break; + /* cfg71: bf16 D192/b16 one warp, the spec dispatch picks for short D192 + * prefill. One copy is 512 halves, 2 2/3 rows, so it is not a whole number + * of tokens and the bf16 loaders must keep the paged-KV descriptor. */ + case 71: + s->head_size = 192; + s->block_size = 16; + s->num_query_heads = 16; + s->num_kv_heads = 2; + s->dtype = "bf16"; + s->use_sinks = false; + s->sliding_window = 0; + s->has_softcap = false; + s->num_seqs = 4; + s->has_waves_per_eu = true; + s->waves_per_eu = 2; + s->use_register_pv = true; + s->has_tile_size = true; + s->tile_size = 32; + break; + default: return -1; } diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_emit.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_emit.py index 1a86134920ca..da8a3ea2608e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_emit.py @@ -1071,6 +1071,80 @@ num_seqs=2, use_qblock_reverse=True, ), + # cfg68: bf16 D64/b16 GQA-4 multi-batch: the shipping spec, whose one K/V copy per + # tile (32 tokens) spans two pages, so the bf16 loaders keep the paged-KV + # descriptor. Every other bf16/fp16 case takes the per-page copy offsets. + 68: dict( + head_size=64, + block_size=16, + num_query_heads=32, + num_kv_heads=8, + dtype="bf16", + use_sinks=False, + sliding_window=0, + has_softcap=False, + num_seqs=2, + num_warps=4, + waves_per_eu=2, + tile_size=32, + block_m_per_warp=32, + use_mfma_32x32=True, + use_transposed_qk_32x32=True, + use_transposed_scalar_state=True, + use_transposed_mask_once=True, + use_transposed_half_local_pv=True, + use_mfma32_skip_legacy_qreg=True, + use_transposed_mask_limit=True, + ), + # cfg69/70: page copies at the 16-page prefetch bound (page ids carried) + # and past it (page ids read in place). + 69: dict( + head_size=64, + block_size=16, + num_query_heads=32, + num_kv_heads=8, + dtype="bf16", + use_sinks=False, + sliding_window=0, + has_softcap=False, + num_seqs=2, + tile_size=256, + block_m_per_warp=32, + use_mfma_32x32=True, + use_transposed_qk_32x32=True, + ), + 70: dict( + head_size=64, + block_size=16, + num_query_heads=32, + num_kv_heads=8, + dtype="bf16", + use_sinks=False, + sliding_window=0, + has_softcap=False, + num_seqs=2, + tile_size=288, + block_m_per_warp=32, + use_mfma_32x32=True, + use_transposed_qk_32x32=True, + ), + # cfg71: bf16 D192/b16 one warp, the spec dispatch picks for short D192 + # prefill. One copy is 512 halves, 2 2/3 rows, so it is not a whole number + # of tokens and the bf16 loaders must keep the paged-KV descriptor. + 71: dict( + head_size=192, + block_size=16, + num_query_heads=16, + num_kv_heads=2, + dtype="bf16", + use_sinks=False, + sliding_window=0, + has_softcap=False, + num_seqs=4, + waves_per_eu=2, + use_register_pv=True, + tile_size=32, + ), } # --- idx64-67: emit_lse (trailing lse_ptr + FP32 LSE store), one per epilogue diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_fastkv_regp_emit.c b/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_fastkv_regp_emit.c index 5a3e7fadf24d..f8b8932e31fc 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_fastkv_regp_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_fastkv_regp_emit.c @@ -80,6 +80,16 @@ static int make_spec(int idx, rocke_attention_tiled_2d_spec_t* s) * wrapper's kernel name). */ s->emit_lse = true; break; + /* The pack predicate admits windowed, sinks and i64 specs; pin each body. */ + case 7: + s->sliding_window = 256; + break; + case 8: + s->use_sinks = true; + break; + case 9: + s->use_i64_kv_addr = true; + break; default: return -1; } diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_fastkv_regp_emit.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_fastkv_regp_emit.py index 7db60271f4d8..652faea6008f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_fastkv_regp_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/parity/gfx950_attention_tiled_2d_fastkv_regp_emit.py @@ -50,6 +50,10 @@ # LSE output through the wrapper: trailing lse_ptr and the wrapper's own # kernel name ("..._lse_fastkv_regp"). 6: dict(_BASE, emit_lse=True), + # The pack predicate admits windowed, sinks and i64 specs; pin each body. + 7: dict(_BASE, sliding_window=256), + 8: dict(_BASE, use_sinks=True), + 9: dict(_BASE, use_i64_kv_addr=True), } diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_builds.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_builds.py index a8dc1bf39d05..eff3260149c6 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_builds.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_builds.py @@ -143,7 +143,7 @@ def _compile_or_skip(kernel, *, arch: str): }, # gfx942 fp16 D128 tiled-2d (shipped default geometry): 2 waves/EU caps VGPR at # 256; the kernel wants 354, so it spills ~40 B to hold 2-wave occupancy. - "rocke_uattn2d_tiled_d128_b64_h32kv8_fp16_w4_wpe2_mw32_mfma32x8_stqk_s1_mask1_hoist_mlim_kvcpall_cfvst_ksring_rd2": { + "rocke_uattn2d_tiled_d128_b64_g4_fp16_w4_wpe2_mw32_mfma32x8_stqk_s1_mask1_hoist_mlim_kvcpall_cfvst_ksring_rd2": { "max_scratch_bytes": 64, "min_waves_per_simd": 2, }, @@ -1167,7 +1167,10 @@ def test_unified_attention_additive_bias_abi_matches_signature(self): build_unified_attention_2d_tiled, build_unified_attention_3d_tiled, ) - from kernels.common.attention_unified import _3d_signature, _attn_signature + from kernels.common.attention_unified import ( + _tiled_2d_signature, + _tiled_3d_signatures, + ) common = dict( head_size=64, @@ -1184,11 +1187,8 @@ def test_unified_attention_additive_bias_abi_matches_signature(self): k = build_unified_attention_2d_tiled( UnifiedAttention2DTiledSpec(**common, use_additive_bias=use_ab) ) - sig = _attn_signature( - "bf16", - include_bt_stride=True, - include_qq_bias_stride=True, - include_additive_bias=use_ab, + sig = _tiled_2d_signature( + "gfx950", "bf16", include_additive_bias=use_ab ) self.assertEqual([p.name for p in k.params], [a["name"] for a in sig]) self.assertEqual( @@ -1200,7 +1200,9 @@ def test_unified_attention_additive_bias_abi_matches_signature(self): **common, num_segments=4, use_additive_bias=use_ab ) ) - sig = _3d_signature("bf16", include_additive_bias=use_ab) + sig, _ = _tiled_3d_signatures( + "gfx950", "bf16", include_additive_bias=use_ab + ) self.assertEqual([p.name for p in seg.params], [a["name"] for a in sig]) def test_unified_attention_additive_bias_must_be_bool(self): diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_dense_kv_split_combine_abi.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_dense_kv_split_combine_abi.py new file mode 100644 index 000000000000..5f3bae4d121b --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_dense_kv_split_combine_abi.py @@ -0,0 +1,96 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""The dense segment-combine launch packs every argument the tiled reduce declares. + +``launch_segment_combine`` launches the tiled 3D reduce kernel. That kernel reads +``num_query_heads`` as a kernarg (``RUNTIME_KERNARG_FIELDS``), so a launcher built +from the pointer-only reduce signature leaves the head count, and with it the +reduce's token strides, undefined. This captures the launcher's signature and +values on the host and checks them against the reduce kernel's params. + +``test_combine_abi_matches_reduce_params`` needs neither torch nor a GPU, so it +runs in every CPU lane; the launch test runs where torch is installed. +""" + +from __future__ import annotations + +import pytest +from kernels.common import attention_dense_kv_split as kv_split +from kernels.gfx942.attention_dense import Gfx942AttentionDenseSpec +from kernels.gfx950.attention_dense import Gfx950AttentionDenseSpec + +_ARCHES = [("gfx942", Gfx942AttentionDenseSpec), ("gfx950", Gfx950AttentionDenseSpec)] + + +def _dense_spec(spec_cls): + return spec_cls( + batch=2, + seqlen_q=256, + seqlen_kv=2048, + num_query_heads=16, + num_kv_heads=2, + head_size=128, + causal=True, + dtype="bf16", + kv_segments=4, + ) + + +@pytest.mark.parametrize("arch, spec_cls", _ARCHES) +def test_combine_abi_matches_reduce_params(arch, spec_cls): + """The signature the launcher is built from names the reduce kernel's params + in order, and every runtime scalar among them gets the combine's value.""" + spec = _dense_spec(spec_cls) + red, signature, scalars = kv_split.combine_launch_abi(spec, arch) + _, build = kv_split._combine_impl(arch) + params = [p.name for p in build(red, arch=arch).params] + assert params == [a["name"] for a in signature] + assert scalars == {"num_query_heads": spec.num_query_heads} + pointers = {"output_ptr", "seq_lens_ptr"} | set(kv_split.SEGMENT_PARAMS) + assert set(params) == pointers | set(scalars) + + +class _Art: + hsaco = b"" + + def __init__(self, kernel): + self.kernel_name = kernel.name + + +@pytest.mark.parametrize("arch, spec_cls", _ARCHES) +def test_combine_launch_matches_reduce_params(monkeypatch, arch, spec_cls): + torch = pytest.importorskip("torch") + import rocke.helpers.compile as compile_mod + import rocke.runtime as runtime_mod + + built = {} + + def fake_compile(kernel, **_): + built["kernel"] = kernel + return _Art(kernel) + + class FakeLauncher: + def __init__(self, *, hsaco, kernel_name, signature): + built["signature"] = signature + + def __call__(self, vals, *, config): + built["vals"] = vals + + monkeypatch.setattr(compile_mod, "compile_kernel", fake_compile) + monkeypatch.setattr(runtime_mod, "KernelLauncher", FakeLauncher) + monkeypatch.setattr(kv_split, "_COMBINE_LAUNCHER_CACHE", {}) + + spec = _dense_spec(spec_cls) + workspace = kv_split.allocate_segment_workspace(spec, "cpu") + out = torch.empty( + (spec.batch, spec.seqlen_q, spec.num_query_heads, spec.head_size), + dtype=torch.bfloat16, + ) + kv_split.launch_segment_combine(spec, arch, out, workspace, stream=0) + + params = [p.name for p in built["kernel"].params] + assert params == [a["name"] for a in built["signature"]] + assert "num_query_heads" in params + assert set(params) <= set(built["vals"]) + assert built["vals"]["num_query_heads"] == spec.num_query_heads diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_spec_predicate_parity.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_spec_predicate_parity.py index 01b218e209d5..675f2c82b899 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_spec_predicate_parity.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_spec_predicate_parity.py @@ -262,7 +262,7 @@ def test_reduce_predicate_admits_the_reduce_spec_of_every_admitted_3d_problem( (dict(dtype="fp32"), "fp16/bf16"), (dict(head_size=96), "divisible by 64"), (dict(num_query_heads=30, num_kv_heads=8), "divisible by num_kv_heads"), - (dict(num_query_heads=-8, num_kv_heads=8), "positive num_query_heads"), + (dict(num_query_heads=-8, num_kv_heads=8), "must be > 0"), (dict(num_segments=40961), "over gfx950's 163840 B LDS (at most 40960)"), ], ) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_3d_running_max.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_3d_running_max.py index 5c678660c6bb..d2bdbe6ad4af 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_3d_running_max.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_3d_running_max.py @@ -206,6 +206,7 @@ def pages(x): "num_seqs": 1, "block_table_stride": blocks, "qq_bias_stride_0": 0, + **au._runtime_kernarg_values(problem), } reduce_values = { "output_ptr": out, @@ -213,6 +214,7 @@ def pages(x): "segm_max_ptr": segm_max, "segm_expsum_ptr": segm_expsum, "seq_lens_ptr": seq_lens, + **au._runtime_kernarg_values(problem), } # The launch fences the stream, so ``out`` is complete when it returns. au._launch_3d_pipeline( diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_cache_key.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_cache_key.py index f9108bf1d9bd..749cc99b737c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_cache_key.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_cache_key.py @@ -19,7 +19,8 @@ different keys; shapes that resolve to the same outputs share a key and lower to byte-identical IR. * Launch geometry: the grid caches still separate batch counts. - * ``_binary_search_iters`` mirrors every spec class's ``binary_search_iters``. + * ``_binary_search_iters`` mirrors every spec class that bakes the search; + gfx942 and gfx950 search at runtime, so their batch sizes share a binary. No GPU required. """ @@ -82,11 +83,23 @@ def routing_env(monkeypatch): def _norm_spec(spec): """Spec identity as the emitter sees it: ``num_seqs`` only reaches IR - through ``binary_search_iters``, which the 3D decode grid does not emit.""" - fields = dataclasses.asdict(spec) + through ``binary_search_iters``, which the 3D decode grid does not emit and + a runtime-search body (``RUNTIME_SEARCH_ITERS``) does not bake.""" + # The spec fields are scalars and strings, so a shallow read is + # ``dataclasses.asdict`` without its recursive copy. + fields = {f.name: getattr(spec, f.name) for f in dataclasses.fields(spec)} fields.pop("num_seqs") - if not getattr(spec, "use_decode_grid", False): + bakes_search = not getattr(type(spec), "RUNTIME_SEARCH_ITERS", False) + if bakes_search and not getattr(spec, "use_decode_grid", False): fields["binary_search_iters"] = spec.binary_search_iters + # A kernarg field reaches IR only through the structure it selects. + runtime = getattr(type(spec), "RUNTIME_KERNARG_FIELDS", ()) + if "sliding_window" in runtime: + fields["sliding_window"] = spec.sliding_window > 0 + if "num_query_heads" in runtime: + fields.pop("num_query_heads") + fields.pop("num_kv_heads") + fields["num_queries_per_kv"] = spec.num_queries_per_kv return fields @@ -100,12 +113,17 @@ def _built_2d(problem, arch): def _built_3d(problem, arch): + reduce_cls = au._tiled_3d_impl(arch)[1] + heads = ( + (problem.num_queries_per_kv,) + if "num_query_heads" in getattr(reduce_cls, "RUNTIME_KERNARG_FIELDS", ()) + else (problem.num_query_heads, problem.num_kv_heads) + ) return dict( spec=_norm_spec(au._tiled_3d_spec_from_problem(problem, arch)), reduce=( problem.head_size, - problem.num_query_heads, - problem.num_kv_heads, + *heads, problem.dtype, au._num_segments(problem, arch), au._select_3d_waves_per_eu(problem, arch), @@ -122,6 +140,11 @@ def _built_3d(problem, arch): # the selectors read: ns 1 vs >=2 and the log2 search buckets; q at 1, 64, 128, # 256, 512, 768, 1024, 1536, 2048; k across 16 * T for T in {32, 64}. _NUM_SEQS = (1, 2, 3, 4) +# A 2D body that searches cu_q at runtime reads num_seqs only as the +# single-batch cohort (1 vs >= 2), so its sweep stops at 2. The 3D split-KV +# selectors size segments from the batch, and gfx1250 bakes the search, so +# both keep every value. +_NUM_SEQS_RUNTIME_SEARCH_2D = (1, 2) _SEQLEN_Q = (1, 2, 64, 65, 256, 257, 512, 513, 768, 769, 1024, 1025, 1536, 2048) _SEQLEN_K = ("q", 512, 513, 1024, 1025) @@ -129,6 +152,7 @@ def _built_3d(problem, arch): _FEATURES = ( (0, False, None), (128, False, None), + (512, False, None), (0, True, None), (0, False, "softcap"), (0, False, "alibi"), @@ -142,7 +166,9 @@ def _static_grid(arch): (False, True), (64, 128, 256), (16, 32, 64), - ((8, 8), (32, 8), (64, 8)), + # Two pairs share each of ratios 1 and 8, so a body that reads the + # head counts as kernargs is checked against an under-key at one ratio. + ((8, 8), (16, 16), (32, 8), (32, 4), (64, 8)), _FEATURES, ): yield dict( @@ -161,9 +187,14 @@ def _static_grid(arch): ) -def _sweep(arch): +def _sweep(arch, path): + num_seqs = ( + _NUM_SEQS_RUNTIME_SEARCH_2D + if path == "2d" and au._searches_at_runtime(arch, "2d") + else _NUM_SEQS + ) for static in _static_grid(arch): - for ns, q, k in itertools.product(_NUM_SEQS, _SEQLEN_Q, _SEQLEN_K): + for ns, q, k in itertools.product(num_seqs, _SEQLEN_Q, _SEQLEN_K): k = q if k == "q" else k if k < q: continue @@ -172,9 +203,11 @@ def _sweep(arch): ) -# Env states to pool per arch. Routing knobs are read on every launch and can -# flip mid-process; pooling them catches a selector whose output only changes -# when a knob flips. +# Env states to pool per arch on the 2D path. Routing knobs are read on every +# launch and can flip mid-process; pooling them catches a selector whose output +# only changes when a knob flips. These knobs steer 2D routing only: on the 3D +# path every flipped state reproduces the default sweep's keys and kernels, so +# 3D sweeps once. _ENV_STATES = { "gfx942": ( {}, @@ -216,10 +249,11 @@ def test_key_matches_build_identity(routing_env, monkeypatch, arch, path): first = {} key_of_build = {} checked = 0 - for env in _ENV_STATES.get(arch, ({},)): + env_states = _ENV_STATES.get(arch, ({},)) if path == "2d" else ({},) + for env in env_states: for var, val in env.items(): monkeypatch.setenv(var, val) - for problem in _sweep(arch): + for problem in _sweep(arch, path): if not supported(problem, arch)[0]: continue try: @@ -275,7 +309,9 @@ def _assert_split(key_fn, built_fn, a, b, field, arch): assert key_fn(a, arch) != key_fn(b, arch) -@pytest.mark.parametrize("arch", ("gfx942", "gfx950", "gfx1250")) +# gfx942 and gfx950 search at runtime, so their buckets share a binary +# (_SHARED_PAIRS). +@pytest.mark.parametrize("arch", ("gfx1250",)) def test_search_bucket_edge_splits_2d_key(routing_env, arch): # num_seqs 3 and 4 fall in different log2 search buckets (2 vs 3 iterations). # d64 GQA-8 (+ fp8 K/V on gfx1250): a geometry every arch's tiled 2D supports. @@ -296,25 +332,6 @@ def test_search_bucket_edge_splits_2d_key(routing_env, arch): assert _lower_2d(a, arch, "llvm22") != _lower_2d(b, arch, "llvm22") -@pytest.mark.parametrize("arch", ("gfx942", "gfx950")) -def test_search_bucket_edge_splits_3d_key(routing_env, arch): - # Two query tokens per sequence, so the launch takes the searching prefill - # grid rather than the decode grid. - base = dict( - num_query_heads=32, - num_kv_heads=8, - head_size=128, - block_size=16, - max_seqlen_q=2, - max_seqlen_k=8192, - dtype="bf16", - ) - a = P(total_q=6, num_seqs=3, **base) - b = P(total_q=8, num_seqs=4, **base) - assert not au._use_decode_grid(a, arch) - _assert_split(au._tiled_3d_cache_key, _built_3d, a, b, "binary_search_iters", arch) - - def test_fp8_mfma_qk_kv_threshold_splits_key(routing_env): # T = 2 * block_size = 32 -> the K-in-LDS fp8 path turns off above # max_seqlen_k = 16 * T = 512. Nothing else in the key reads max_seqlen_k. @@ -427,6 +444,45 @@ def _lower_3d(problem, arch, flavor): dtype="bf16", ) +_D256_4WARP = dict( + num_query_heads=16, + num_kv_heads=2, + head_size=256, + block_size=16, + max_seqlen_q=4096, + max_seqlen_k=4096, + dtype="bf16", +) +_D128_FOLD = dict( + num_query_heads=32, + num_kv_heads=8, + head_size=128, + block_size=16, + max_seqlen_q=8192, + max_seqlen_k=8192, + sliding_window=4096, + dtype="bf16", +) +_D128_SB_FP16 = dict( + num_query_heads=32, + num_kv_heads=8, + head_size=128, + block_size=16, + max_seqlen_q=4096, + max_seqlen_k=4096, + dtype="fp16", +) +# Two query tokens per sequence: the 3D searching grid, not the decode grid. +_D128_TWO_TOKENS = dict( + num_query_heads=32, + num_kv_heads=8, + head_size=128, + block_size=16, + max_seqlen_q=2, + max_seqlen_k=8192, + dtype="bf16", +) + # (id, arch, path, problem_a, problem_b). Each pair resolves to the same # selector outputs, so the pair must share one binary. _SHARED_PAIRS = [ @@ -476,6 +532,261 @@ def _lower_3d(problem, arch, flavor): ) for arch in ("gfx942", "gfx950") ), + # gfx950 reads the search trip count from num_seqs at runtime too. + ( + "gfx950-2d-ns3-vs-ns9", + "gfx950", + "2d", + P( + total_q=3 * 1024, + num_seqs=3, + max_seqlen_q=1024, + max_seqlen_k=1024, + **_D64_GQA8, + ), + P( + total_q=9 * 1024, + num_seqs=9, + max_seqlen_q=1024, + max_seqlen_k=1024, + **_D64_GQA8, + ), + ), + ( + "gfx950-3d-prefill-ns3-vs-ns9", + "gfx950", + "3d", + P(total_q=6, num_seqs=3, **_D128_TWO_TOKENS), + P(total_q=18, num_seqs=9, **_D128_TWO_TOKENS), + ), + # gfx950 reads the window size as a kernarg, so window sizes share a binary. + ( + "gfx950-2d-sw256-vs-sw1024", + "gfx950", + "2d", + P( + total_q=2048, + num_seqs=2, + max_seqlen_q=1024, + max_seqlen_k=1024, + sliding_window=256, + **_D64_GQA8, + ), + P( + total_q=2048, + num_seqs=2, + max_seqlen_q=1024, + max_seqlen_k=1024, + sliding_window=1024, + **_D64_GQA8, + ), + ), + # The single-batch fp16 d128 window cohort picks its own KV tile. + ( + "gfx950-2d-single-batch-sw256-vs-sw1024", + "gfx950", + "2d", + P(total_q=4096, num_seqs=1, sliding_window=256, **_D128_SB_FP16), + P(total_q=4096, num_seqs=1, sliding_window=1024, **_D128_SB_FP16), + ), + ( + "gfx950-3d-decode-sw256-vs-sw4096", + "gfx950", + "3d", + P(total_q=4, num_seqs=4, sliding_window=256, **_D128_DECODE), + P(total_q=4, num_seqs=4, sliding_window=4096, **_D128_DECODE), + ), + # gfx950 reads the head counts as kernargs, so head pairs at one GQA ratio + # share a binary. 32/4 and 16/2 stay off the 64/8 fast paged-KV descriptor. + ( + "gfx950-2d-h32kv4-vs-h16kv2", + "gfx950", + "2d", + P( + total_q=2048, + num_seqs=2, + max_seqlen_q=1024, + max_seqlen_k=1024, + **dict(_D64_GQA8, num_query_heads=32, num_kv_heads=4), + ), + P( + total_q=2048, + num_seqs=2, + max_seqlen_q=1024, + max_seqlen_k=1024, + **dict(_D64_GQA8, num_query_heads=16, num_kv_heads=2), + ), + ), + ( + "gfx950-2d-single-batch-h32kv8-vs-h64kv16", + "gfx950", + "2d", + P(total_q=4096, num_seqs=1, **_D128_SB_FP16), + P( + total_q=4096, + num_seqs=1, + **dict(_D128_SB_FP16, num_query_heads=64, num_kv_heads=16), + ), + ), + ( + "gfx950-2d-d256-h16kv2-vs-h32kv4", + "gfx950", + "2d", + P(total_q=2 * 4096, num_seqs=2, **_D256_4WARP), + P( + total_q=2 * 4096, + num_seqs=2, + **dict(_D256_4WARP, num_query_heads=32, num_kv_heads=4), + ), + ), + ( + "gfx950-3d-prefill-h32kv8-vs-h64kv16", + "gfx950", + "3d", + P(total_q=6, num_seqs=3, **_D128_TWO_TOKENS), + P( + total_q=6, + num_seqs=3, + **dict(_D128_TWO_TOKENS, num_query_heads=64, num_kv_heads=16), + ), + ), + ( + "gfx950-3d-decode-h32kv8-vs-h64kv16", + "gfx950", + "3d", + P(total_q=31, num_seqs=31, **_D128_DECODE), + P( + total_q=31, + num_seqs=31, + **dict(_D128_DECODE, num_query_heads=64, num_kv_heads=16), + ), + ), + # gfx942 reads the search trip count from num_seqs at runtime, so batch + # sizes several log2 buckets apart share a binary on every builder route. + ( + "gfx942-2d-ns3-vs-ns9", + "gfx942", + "2d", + P( + total_q=3 * 1024, + num_seqs=3, + max_seqlen_q=1024, + max_seqlen_k=1024, + **_D64_GQA8, + ), + P( + total_q=9 * 1024, + num_seqs=9, + max_seqlen_q=1024, + max_seqlen_k=1024, + **_D64_GQA8, + ), + ), + ( + "gfx942-2d-4warp-ns2-vs-ns9", + "gfx942", + "2d", + P(total_q=2 * 4096, num_seqs=2, **_D256_4WARP), + P(total_q=9 * 4096, num_seqs=9, **_D256_4WARP), + ), + ( + "gfx942-2d-4warp-fold-ns2-vs-ns9", + "gfx942", + "2d", + P(total_q=2 * 8192, num_seqs=2, **_D128_FOLD), + P(total_q=9 * 8192, num_seqs=9, **_D128_FOLD), + ), + ( + "gfx942-3d-prefill-ns3-vs-ns9", + "gfx942", + "3d", + P(total_q=6, num_seqs=3, **_D128_TWO_TOKENS), + P(total_q=18, num_seqs=9, **_D128_TWO_TOKENS), + ), + # gfx942 reads the window size as a kernarg, so window sizes share a binary. + ( + "gfx942-2d-sw256-vs-sw1024", + "gfx942", + "2d", + P( + total_q=2048, + num_seqs=2, + max_seqlen_q=1024, + max_seqlen_k=1024, + sliding_window=256, + **_D64_GQA8, + ), + P( + total_q=2048, + num_seqs=2, + max_seqlen_q=1024, + max_seqlen_k=1024, + sliding_window=1024, + **_D64_GQA8, + ), + ), + ( + "gfx942-2d-4warp-fold-sw1024-vs-sw4096", + "gfx942", + "2d", + P(total_q=2 * 8192, num_seqs=2, **dict(_D128_FOLD, sliding_window=1024)), + P(total_q=2 * 8192, num_seqs=2, **_D128_FOLD), + ), + ( + "gfx942-3d-decode-sw256-vs-sw4096", + "gfx942", + "3d", + P(total_q=4, num_seqs=4, sliding_window=256, **_D128_DECODE), + P(total_q=4, num_seqs=4, sliding_window=4096, **_D128_DECODE), + ), + # gfx942 reads the head counts as kernargs, so head pairs at one GQA ratio + # share a binary. + ( + "gfx942-2d-h64kv8-vs-h32kv4", + "gfx942", + "2d", + P(total_q=2048, num_seqs=2, max_seqlen_q=1024, max_seqlen_k=1024, **_D64_GQA8), + P( + total_q=2048, + num_seqs=2, + max_seqlen_q=1024, + max_seqlen_k=1024, + **dict(_D64_GQA8, num_query_heads=32, num_kv_heads=4), + ), + ), + ( + "gfx942-2d-4warp-h16kv2-vs-h32kv4", + "gfx942", + "2d", + P(total_q=2 * 4096, num_seqs=2, **_D256_4WARP), + P( + total_q=2 * 4096, + num_seqs=2, + **dict(_D256_4WARP, num_query_heads=32, num_kv_heads=4), + ), + ), + ( + "gfx942-2d-4warp-fold-h32kv8-vs-h16kv4", + "gfx942", + "2d", + P(total_q=2 * 8192, num_seqs=2, **_D128_FOLD), + P( + total_q=2 * 8192, + num_seqs=2, + **dict(_D128_FOLD, num_query_heads=16, num_kv_heads=4), + ), + ), + ( + "gfx942-3d-prefill-h32kv8-vs-h64kv16", + "gfx942", + "3d", + P(total_q=6, num_seqs=3, **_D128_TWO_TOKENS), + P( + total_q=6, + num_seqs=3, + **dict(_D128_TWO_TOKENS, num_query_heads=64, num_kv_heads=16), + ), + ), # gfx1250 3D floors the search at 32 iterations, so ns 3 and 4 share a # binary even though they sit in different log2 buckets. ( @@ -498,11 +809,24 @@ def test_same_outputs_share_key_and_ir(routing_env, arch, path, a, b, flavor): if path == "2d" else (au._tiled_3d_cache_key, _lower_3d) ) - assert a.num_seqs != b.num_seqs or a.max_seqlen_q != b.max_seqlen_q + assert a != b assert key_fn(a, arch) == key_fn(b, arch) assert lower(a, arch, flavor) == lower(b, arch, flavor) +def test_fast_paged_kv_desc_splits_head_pairs_at_one_ratio(routing_env): + # The gfx950 fast paged-KV descriptor bakes the 64/8 head geometry. The key + # carries only the GQA ratio, so it must carry the fast-desc flag too: a + # 32/4 problem at ratio 8 must not reuse the 64/8 fast binary. + arch = "gfx950" + base = dict(total_q=2048, num_seqs=2, max_seqlen_q=1024, max_seqlen_k=1024) + a = P(**base, **_D64_GQA8) + b = P(**base, **dict(_D64_GQA8, num_query_heads=32, num_kv_heads=4)) + assert au._tiled_spec_from_problem(a, arch).use_fast_paged_kv_desc + assert not au._tiled_spec_from_problem(b, arch).use_fast_paged_kv_desc + assert au._tiled_cache_key(a, arch) != au._tiled_cache_key(b, arch) + + # --------------------------------------------------------------------------- # Launch geometry still separates batch counts # --------------------------------------------------------------------------- @@ -533,17 +857,43 @@ def test_launch_meta_separates_num_seqs_under_one_key(routing_env): assert grid_b[1] - grid_a[1] == b.num_seqs - a.num_seqs +@pytest.mark.parametrize("arch,num_cus", [("gfx942", 304), ("gfx950", 256)]) +def test_launch_geometry_separates_head_counts_under_one_key( + routing_env, arch, num_cus +): + # gfx942 and gfx950 read the head counts as kernargs, so one binary serves + # 32/8 and 64/16. The 2D grid spans the kv heads and the 3D workspace the + # query heads, so with equal token and sequence counts the launch caches + # must still tell them apart. + base = dict( + total_q=1025, + num_seqs=1, + head_size=128, + block_size=16, + max_seqlen_q=1025, + max_seqlen_k=1025, + dtype="bf16", + num_cus=num_cus, + ) + a = P(num_query_heads=32, num_kv_heads=8, **base) + b = P(num_query_heads=64, num_kv_heads=16, **base) + key = au._tiled_cache_key(a, arch) + assert key == au._tiled_cache_key(b, arch) + assert au._shape_key(key, a) != au._shape_key(key, b) + au._2D_LAUNCH_META.clear() + grid_a = au._get_2d_launch_meta(a, key, arch=arch).grid + grid_b = au._get_2d_launch_meta(b, key, arch=arch).grid + assert (grid_a[0], grid_b[0]) == (a.num_kv_heads, b.num_kv_heads) + + # --------------------------------------------------------------------------- # _binary_search_iters mirrors the spec classes # --------------------------------------------------------------------------- -# (spec class, floor). gfx1250 3D floors the search at 32 iterations. +# (spec class, floor) for every class that bakes the search. gfx1250 3D floors +# it at 32 iterations. _SPEC_CLASSES = ( - (Gfx942Spec2D, 1), - (Gfx942Spec3D, 1), - (Gfx950Spec2D, 1), - (Gfx950Spec3D, 1), (Gfx1250Spec2D, 1), (Gfx1250Spec3D, 32), ) @@ -570,3 +920,116 @@ def test_binary_search_iters_mirrors_spec(spec_cls, floor, num_seqs): dtype="bf16", ) assert au._binary_search_iters(problem, floor=floor) == spec.binary_search_iters + + +@pytest.mark.parametrize( + "arch, path, spec_cls", + ( + ("gfx942", "2d", Gfx942Spec2D), + ("gfx942", "3d", Gfx942Spec3D), + ("gfx950", "2d", Gfx950Spec2D), + ("gfx950", "3d", Gfx950Spec3D), + ), +) +def test_specs_search_at_runtime(arch, path, spec_cls): + assert spec_cls.RUNTIME_SEARCH_ITERS + assert not hasattr(spec_cls, "binary_search_iters") + assert au._searches_at_runtime(arch, path) + assert not au._searches_at_runtime("gfx1250", path) + + +def _trip_count(num_seqs: int) -> int: + """``binary_search_seq_idx_trip_count`` on the host: the exponent of the + round-to-nearest f32 conversion LLVM's ``sitofp`` performs.""" + np = pytest.importorskip("numpy") + bits = int( + np.array([num_seqs], dtype=np.int32).astype(np.float32).view(np.int32)[0] + ) + return max((bits >> 23) - 126, 1) + + +_BATCHES = sorted( + {n for n in range(1, 4097)} + | {(1 << k) + d for k in range(12, 31) for d in (-1, 0, 1)} + | {(1 << 24) + d for d in range(-8, 9)} + | {(1 << 31) - 1} +) + + +def test_runtime_trip_count_covers_every_batch(): + # The search needs bit_length(num_seqs) iterations. Exact below 2**24; past + # it the conversion may round up to the next power of two, adding one + # iteration the done guard makes a no-op. + for n in _BATCHES: + trip = _trip_count(n) + if n < 1 << 24: + assert ( + trip + == n.bit_length() + == au._binary_search_iters( + P( + total_q=n, + num_seqs=n, + num_query_heads=8, + num_kv_heads=8, + head_size=128, + block_size=16, + max_seqlen_q=1, + max_seqlen_k=1, + dtype="bf16", + ) + ) + ), n + else: + assert n.bit_length() <= trip <= n.bit_length() + 1, n + + +@pytest.mark.parametrize("arch,num_cus", [("gfx942", 304), ("gfx950", 256)]) +def test_i64_kv_addressing_is_in_the_2d_kernel_name(routing_env, arch, num_cus): + # i64 K/V addressing changes the 2D loaders, so a binary is identified by + # it: the name, not only the in-process key, must tell the two apart. + base = dict( + total_q=2048, + num_seqs=2, + num_query_heads=32, + num_kv_heads=8, + head_size=128, + block_size=16, + max_seqlen_q=1024, + max_seqlen_k=1024, + dtype="bf16", + num_cus=num_cus, + ) + block_bytes = 16 * 8 * 128 * 2 + small = P(num_kv_blocks=64, **base) + large = P(num_kv_blocks=(1 << 31) // block_bytes + 1, **base) + assert not au._enable_i64_kv_addr(small) and au._enable_i64_kv_addr(large) + names = [au._tiled_spec_from_problem(p, arch).kernel_name() for p in (small, large)] + assert "i64" not in names[0].split("_") + assert "i64" in names[1].split("_") + assert names[0].replace("_bf16_", "_bf16_i64_", 1) == names[1] + + +@pytest.mark.parametrize( + "spec_cls", [Gfx942Spec2D, Gfx950Spec2D], ids=["gfx942", "gfx950"] +) +def test_fast_paged_kv_desc_names_its_head_counts(spec_cls): + # The fast paged-KV descriptor body shifts by a 64/8 K/V stride instead of + # reading the head-count kernargs, so its name must not share the plain + # g8 symbol that serves every head pair at that ratio. + base = dict( + head_size=64, + block_size=32, + tile_size=64, + num_query_heads=64, + num_kv_heads=8, + dtype="bf16", + num_warps=4, + use_sinks=False, + sliding_window=0, + has_softcap=False, + ) + plain = spec_cls(**base).kernel_name() + fast = spec_cls(**base, use_fast_paged_kv_desc=True).kernel_name() + assert "h64kv8" not in plain.split("_") + assert fast == plain.replace("_g8_", "_g8_h64kv8_", 1) + "_fastkvdesc" diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_cpp_entry_guards.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_cpp_entry_guards.py new file mode 100644 index 000000000000..76ccc968bce0 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_cpp_entry_guards.py @@ -0,0 +1,75 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""The C++ tiled-attention entries refuse what the Python builders refuse, with +the same reason, instead of crashing. + +* A zero or non-dividing head pair: the 2D ``_new`` entries used to fail at + naming before initialising the builder, and the caller then freed that + uninitialised builder (signal 11). +* fastKV register-P on gfx942: the body is gfx950-only, so the C++ builder must + refuse it as Python does rather than emit gfx950 MFMA for gfx942. +""" + +from __future__ import annotations + +import dataclasses + +import pytest +from kernels.gfx942.attention_tiled_2d import ( + UnifiedAttention2DTiledSpec as Gfx942Spec2D, +) +from kernels.gfx950.attention_tiled_2d import ( + UnifiedAttention2DTiledSpec as Gfx950Spec2D, +) +from kernels.gfx950.attention_tiled_2d_fastkv_regp import ( + build_unified_attention_2d_fastkv_register_p, + make_fastkv_register_p_spec, +) + +rocke_engine = pytest.importorskip("rocke_engine") + +_PROBLEM = { + "head_size": 128, + "block_size": 16, + "dtype": "bf16", + "use_sinks": False, + "sliding_window": 0, + "has_softcap": False, +} + + +@pytest.mark.parametrize( + "arch, spec_cls", + [("gfx942", Gfx942Spec2D), ("gfx950", Gfx950Spec2D)], +) +@pytest.mark.parametrize( + "heads, reason", + [((32, 0), "must be > 0"), ((32, 6), "must be divisible by num_kv_heads")], +) +def test_2d_entry_refuses_bad_head_counts_like_python(arch, spec_cls, heads, reason): + hq, hkv = heads + spec = spec_cls(num_query_heads=hq, num_kv_heads=hkv, **_PROBLEM) + with pytest.raises(ValueError, match=reason) as py_err: + spec.kernel_name() + lower = getattr(rocke_engine, f"{arch}_attention_tiled_2d_lower_llvm") + with pytest.raises(Exception) as cpp_err: + lower(dataclasses.asdict(spec), arch) + assert str(py_err.value) in str(cpp_err.value) + + +def test_fastkv_register_p_entry_refuses_gfx942_like_python(): + spec = make_fastkv_register_p_spec( + Gfx950Spec2D( + num_query_heads=64, + num_kv_heads=8, + **{**_PROBLEM, "head_size": 64, "block_size": 32}, + ) + ) + with pytest.raises(ValueError, match="gfx950 kernel") as py_err: + build_unified_attention_2d_fastkv_register_p(spec, arch="gfx942") + with pytest.raises(Exception) as cpp_err: + rocke_engine.gfx950_attention_tiled_2d_fastkv_regp_lower_llvm( + dataclasses.asdict(spec), "gfx942" + ) + assert str(py_err.value) in str(cpp_err.value) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_launch_signature.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_launch_signature.py new file mode 100644 index 000000000000..8e900974a432 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_launch_signature.py @@ -0,0 +1,232 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""Launch-signature contract for the tiled 2D / 3D attention kernels. + +``KernelLauncher`` packs kernargs by the signature's names and order, so a kernel +whose params drift from its launch signature reads the wrong argument without +faulting. Every builder route the production path takes is built here and its +params are compared, by name and type in order, with the signature its launcher +uses (``_tiled_2d_signature`` / ``_tiled_3d_signatures``). That includes the +runtime shape kernargs a spec class declares (``RUNTIME_KERNARG_FIELDS``). + +No GPU required. +""" + +from __future__ import annotations + +import pytest + +from kernels.common import attention_unified as au + +P = au.UnifiedAttentionProblem + +_ROUTING_ENV = ( + "HIPDNN_GFX942_K_SLICE_HD", + "HIPDNN_GFX942_K_SLICED_RING", + "HIPDNN_GFX942_K_LDSSEQ", + "HIPDNN_GFX942_FLASH_WIDE", + "HIPDNN_GFX942_FLASH_MLIM", + "HIPDNN_GFX942_BF16_CFVST", + "HIPDNN_GFX942_D128_SMALLTILE_DK", + "HIPDNN_GFX942_3D_HOIST", + "HIPDNN_GFX942_3D_WKV", + "HIPDNN_GFX950_D128_SMALL_TILE", + "HIPDNN_GFX950_D128_SOFTMAX_INTERLEAVE", +) + + +@pytest.fixture(autouse=True) +def routing_env(monkeypatch): + """Production routing: no env knob steers a selector.""" + for var in _ROUTING_ENV: + monkeypatch.delenv(var, raising=False) + + +def _prefill(*, dtype="bf16", head_size=128, hq=32, hkv=8, sq=1024, **kw): + return P( + total_q=2 * sq, + num_seqs=2, + num_query_heads=hq, + num_kv_heads=hkv, + head_size=head_size, + block_size=kw.pop("block_size", 16), + max_seqlen_q=sq, + max_seqlen_k=sq, + dtype=dtype, + **kw, + ) + + +def _decode(*, dtype="bf16", head_size=128, hq=32, hkv=8, sq=1, **kw): + return P( + total_q=4 * sq, + num_seqs=4, + num_query_heads=hq, + num_kv_heads=hkv, + head_size=head_size, + block_size=16, + max_seqlen_q=sq, + max_seqlen_k=4096, + dtype=dtype, + **kw, + ) + + +# (arch, case id, problem, kernel-name suffix the route must emit or None) +_CASES_2D = ( + ("gfx942", "narrow_d64", _prefill(head_size=64, hq=8, hkv=8, sq=128), None), + ("gfx942", "fp16_flash_d128", _prefill(dtype="fp16", sq=2048), None), + ("gfx942", "bf16_flash_d128", _prefill(sq=2048), None), + ("gfx942", "window", _prefill(sliding_window=256, hq=8, hkv=8), None), + ("gfx942", "sinks", _prefill(use_sinks=True), None), + ( + "gfx942", + "4warp_d256", + _prefill(head_size=256, hq=16, hkv=2, sq=4096), + "_4wgqa", + ), + ( + "gfx942", + "4warp_fold", + _prefill(sliding_window=4096, sq=8192), + "_4wgqa_fold", + ), + ( + "gfx942", + "4warp_fp16_window", + _prefill(dtype="fp16", sliding_window=4096, sq=8192), + "_4wgqa", + ), + ("gfx950", "bf16_d128", _prefill(), None), + ( + "gfx950", + "fp16_d64_window", + _prefill(dtype="fp16", head_size=64, sliding_window=256), + None, + ), +) + + +def _assert_matches(kernel, signature): + names = [p.name for p in kernel.params] + assert names == [a["name"] for a in signature], kernel.name + types = [p.type.name.replace(" ", "") for p in kernel.params] + assert types == [a["type"].replace(" ", "") for a in signature], kernel.name + + +@pytest.mark.parametrize( + "arch, problem, suffix", + [pytest.param(a, p, s, id=f"{a}-{i}") for a, i, p, s in _CASES_2D], +) +def test_2d_kernel_params_match_launch_signature(arch, problem, suffix): + kernel = au._tiled_2d_kernel(problem, arch) + if suffix is not None: + assert kernel.name.endswith(suffix), kernel.name + signature = au._tiled_2d_signature( + arch, + problem.dtype, + kv_dtype=au._kv_storage_dtype(problem), + include_additive_bias=problem.use_additive_bias, + ) + _assert_matches(kernel, signature) + + +_CASES_3D = ( + ("gfx942", "decode", _decode(), False), + ("gfx942", "decode_window", _decode(sliding_window=256), False), + ("gfx942", "decode_lse", _decode(emit_lse=True), False), + # Two queries per sequence leave the decode grid for the runtime search. + ("gfx942", "two_token", _decode(sq=2), False), + ("gfx942", "strided", _decode(hq=4, hkv=2), True), + ("gfx950", "decode", _decode(), False), + ("gfx950", "strided", _decode(hq=4, hkv=2), True), +) + + +@pytest.mark.parametrize( + "arch, problem, strided_kv", + [pytest.param(a, p, s, id=f"{a}-{i}") for a, i, p, s in _CASES_3D], +) +def test_3d_kernel_params_match_launch_signatures(arch, problem, strided_kv): + if problem.max_seqlen_q > 1: + assert not au._use_decode_grid(problem, arch) + num_segments = au._num_segments(problem, arch) + segment, reduce = au._tiled_3d_kernels( + problem, arch, num_segments, strided_kv=strided_kv + ) + seg_sig, red_sig = au._tiled_3d_signatures( + arch, + problem.dtype, + kv_dtype=au._kv_storage_dtype(problem), + strided_kv=strided_kv, + include_additive_bias=problem.use_additive_bias, + emit_lse=problem.emit_lse, + ) + _assert_matches(segment, seg_sig) + _assert_matches(reduce, red_sig) + + +@pytest.mark.parametrize( + "codepath", ["narrow", "transposed_x8", "gfx942_4warp", "splitkv"] +) +def test_tuning_candidate_signature_matches_its_kernel(codepath): + """The registry's tuning candidates launch through ``candidate.signature``.""" + from dispatch.attention import attention_candidates + from dispatch.attention.common import AttentionMaskType, AttentionRequest + from dispatch.attention.gfx942_unified import GFX942_TUNING_VARIANTS + + variants = [v for v in GFX942_TUNING_VARIANTS if v.codepath == codepath] + candidates = {c.spec_id: c for c in attention_candidates()} + head_size = 256 if codepath == "gfx942_4warp" else 128 + # Two queries per sequence keep the split-KV segment off the decode grid. + seqlen_q = 2 if codepath == "splitkv" else 1024 + checked = 0 + for variant in variants: + request = AttentionRequest( + batch=1, + nhead_q=16, + nhead_k=2, + seqlen_q=seqlen_q, + seqlen_k=1024, + hdim_q=head_size, + hdim_v=head_size, + arch="gfx942", + dtype="bf16", + mask_type=AttentionMaskType.BOTTOM_RIGHT_CAUSAL, + num_cus=304, + algorithm="unified_tuning", + spec_id=variant.spec_id, + ) + candidate = candidates[variant.spec_id] + if not candidate.admits(request)[0]: + continue + spec = candidate.select_spec(request) + kernel = candidate.build(spec, "gfx942") + if spec.path == "3d": + # The signature is the segment's; _get_3d_pipeline launches the + # reduce with the production 3D pair. + kernel, reduce = kernel + assert "dg" not in kernel.name.split("_"), kernel.name + _, red_sig = au._tiled_3d_signatures( + "gfx942", + spec.kernel_spec.dtype, + kv_dtype=spec.kernel_spec.kv_storage_dtype, + ) + _assert_matches(reduce, red_sig) + _assert_matches(kernel, candidate.signature(spec)) + checked += 1 + break + assert checked, f"no {codepath} tuning candidate admits the request" + + +@pytest.mark.parametrize("arch", ["gfx942", "gfx950"]) +def test_runtime_values_cover_every_declared_field(arch): + """The shared argument pack supplies every field a spec class can declare, + so a launcher never raises on a runtime kernarg.""" + segment_cls, reduce_cls, _, _, _ = au._tiled_3d_impl(arch) + spec_2d_cls, _, _ = au._tiled_2d_impl(arch) + values = au._runtime_kernarg_values(_prefill()) + for cls in (spec_2d_cls, segment_cls, reduce_cls): + missing = set(au._runtime_kernarg_fields(cls)) - set(values) + assert not missing, (cls.__name__, missing) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_runtime_shape_numeric.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_runtime_shape_numeric.py new file mode 100644 index 000000000000..0bfba7f0d7a4 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attention_tiled_runtime_shape_numeric.py @@ -0,0 +1,484 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""One gfx942 or gfx950 tiled binary serves every head pair, window size and batch. + +The tiled 2D (gfx942 also 4-warp GQA) and 3D kernels read the head counts and +the sliding window as kernargs and derive the cu_q search trip count from +``num_seqs``, so shapes that differ only in those share one cache key and one +compiled binary. Each case below runs a pair of such shapes back to back: the +second launch must reuse the first binary (no new compile) and both must match +an FP32 paged reference. + +The second shape of each pair changes at least one of the head counts (at the +same GQA ratio), the window size, or the batch (by several log2 search +buckets), so a value that is still baked into the binary corrupts it. Block-table +rows are wider than the batch needs and unused slots point at a poison block, +so a wrong stride cannot pass by luck. + +gfx942 routes windowed decode to 2D, so its 3D pairs keep the window at 0. The +gfx950 fast paged-KV descriptor bakes the 64/8 head geometry, so its pair keeps +64/8 and changes the batch; its 32/4 and 16/2 neighbours at the same ratio take +the plain combo binary. + +The kernels run on ``rocke.runtime`` device buffers filled from numpy, and the +reference is numpy over the fp16/bf16-rounded inputs, so the launches run in GPU +lanes without torch. 2D launches go through ``run_unified_attention_torch``; +3D launches build and launch the segment + reduce pair as its 3D path does. +The routing and key checks run on any host; the launches need the case's GPU. +""" + +from __future__ import annotations + +import itertools +import math + +import numpy as np +import pytest + +from kernels.common import attention_unified as au +from rocke.numeric.bf16 import bf16_to_f32, f32_to_bf16 +from rocke.runtime.hip_module import get_device_arch + +from ._attention_device_array import DeviceArray + +_CUS = {"gfx942": 304, "gfx950": 256} + + +def _shape(q_lens, kv_lens, hq, hkv, window): + return { + "q_lens": q_lens, + "kv_lens": kv_lens, + "hq": hq, + "hkv": hkv, + "window": window, + } + + +# Pairs both arches run unchanged. +_HEADS_ONLY = (_shape([1025], [1025], 32, 8, 0), _shape([1025], [1025], 64, 16, 0)) +_D64_WINDOW = ( + _shape([260, 180], [400, 180], 8, 8, 128), + _shape( + [70, 90, 50, 100, 60, 80, 40, 30, 110], + [200, 90, 300, 100, 60, 250, 40, 30, 110], + 16, + 16, + 512, + ), +) +# The decode split sizes segments by num_seqs * num_kv_heads, so the 3D pairs +# hold that product fixed; windowed decode routes to 2D. +_DECODE = ( + _shape([1, 1], [4096, 2500], 64, 16, 0), + _shape([1] * 8, [3000, 900, 4096, 1700, 2200, 3500, 1200, 2800], 16, 4, 0), +) +_TWO_TOKEN = ( + _shape([2, 2], [4096, 3100], 64, 16, 0), + _shape([2] * 8, [2500, 4096, 1900, 2600, 3100, 2200, 2400, 2700], 16, 4, 0), +) +_D64_BATCH_KV = [300, 600, 400, 800, 350, 310, 500, 290, 330] +_D64_BATCH_Q = [300, 280, 400, 260, 350, 310, 270, 290, 330] + +# (arch, id, path, route, (d, bs, dtype), first shape, second shape). +# gfx942: ``route`` is the 4-warp GQA cohort both 2D shapes take, or None for +# the plain tiled kernel. gfx950: ``route`` names an attention_unified selector +# both shapes must satisfy, or None for no check. gfx950 pairs sit on one side +# of every seqlen threshold the selectors read (combo above 256 query tokens, +# hipcc above 512), so only the head counts, the window and the batch differ. +_CASES = ( + ( + "gfx942", + "2d-bf16-d128", + "2d", + None, + (128, 16, "bf16"), + _shape([300, 200, 250], [300, 450, 250], 32, 8, 0), + _shape( + [90, 130, 70, 110, 40, 150, 60, 80, 120], + [90, 300, 70, 210, 40, 150, 60, 380, 120], + 16, + 4, + 0, + ), + ), + # Head counts alone: the token and sequence counts match, so only the head + # counts tell the two launches apart. + ("gfx942", "2d-bf16-d128-heads-only", "2d", None, (128, 16, "bf16"), *_HEADS_ONLY), + ("gfx942", "2d-fp16-d64-window", "2d", None, (64, 32, "fp16"), *_D64_WINDOW), + ( + "gfx942", + "4warp-d256", + "2d", + "gfx942_d256_causal", + (256, 16, "bf16"), + _shape([1024, 1024], [1024, 1024], 16, 2, 0), + _shape( + [300, 500, 200, 400, 250, 350, 150, 450, 280], + [300, 700, 200, 400, 250, 600, 150, 450, 280], + 32, + 4, + 0, + ), + ), + ( + "gfx942", + "4warp-fold-window", + "2d", + "gfx942_d128_swa", + (128, 16, "bf16"), + _shape([1024], [1024], 32, 8, 512), + _shape( + [200, 300, 150, 250, 180, 220, 260, 140, 310], + [400, 300, 350, 250, 180, 520, 260, 140, 310], + 16, + 4, + 256, + ), + ), + ("gfx942", "3d-decode", "3d", None, (128, 16, "bf16"), *_DECODE), + ("gfx942", "3d-two-token", "3d", None, (128, 16, "bf16"), *_TWO_TOKEN), + ( + "gfx950", + "2d-bf16-d128", + "2d", + None, + (128, 16, "bf16"), + _shape([300, 200, 250], [300, 450, 250], 32, 8, 0), + _shape( + [90, 130, 70, 110, 40, 300, 60, 80, 120], + [90, 300, 70, 210, 40, 300, 60, 380, 120], + 16, + 4, + 0, + ), + ), + ( + "gfx950", + "2d-bf16-d128-single-batch-heads-only", + "2d", + "_enable_single_batch_combo", + (128, 16, "bf16"), + *_HEADS_ONLY, + ), + ("gfx950", "2d-fp16-d64-window", "2d", None, (64, 32, "fp16"), *_D64_WINDOW), + # The single-batch fp16 d128 window cohort: heads and window change. + ( + "gfx950", + "2d-fp16-d128-single-batch-window", + "2d", + "_enable_single_batch_combo", + (128, 16, "fp16"), + _shape([2048], [2048], 32, 8, 256), + _shape([2048], [2048], 64, 16, 1024), + ), + ( + "gfx950", + "2d-bf16-d64-fast-desc-batch", + "2d", + "_enable_fast_paged_kv_desc", + (64, 32, "bf16"), + _shape([500, 400], [700, 900], 64, 8, 0), + _shape(_D64_BATCH_Q, _D64_BATCH_KV, 64, 8, 0), + ), + ( + "gfx950", + "2d-bf16-d64-combo-heads", + "2d", + "_enable_combo_2d", + (64, 32, "bf16"), + _shape([500, 400], [700, 900], 32, 4, 0), + _shape(_D64_BATCH_Q, _D64_BATCH_KV, 16, 2, 0), + ), + ( + "gfx950", + "2d-bf16-d256", + "2d", + "_d256_gfx950_fast", + (256, 16, "bf16"), + _shape([512, 512], [1024, 1024], 16, 2, 0), + _shape( + [300, 500, 200, 400, 250, 350, 150, 450, 280], + [300, 700, 200, 400, 250, 600, 150, 450, 280], + 32, + 4, + 0, + ), + ), + # D192 short prefill takes the one-warp 2D kernel, whose 512-half K/V copy + # ends mid-row: the loaders must keep the paged-KV descriptor. + ( + "gfx950", + "2d-bf16-d192-one-warp", + "2d", + None, + (192, 16, "bf16"), + _shape([64, 48], [256, 192], 16, 2, 0), + _shape([64, 60, 50, 40, 56], [256, 192, 320, 200, 280], 32, 4, 0), + ), + ("gfx950", "3d-decode", "3d", None, (128, 16, "bf16"), *_DECODE), + ("gfx950", "3d-two-token", "3d", None, (128, 16, "bf16"), *_TWO_TOKEN), +) + + +def _params(*fields): + return [pytest.param(*(c[i] for i in fields), id=f"{c[0]}-{c[1]}") for c in _CASES] + + +def _problem(arch, geometry, shape): + d, bs, dtype = geometry + q_lens, kv_lens = shape["q_lens"], shape["kv_lens"] + return au.UnifiedAttentionProblem( + total_q=sum(q_lens), + num_seqs=len(q_lens), + num_query_heads=shape["hq"], + num_kv_heads=shape["hkv"], + head_size=d, + block_size=bs, + max_seqlen_q=max(q_lens), + max_seqlen_k=max(kv_lens), + dtype=dtype, + sliding_window=shape["window"], + num_cus=_CUS[arch], + num_kv_blocks=sum(math.ceil(n / bs) for n in kv_lens) + 1, + ) + + +def _key(problem, path, arch): + if path == "2d": + return au._tiled_cache_key(problem, arch) + return au._tiled_3d_cache_key(problem, arch) + + +def _check_route(arch, path, route, problem): + if arch == "gfx942": + found = au._gfx942_4warp_route(problem, arch) if path == "2d" else None + assert (found.cohort if found else None) == route + elif route is not None: + assert getattr(au, route)(problem, arch) + + +@pytest.mark.parametrize( + "arch, path, route, geometry, first, second", _params(0, 2, 3, 4, 5, 6) +) +def test_pair_shares_one_binary(arch, path, route, geometry, first, second): + """Both shapes take the intended path and route and share one key, so the + GPU test below exercises one binary on two shapes.""" + a, b = _problem(arch, geometry, first), _problem(arch, geometry, second) + for p in (a, b): + assert p.select_path() == path + _check_route(arch, path, route, p) + assert a.num_queries_per_kv == b.num_queries_per_kv + batch_changes = a.num_seqs != b.num_seqs + if batch_changes and not (path == "3d" and au._use_decode_grid(a, arch)): + # The decode grid indexes sequences directly and emits no search. + assert au._binary_search_iters(a) != au._binary_search_iters(b) + assert _key(a, path, arch) == _key(b, path, arch) + + +def test_d192_pair_takes_the_one_warp_kernel(): + """The D192 pair must stay on the one-warp spec: that is the geometry whose + K/V copy ends mid-row, which the numeric pair is there to cover.""" + case = next(c for c in _CASES if c[1] == "2d-bf16-d192-one-warp") + for shape in case[5:]: + spec = au._tiled_spec_from_problem(_problem("gfx950", case[4], shape), "gfx950") + assert spec.head_size == 192 and spec.num_warps == 1, spec + + +def _encode(x, dtype): + """fp32 values as the kernel's fp16 or bf16 (raw uint16) host array.""" + return x.astype(np.float16) if dtype == "fp16" else f32_to_bf16(x) + + +def _decode(x, dtype): + return x.astype(np.float32) if dtype == "fp16" else bf16_to_f32(x) + + +def _case(geometry, shape, seed): + """fp32 q, k, v exact in the kernel dtype, cu_seqlens_q and a block table + wider than needed whose unused slots point at a poison block.""" + d, bs, dtype = geometry + q_lens, kv_lens, hq, hkv = (shape[f] for f in ("q_lens", "kv_lens", "hq", "hkv")) + rng = np.random.default_rng(seed) + needed = [math.ceil(n / bs) for n in kv_lens] + num_blocks = sum(needed) + 1 + poison = num_blocks - 1 + order = iter(rng.permutation(poison)) + table = np.full((len(kv_lens), max(needed) + 3), poison, dtype=np.int32) + for s, n in enumerate(needed): + table[s, :n] = [next(order) for _ in range(n)] + + def randn(*dims): + return _decode(_encode(rng.standard_normal(dims, np.float32), dtype), dtype) + + q = randn(sum(q_lens), hq, d) + k = randn(num_blocks, bs, hkv, d) + v = randn(num_blocks, bs, hkv, d) + k[poison] = 64.0 + v[poison] = 64.0 + cu = np.array([0, *itertools.accumulate(q_lens)], dtype=np.int32) + return q, k, v, cu, table + + +def _reference(q, k, v, cu, table, shape, bs): + """FP32 causal (optionally windowed) paged attention, one KV head group at + a time.""" + q_lens, kv_lens, hkv, window = ( + shape[f] for f in ("q_lens", "kv_lens", "hkv", "window") + ) + d = q.shape[-1] + ratio = shape["hq"] // hkv + out = np.zeros(q.shape, dtype=np.float32) + for s, (q_len, kv_len) in enumerate(zip(q_lens, kv_lens)): + blocks = table[s, : math.ceil(kv_len / bs)] + keys = k[blocks].reshape(-1, hkv, d)[:kv_len] + values = v[blocks].reshape(-1, hkv, d)[:kv_len] + start = int(cu[s]) + q_pos = (kv_len - q_len) + np.arange(q_len)[:, None] + k_pos = np.arange(kv_len)[None, :] + visible = k_pos <= q_pos + if window > 0: + visible &= q_pos - k_pos < window + for g in range(hkv): + heads = slice(g * ratio, (g + 1) * ratio) + queries = q[start : start + q_len, heads].transpose(1, 0, 2) + scores = (queries @ keys[:, g].T) / math.sqrt(d) + scores = np.where(visible, scores, -np.inf) + probs = np.exp(scores - scores.max(-1, keepdims=True)) + probs /= probs.sum(-1, keepdims=True) + out[start : start + q_len, heads] = (probs @ values[:, g]).transpose( + 1, 0, 2 + ) + return out + + +def _launch_3d(rt, arch, problem, q, k, v, out, cu, seq_lens, table, scale): + """The 3D segment + reduce pair, launched as ``run_unified_attention_torch``'s + 3D path does (same support check, cache key, segment count and grids).""" + ok, why = au._explicit_path_supported(problem, None, "3d", arch) + assert ok, why + num_segments = au._num_segments(problem, arch) + prepared = au._get_3d_pipeline( + problem, au._tiled_3d_cache_key(problem, arch), num_segments, arch=arch + ) + # NaN until the kernels write it, so a skipped workspace slot fails. + segm_output, segm_max, segm_expsum = ( + DeviceArray(rt, np.full(spec.shape, np.nan, np.float32)) + for spec in au._attention_3d_workspace_specs(problem, num_segments, None) + ) + seg_vals = { + "segm_output_ptr": segm_output, + "segm_max_ptr": segm_max, + "segm_expsum_ptr": segm_expsum, + "query_ptr": q, + "key_cache_ptr": k, + "value_cache_ptr": v, + "sink_ptr": None, + "block_tables_ptr": table, + "seq_lens_ptr": seq_lens, + "alibi_slopes_ptr": 0, + "qq_bias_ptr": 0, + "query_start_len_ptr": cu, + "scale": scale, + "k_scale": 1.0, + "v_scale": 1.0, + "softcap": 0.0, + "num_seqs": problem.num_seqs, + "block_table_stride": table.stride(0), + "qq_bias_stride_0": 0, + **au._runtime_kernarg_values(problem), + } + red_vals = { + "output_ptr": out, + "segm_output_ptr": segm_output, + "segm_max_ptr": segm_max, + "segm_expsum_ptr": segm_expsum, + "seq_lens_ptr": seq_lens, + **au._runtime_kernarg_values(problem), + } + au._launch_3d_pipeline(prepared, seg_vals, red_vals, stream=0, capturing=False) + + +def _launch(rt, arch, path, problem, geometry, q, k, v, cu, table, kv_lens): + """Run one shape on ``path``; returns O as float32.""" + d, _, dtype = geometry + host_q = _encode(q, dtype) + # Every byte 0xFF is NaN in fp16 and bf16: an element the kernel skips + # fails the check instead of passing on stale memory. + out_host = np.empty_like(host_q) + out_host.view(np.uint8)[...] = 0xFF + out = DeviceArray(rt, out_host) + args = { + "q": DeviceArray(rt, host_q), + "k": DeviceArray(rt, _encode(k, dtype)), + "v": DeviceArray(rt, _encode(v, dtype)), + "out": out, + "cu": DeviceArray(rt, cu), + "seq_lens": DeviceArray(rt, np.array(kv_lens, dtype=np.int32)), + "table": DeviceArray(rt, table), + "scale": 1.0 / math.sqrt(d), + } + if path == "3d": + _launch_3d(rt, arch, problem, **args) + else: + au.run_unified_attention_torch( + problem=problem, + q=args["q"], + k=args["k"], + v=args["v"], + out=out, + cu_seqlens_q=args["cu"], + seqused_k=args["seq_lens"], + softmax_scale=args["scale"], + block_table=args["table"], + softcap=0.0, + backend="tiled", + use_graph=False, + ) + # The read is a blocking copy, so the launch has finished. + return _decode(out.read(), dtype) + + +def _cache(path): + return au._ATTN_3D_TILED_CACHE if path == "3d" else au._ATTN_TILED_CACHE + + +def _compiled(): + return len(au._ATTN_TILED_CACHE) + len(au._ATTN_3D_TILED_CACHE) + + +@pytest.fixture(scope="module") +def device_arch(): + return get_device_arch() + + +@pytest.mark.gpu +@pytest.mark.parametrize("arch, path, geometry, first, second", _params(0, 2, 4, 5, 6)) +def test_pair_numeric_on_one_binary( + device_arch, monkeypatch, arch, path, geometry, first, second +): + if device_arch != arch: + pytest.skip(f"requires a {arch} GPU, found {device_arch or 'no GPU'}") + from rocke.runtime.hip_module import Runtime + + # The gfx942 2D graph replay captures through torch; launch eagerly. + monkeypatch.setenv("HIPDNN_GFX942_2D_GRAPH", "0") + rt = Runtime() + _, bs, _ = geometry + compiled = None + for i, shape in enumerate((first, second)): + problem = _problem(arch, geometry, shape) + q, k, v, cu, table = _case(geometry, shape, seed=i) + got = _launch( + rt, arch, path, problem, geometry, q, k, v, cu, table, shape["kv_lens"] + ) + missed = f"shape {i} missed the {path} cache" + assert _key(problem, path, arch) in _cache(path), missed + if compiled is None: + compiled = _compiled() + else: + assert _compiled() == compiled, "the second shape compiled a new binary" + want = _reference(q, k, v, cu, table, shape, bs) + max_abs = float(np.abs(got - want).max()) + error = f"shape {i}: max_abs {max_abs:.3e} vs fp32 reference" + assert math.isfinite(max_abs) and max_abs <= 4e-2, error diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attn_bf16_d128_ring.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attn_bf16_d128_ring.py index 2f9fdb58d5b9..6736f225c5cd 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attn_bf16_d128_ring.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_attn_bf16_d128_ring.py @@ -267,7 +267,7 @@ def test_d128_numeric_vs_fp32_oracle_at_magnitude(monkeypatch, dtype, tol, hq, h from rocke import compile_kernel from kernels import build_unified_attention_2d_tiled - from kernels.common.attention_unified import _attn_signature + from kernels.common.attention_unified import _tiled_2d_signature from rocke.runtime import KernelLauncher kernel = build_unified_attention_2d_tiled(spec, arch="gfx942") @@ -275,9 +275,7 @@ def test_d128_numeric_vs_fp32_oracle_at_magnitude(monkeypatch, dtype, tol, hq, h launcher = KernelLauncher( hsaco=artifact.hsaco, kernel_name=artifact.kernel_name, - signature=_attn_signature( - dtype, include_bt_stride=True, include_qq_bias_stride=True - ), + signature=_tiled_2d_signature("gfx942", dtype), cache_key=("d128guard", spec.kernel_name(), dtype, sq, hq, hk), ) out, _ = H._run_rocke(s, data, launcher, spec, warmup=3, attempts=3) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_strided_kv_decode.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_strided_kv_decode.py index 393c50adb068..4ce132b354ac 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_strided_kv_decode.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_strided_kv_decode.py @@ -16,7 +16,7 @@ import pytest from kernels.common.attention_kv_cache import KvTensorLayout, StridedKvCacheLayout -from kernels.common.attention_unified import _3d_signature +from kernels.common.attention_unified import _tiled_3d_signatures from rocke.core.verify import verify from ._tiled_3d_binding_ir import lower_like_tiled_3d_binding @@ -107,7 +107,7 @@ def test_strided_segment_signature_and_identity(arch, dtype): assert strided.kernel_name() == paged.kernel_name() + "_stridedkv" kernel = module.build_unified_attention_3d_tiled(strided, arch=arch) assert not verify(kernel) - signature = _3d_signature(dtype, strided_kv=True) + signature, _ = _tiled_3d_signatures(arch, dtype, strided_kv=True) assert [p.name for p in kernel.params] == [p["name"] for p in signature] assert [p.type.name.replace(" ", "") for p in kernel.params] == [ p["type"].replace(" ", "") for p in signature diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/CMakeLists.txt b/dnn-providers/hip-kernel-provider/rocke/platform/CMakeLists.txt index 37f2e0ede273..956d89785a43 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/CMakeLists.txt +++ b/dnn-providers/hip-kernel-provider/rocke/platform/CMakeLists.txt @@ -731,8 +731,9 @@ if(ROCKE_INSTALL_TESTS) # host-side SDPA sequence-length and padding-mask adapters/specs, the # fmha_mfma free-stride ABI and layout contracts, the 2D/3D mask_type spec # fields, the 3D segment+reduce module's pairing, the reduce's LDS segment - # bound against the compiled code object, and the backward-test memo bound - # in tests/conftest.py. + # bound against the compiled code object, the tiled C++ entries' head-count + # and arch refusals, the dense segment-combine launch ABI, and the + # backward-test memo bound in tests/conftest.py. # File-list style (same pattern as conv_pytest) so PYTHONPATH resolves rocke/ # correctly from the installed tree. set(_ROCKE_LIBRARY_ATTENTION_CPU_TEST_RELPATHS @@ -743,6 +744,9 @@ if(ROCKE_INSTALL_TESTS) "tests/library/tests/test_attention_tiled_2d_mask_type.py" "tests/library/tests/test_attention_tiled_3d_decode_split.py" "tests/library/tests/test_attention_tiled_3d_kv_cache_policy.py" + "tests/library/tests/test_attention_tiled_launch_signature.py" + "tests/library/tests/test_attention_tiled_cpp_entry_guards.py" + "tests/library/tests/test_attention_dense_kv_split_combine_abi.py" "tests/library/tests/test_attention_tiled_3d_mask_type.py" "tests/library/tests/test_attention_tiled_3d_module.py" "tests/library/tests/test_attention_tiled_3d_reduce_lds.py" @@ -767,6 +771,26 @@ if(ROCKE_INSTALL_TESTS) " ENVIRONMENT \"PYTHONPATH=${ROCKE_TEST_CTEST_REL}tests/library\"\n" " TIMEOUT 180)\n") + # Tiled attention cache-key contract (no GPU). The targeted share/split + # checks take seconds and run in the quick tier. The invariant sweep compares + # key equality with build equality across every selector threshold for each + # tiled arch, so each sweep case runs for tens of seconds: it is a separate + # standard-tier entry with its own budget. + string(APPEND _ROCKE_INSTALLED_CTEST_ENTRIES + "add_test(rocke_library_attention_cache_key_pytest python -m pytest \"${ROCKE_TEST_CTEST_REL}tests/library/tests/test_attention_tiled_cache_key.py\" -k \"not test_key_matches_build_identity\" -v -rs --timeout=30)\n") + string(APPEND _ROCKE_INSTALLED_CTEST_ENTRIES + "set_tests_properties(rocke_library_attention_cache_key_pytest PROPERTIES\n" + " LABELS \"ckc\\;python\\;attention\\;host\\;quick\"\n" + " ENVIRONMENT \"PYTHONPATH=${ROCKE_TEST_CTEST_REL}tests/library\"\n" + " TIMEOUT 180)\n") + string(APPEND _ROCKE_INSTALLED_CTEST_ENTRIES + "add_test(rocke_library_attention_cache_key_sweep_pytest python -m pytest \"${ROCKE_TEST_CTEST_REL}tests/library/tests/test_attention_tiled_cache_key.py\" -k test_key_matches_build_identity -v -rs --timeout=240)\n") + string(APPEND _ROCKE_INSTALLED_CTEST_ENTRIES + "set_tests_properties(rocke_library_attention_cache_key_sweep_pytest PROPERTIES\n" + " LABELS \"ckc\\;python\\;attention\\;host\\;standard\"\n" + " ENVIRONMENT \"PYTHONPATH=${ROCKE_TEST_CTEST_REL}tests/library\"\n" + " TIMEOUT 900)\n") + # DSA lightning-indexer CPU tests (no GPU: validator, dispatch wiring, and the # Python-lowered IR-sha golden). Same staged-library PYTHONPATH as the conv # lane. The on-GPU numeric tests are enrolled in the GPU lane below. @@ -1012,6 +1036,7 @@ if(ROCKE_INSTALL_TESTS) "tests/library/tests/test_gfx942_gqa_head_fold_numeric.py" "tests/library/tests/test_gfx942_4wgqa_bounds_numeric.py" "tests/library/tests/test_attention_gfx942_kv_pages.py" + "tests/library/tests/test_attention_tiled_runtime_shape_numeric.py" "tests/library/tests/test_attention_tiled_3d_kv_lds_swizzle.py" "tests/library/tests/test_attention_tiled_3d_running_max.py" "tests/library/tests/test_attention_unified_2d_masked_tile_numeric.py" diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/bindings/attention_tiled_parity_configs.py b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/bindings/attention_tiled_parity_configs.py index b15211806f7f..f69c46110918 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/bindings/attention_tiled_parity_configs.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/bindings/attention_tiled_parity_configs.py @@ -1280,6 +1280,72 @@ "use_alibi": True, "emit_lse": True, }, + { + "head_size": 64, + "block_size": 16, + "num_query_heads": 32, + "num_kv_heads": 8, + "dtype": "bf16", + "use_sinks": False, + "sliding_window": 0, + "has_softcap": False, + "num_seqs": 2, + "num_warps": 4, + "waves_per_eu": 2, + "tile_size": 32, + "block_m_per_warp": 32, + "use_mfma_32x32": True, + "use_transposed_qk_32x32": True, + "use_transposed_scalar_state": True, + "use_transposed_mask_once": True, + "use_transposed_half_local_pv": True, + "use_mfma32_skip_legacy_qreg": True, + "use_transposed_mask_limit": True, + }, + { + "head_size": 64, + "block_size": 16, + "num_query_heads": 32, + "num_kv_heads": 8, + "dtype": "bf16", + "use_sinks": False, + "sliding_window": 0, + "has_softcap": False, + "num_seqs": 2, + "tile_size": 256, + "block_m_per_warp": 32, + "use_mfma_32x32": True, + "use_transposed_qk_32x32": True, + }, + { + "head_size": 64, + "block_size": 16, + "num_query_heads": 32, + "num_kv_heads": 8, + "dtype": "bf16", + "use_sinks": False, + "sliding_window": 0, + "has_softcap": False, + "num_seqs": 2, + "tile_size": 288, + "block_m_per_warp": 32, + "use_mfma_32x32": True, + "use_transposed_qk_32x32": True, + }, + { + "head_size": 192, + "block_size": 16, + "num_query_heads": 16, + "num_kv_heads": 2, + "dtype": "bf16", + "use_sinks": False, + "sliding_window": 0, + "has_softcap": False, + "num_seqs": 4, + "waves_per_eu": 2, + "use_register_pv": True, + "tile_size": 32, + }, ] GFX942_ATTENTION_TILED_3D = [ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/attention_ext.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/attention_ext.cpp index c4fa74ed12a9..30451ad8e598 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/attention_ext.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/attention_ext.cpp @@ -129,13 +129,28 @@ rocke_value_t* rocke_wave64_reduce_sum(rocke_ir_builder_t* b, rocke_value_t* v) /* ----------------------------------------------- binary search on cu_q */ -rocke_value_t* rocke_binary_search_seq_idx(rocke_ir_builder_t* b, - rocke_value_t* cu_q, - rocke_value_t* q_block_global_idx, - rocke_value_t* num_seqs, - int block_q, - int iterations, - bool per_token) +rocke_value_t* rocke_binary_search_seq_idx_trip_count(rocke_ir_builder_t* b, + rocke_value_t* num_seqs) +{ + /* bits = b.bitcast(b.sitofp_f32(num_seqs), I32) + * exponent = b.lshr(bits, b.const_i32(23)) + * return b.smax(b.sub(exponent, b.const_i32(126)), b.const_i32(1)) */ + rocke_value_t* as_f32 = rocke_b_sitofp_f32(b, num_seqs); + rocke_value_t* bits = rocke_b_bitcast(b, as_f32, rocke_i32()); + rocke_value_t* exponent = rocke_b_lshr(b, bits, rocke_b_const_i32(b, 23)); + rocke_value_t* unbiased = rocke_b_sub(b, exponent, rocke_b_const_i32(b, 126)); + return rocke_b_smax(b, unbiased, rocke_b_const_i32(b, 1)); +} + +/* The search with a constant trip count (`upper == NULL`) or a runtime one. */ +static rocke_value_t* binary_search_seq_idx_impl(rocke_ir_builder_t* b, + rocke_value_t* cu_q, + rocke_value_t* q_block_global_idx, + rocke_value_t* num_seqs, + int block_q, + int iterations, + rocke_value_t* upper, + bool per_token) { rocke_value_t* bq; rocke_iter_arg_t iter_args[2]; @@ -162,7 +177,7 @@ rocke_value_t* rocke_binary_search_seq_idx(rocke_ir_builder_t* b, * hoist lb/ub/step first, then build the iter inits, to pin IR order. */ { rocke_value_t* lb = rocke_b_const_i32(b, 0); - rocke_value_t* ub = rocke_b_const_i32(b, (int64_t)iterations); + rocke_value_t* ub = upper != NULL ? upper : rocke_b_const_i32(b, (int64_t)iterations); rocke_value_t* step = rocke_b_const_i32(b, 1); iter_args[0].name = "left"; iter_args[0].init = rocke_b_const_i32(b, 0); @@ -216,3 +231,27 @@ rocke_value_t* rocke_binary_search_seq_idx(rocke_ir_builder_t* b, res0 = (loop.op != NULL) ? loop.op->results[0] : NULL; return rocke_b_sub(b, res0, rocke_b_const_i32(b, 1)); } + +rocke_value_t* rocke_binary_search_seq_idx(rocke_ir_builder_t* b, + rocke_value_t* cu_q, + rocke_value_t* q_block_global_idx, + rocke_value_t* num_seqs, + int block_q, + int iterations, + bool per_token) +{ + return binary_search_seq_idx_impl( + b, cu_q, q_block_global_idx, num_seqs, block_q, iterations, NULL, per_token); +} + +rocke_value_t* rocke_binary_search_seq_idx_dyn(rocke_ir_builder_t* b, + rocke_value_t* cu_q, + rocke_value_t* q_block_global_idx, + rocke_value_t* num_seqs, + int block_q, + rocke_value_t* iterations, + bool per_token) +{ + return binary_search_seq_idx_impl( + b, cu_q, q_block_global_idx, num_seqs, block_q, 0, iterations, per_token); +} diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/transforms.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/transforms.cpp index b960a6ff4706..939731e179ce 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/transforms.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/transforms.cpp @@ -1646,6 +1646,10 @@ bool rocke_transforms_descriptor_offset_i64_split(rocke_ir_builder_t* b, { const char* name = desc->base_names[i]; int stride = desc->base_strides[i]; + rocke_value_t* stride_v + = desc->is_dynamic + ? ((const rocke_dynamic_tensor_descriptor_t*)desc)->dynamic_strides[i] + : NULL; const rocke_coord_var_t* c = rocke_i_map_get(&coords, name); if(c == NULL) { @@ -1653,7 +1657,19 @@ bool rocke_transforms_descriptor_offset_i64_split(rocke_ir_builder_t* b, return false; } valid = rocke_i_and(b, valid, c->valid); - if(strcmp(name, base_coord) == 0) + if(stride_v != NULL && strcmp(name, base_coord) == 0) + { + rocke_value_t* base_val = rocke_b_to_sgpr_u32(b, c->value); + rocke_value_t* base_w = rocke_b_zext(b, base_val, rocke_i64()); + rocke_value_t* stride_w = rocke_b_zext(b, stride_v, rocke_i64()); + base_i64 = rocke_b_mul(b, base_w, stride_w); + } + else if(stride_v != NULL) + { + rocke_value_t* term = rocke_b_mul(b, c->value, stride_v); + within = (within == NULL) ? term : rocke_b_add(b, within, term); + } + else if(strcmp(name, base_coord) == 0) { /* i64 term: pin the wave-uniform block id to an SGPR before widening * (Python b.mul(b.zext(b.to_sgpr_u32(c.value), I64), const_i64(stride))). @@ -1909,54 +1925,66 @@ rocke_dynamic_tensor_descriptor_t* const char* const* coord_names, int n_coords, rocke_value_t* const* strides) +{ + /* All-Value create(): a NULL stride takes the static rule's 1. The base is + * the first field, so the cast recovers the dynamic descriptor. */ + return (rocke_dynamic_tensor_descriptor_t*)rocke_tensor_descriptor_naive_dynamic_mixed( + b, name, coord_names, n_coords, strides, NULL); +} + +rocke_tensor_descriptor_t* + rocke_tensor_descriptor_naive_dynamic_mixed(rocke_ir_builder_t* b, + const char* name, + const char* const* coord_names, + int n_coords, + rocke_value_t* const* strides, + const int* static_strides) { rocke_dynamic_tensor_descriptor_t* ddesc; - int i; - /* Fake lengths (all 0) — never used for offset computation in dynamic path */ + rocke_tensor_descriptor_t* base; int fake_lengths[8]; + int base_strides[8]; + int i; - if(!rocke_i_live(b) || n_coords < 1 || n_coords > 8) + if(!rocke_i_live(b)) + { + return NULL; + } + if(n_coords < 1 || n_coords > 8) { + rocke_i_set_err(b, + ROCKE_ERR_VALUE, + "DynamicTensorDescriptor.create: %d coords (expected 1..8)", + n_coords); return NULL; } ddesc = (rocke_dynamic_tensor_descriptor_t*)rocke_arena_calloc( &b->arena, sizeof(rocke_dynamic_tensor_descriptor_t)); if(ddesc == NULL) { + rocke_i_set_err(b, ROCKE_ERR_OOM, "DynamicTensorDescriptor.create: OOM"); return NULL; } - + /* Python create(): base_lengths all 0, base_strides the int stride or 1. */ for(i = 0; i < n_coords; ++i) { fake_lengths[i] = 0; + base_strides[i] = (strides[i] != NULL || static_strides == NULL) ? 1 : static_strides[i]; } - - /* Build the static base using fake lengths (never used for dynamic offset). */ + base = rocke_tensor_descriptor_naive( + b, name, fake_lengths, n_coords, base_strides, coord_names, n_coords); + if(base == NULL) { - /* Use row-major int strides of all-1 as placeholder. */ - int fake_strides[8]; - for(i = 0; i < n_coords; ++i) - { - fake_strides[i] = 1; - } - /* We construct the base descriptor fields directly. */ - rocke_tensor_descriptor_t* base = rocke_tensor_descriptor_naive( - b, name, fake_lengths, n_coords, fake_strides, coord_names, n_coords); - if(base == NULL) - { - return NULL; - } - ddesc->base = *base; + return NULL; } - - /* Override with runtime strides. */ + ddesc->base = *base; ddesc->n_dynamic = n_coords; for(i = 0; i < n_coords; ++i) { ddesc->dynamic_strides[i] = strides[i]; } ddesc->base.is_dynamic = 1; - return ddesc; + return &ddesc->base; } bool rocke_dynamic_tensor_descriptor_offset(rocke_ir_builder_t* b, @@ -2013,7 +2041,18 @@ bool rocke_dynamic_tensor_descriptor_offset(rocke_ir_builder_t* b, } valid = rocke_i_and(b, valid, cv->valid); stride_v = desc->dynamic_strides[i]; - term = rocke_b_mul(b, cv->value, stride_v); + if(stride_v != NULL) + { + term = rocke_b_mul(b, cv->value, stride_v); + } + else if(base->base_strides[i] == 1) + { + term = cv->value; + } + else + { + term = rocke_b_mul(b, cv->value, rocke_b_const_i32(b, (int64_t)base->base_strides[i])); + } off = (off == NULL) ? term : rocke_b_add(b, off, term); } if(off == NULL) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_helper_rocke.helpers.attention.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_helper_rocke.helpers.attention.h index c68833fcc9a9..6f1d9886df89 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_helper_rocke.helpers.attention.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_helper_rocke.helpers.attention.h @@ -9,6 +9,8 @@ * PORTED SYMBOLS (this phase): * - rocke_apply_softcap_log2 (apply_softcap_log2) * - rocke_binary_search_seq_idx (binary_search_seq_idx) + * - rocke_binary_search_seq_idx_dyn (binary_search_seq_idx, runtime trip count) + * - rocke_binary_search_seq_idx_trip_count (binary_search_seq_idx_trip_count) * - rocke_mfma_16x16x16_for_dtype (mfma_16x16x16_for_dtype) * - rocke_wave64_reduce_max (wave64_reduce_max) * - rocke_wave64_reduce_sum (wave64_reduce_sum) @@ -71,6 +73,21 @@ rocke_value_t* rocke_binary_search_seq_idx(rocke_ir_builder_t* b, int iterations, bool per_token); +/* The same search with a runtime trip count, for kernels that serve any batch. */ +rocke_value_t* rocke_binary_search_seq_idx_dyn(rocke_ir_builder_t* b, + rocke_value_t* cu_q, + rocke_value_t* q_block_global_idx, + rocke_value_t* num_seqs, + int block_q, + rocke_value_t* iterations, + bool per_token); + +/* binary_search_seq_idx_trip_count: max(1, bit_length(num_seqs)) from the exponent + * of (float)num_seqs. Exact below 2^24 sequences; past that it may add one no-op + * iteration. */ +rocke_value_t* rocke_binary_search_seq_idx_trip_count(rocke_ir_builder_t* b, + rocke_value_t* num_seqs); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_helper_rocke.instances.gfx942.attention_tiled_2d.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_helper_rocke.instances.gfx942.attention_tiled_2d.h index 8b432fb14cc1..4969371817ef 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_helper_rocke.instances.gfx942.attention_tiled_2d.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_helper_rocke.instances.gfx942.attention_tiled_2d.h @@ -208,7 +208,8 @@ int rocke_attention_tiled_2d_spec_n_blocks_per_tile(const rocke_attention_tiled_ /* dtype_ir: F16 for "fp16", BF16 otherwise (returns a rocke_type_t*). */ const rocke_type_t* rocke_attention_tiled_2d_spec_dtype_ir(const rocke_attention_tiled_2d_spec_t* s); -/* binary_search_iters: 32 if num_seqs<=0, else max(1, ceil(log2(num_seqs+1))) */ +/* binary_search_iters: 32 if num_seqs<=0, else max(1, ceil(log2(num_seqs+1))). + * Exported for existing callers; the kernels derive the search at runtime. */ int rocke_attention_tiled_2d_spec_binary_search_iters(const rocke_attention_tiled_2d_spec_t* s); /* __post_init__ validator (builder variant). Reproduces the Python validation diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.transforms.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.transforms.h index 4798299c65e4..4f2f7a912f7e 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.transforms.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.helpers.transforms.h @@ -397,7 +397,9 @@ typedef struct rocke_dynamic_tensor_descriptor { rocke_tensor_descriptor_t base; /* must be first for safe cast */ int n_dynamic; /* == base.n_base */ - rocke_value_t* dynamic_strides[8]; /* parallel to base.base_names */ + /* Parallel to base.base_names. NULL marks a compile-time stride, read from + * base.base_strides with the static rule (no multiply for 1). */ + rocke_value_t* dynamic_strides[8]; } rocke_dynamic_tensor_descriptor_t; rocke_dynamic_tensor_descriptor_t* @@ -407,6 +409,21 @@ rocke_dynamic_tensor_descriptor_t* int n_coords, rocke_value_t* const* strides); +/* Python: DynamicTensorDescriptor.create with a mix of Value and int strides. + * strides[i] == NULL takes static_strides[i] as a compile-time stride (1 when + * static_strides is NULL). Returns the descriptor as its base view (is_dynamic + * set; pass it to rocke_tensor_descriptor_transform / _offset directly), or NULL + * with the builder's error set -- the descriptor calls then no-op on the dead + * builder, so callers need not check. rocke_tensor_descriptor_naive_dynamic is + * this with every stride a Value. */ +rocke_tensor_descriptor_t* + rocke_tensor_descriptor_naive_dynamic_mixed(rocke_ir_builder_t* b, + const char* name, + const char* const* coord_names, + int n_coords, + rocke_value_t* const* strides, + const int* static_strides); + bool rocke_dynamic_tensor_descriptor_offset(rocke_ir_builder_t* b, const rocke_dynamic_tensor_descriptor_t* desc, const char* const* in_names, @@ -495,7 +512,8 @@ bool rocke_transforms_descriptor_offset(rocke_ir_builder_t* b, * terms are summed into a small i32 within-block offset. *out_base_i64 and * *out_within are written on success (within is const_i32(0) when no other * terms); *out_valid is NULL == Python None. Returns false (builder error set) - * if base_coord is not among the descriptor's base coords. */ + * if base_coord is not among the descriptor's base coords. A dynamic descriptor + * multiplies by its runtime strides, widening the base_coord one to i64. */ bool rocke_transforms_descriptor_offset_i64_split(rocke_ir_builder_t* b, const rocke_tensor_descriptor_t* desc, const char* base_coord, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.instances.common._fmha_common.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.instances.common._fmha_common.h index d0e0dfd9922e..ee5ae7fa4c6b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.instances.common._fmha_common.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/helper_rocke.instances.common._fmha_common.h @@ -175,6 +175,17 @@ static inline bool rocke_fmha_common_spec_use_custom_mask(const rocke_fmha_commo * ("ok" on the accept path); pass a live `arena`. The reason strings reproduce * the Python messages byte-for-byte. On a formatting OOM the predicate result is * still returned faithfully and the reason is a best-effort static fallback. */ +/* Python: kernels.common._fmha_common.check_head_counts. True when both head + * counts are positive and num_query_heads splits evenly over num_kv_heads; + * otherwise false, with the Python reason text written to `reason`. The one + * head-count rule: rocke_fmha_validate_common_spec and every tiled attention + * entry run it (the tiled entries before naming the kernel, whose name + * carries the GQA ratio). */ +bool rocke_attention_check_head_counts(int num_query_heads, + int num_kv_heads, + char* reason, + size_t reason_cap); + bool rocke_fmha_validate_common_spec(rocke_arena_t* arena, const rocke_fmha_common_spec_t* spec, const char** out_reason); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx942_attention_tiled_2d_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx942_attention_tiled_2d_internal.h index 55c8ebbeb4a2..b757130d79ba 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx942_attention_tiled_2d_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx942_attention_tiled_2d_internal.h @@ -178,6 +178,10 @@ typedef struct rocke_gfx942_attn2d_build_ctx rocke_value_t* num_seqs_p; rocke_value_t* bt_stride_p; rocke_value_t* qq_bias_stride0_p; + /* RUNTIME_KERNARG_FIELDS params, in declaration order. */ + rocke_value_t* num_qh_p; + rocke_value_t* num_kv_p; + rocke_value_t* sliding_window_p; /* Trailing lse_ptr (F32*), declared only when spec->emit_lse; else NULL. */ rocke_value_t* lse; @@ -271,20 +275,21 @@ typedef struct rocke_gfx942_attn2d_build_ctx rocke_value_t* lane_rg_is2_v; /* ---- SSA constants (lines 1891-1912) ---- */ - rocke_value_t* c0; rocke_value_t* zero_f; rocke_value_t* neg_inf_v; /* b.const_f32(-inf) (1892) */ rocke_value_t* one_f_v; /* b.const_f32(1.0) (1894) */ rocke_value_t* rcp_ln2_v; /* b.const_f32(1.4426950408889634) (1895) */ rocke_value_t* qk_scale_v; /* derived from scale_p */ - rocke_value_t* sw_const_v; /* b.const_i32(SLIDING_WINDOW) (1910) */ + rocke_value_t* sw_const_v; /* sliding_window_p, else const_i32(SLIDING_WINDOW) */ /* ---- paged-KV byte descriptor (full transform DAG, lines 2163-2351) ---- */ rocke_tensor_descriptor_t* q_desc; /* output/query [token,head,dim] */ rocke_tensor_descriptor_t* kv_desc; /* paged K/V byte descriptor */ rocke_value_t* seq_base; /* to_sgpr_u32(seq_idx*bt_stride) */ rocke_value_t* block_table_max_idx; /* to_sgpr_u32(num_seqs*bt_stride) */ - rocke_value_t* kv_block_bytes_c_v; /* const BS*NUM_KV*HD*KV_BYTES (2227) */ + rocke_value_t* kv_block_bytes_c_v; /* one-block bound: kv_stride_blk_b_v, or its const */ + rocke_value_t* kv_stride_tok_b_v; /* num_kv_p*HD*KV_BYTES */ + rocke_value_t* kv_stride_blk_b_v; /* kv_stride_tok_b_v*BS */ rocke_value_t* lane_half_base_v; /* tid * KV_HALVES_PER_LANE (2232) */ rocke_value_t* zero_soff_v; /* const_i32(0) soffset (2238) */ rocke_value_t* wave_lds_off_i64_v; /* wave_lds_offset_i64 (2254/2263) */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx942_attention_tiled_3d.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx942_attention_tiled_3d.h index 6729eff640d4..95bf391927cd 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx942_attention_tiled_3d.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx942_attention_tiled_3d.h @@ -34,9 +34,9 @@ * with a 4-reg-per-lane accumulator decoded via _mfma_16x16_c_row. * * REDUCE KERNEL (build_unified_attention_reduce_tiled, lines 1002-1133). Arch- - * neutral, pure f32 load / exp2 / store (byte-for-byte port of the gfx950 - * reduce): 64 threads, single active per (q_token, q_head), three passes (max, - * combine, normalize). + * neutral, pure f32 load / exp2 / store (the gfx950 reduce with a runtime + * query head count): 64 threads, single active per (q_token, q_head), three + * passes (max, combine, normalize). * * REUSED HELPER PORTS (no IR re-emitted here): * rocke/helper_helper_rocke.helpers.attention.h -- apply_softcap_log2, @@ -202,7 +202,8 @@ int rocke_unified_attention_3d_tiled_spec_tile_size( /* dtype_ir: F16 for "fp16", BF16 otherwise (returns a rocke_type_t*). */ const rocke_type_t* rocke_unified_attention_3d_tiled_spec_dtype_ir( const rocke_unified_attention_3d_tiled_spec_t* s); -/* binary_search_iters: 32 if num_seqs<=0, else max(1, ceil(log2(num_seqs+1))) */ +/* binary_search_iters: 32 if num_seqs<=0, else max(1, ceil(log2(num_seqs+1))). + * Exported for existing callers; the kernels derive the search at runtime. */ int rocke_unified_attention_3d_tiled_spec_binary_search_iters( const rocke_unified_attention_3d_tiled_spec_t* s); /* kernel_name(): kernel_name_join(...) into caller's buf (NUL-terminated, up to diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx942_attention_tiled_3d_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx942_attention_tiled_3d_internal.h index 1f58dc76c378..aee975d2ad90 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx942_attention_tiled_3d_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx942_attention_tiled_3d_internal.h @@ -122,7 +122,6 @@ typedef struct rocke_gfx942_attn_tiled_3d_config bool Q_IN_V_SLOT; /* HD == 192 && T >= BLOCK_M: stage Q through V_lds */ int THREADS; /* 64 */ - int binary_search_iters; /* spec.binary_search_iters */ /* ---- async / wide / fp8 KV feed geometry (lines 549-667) ---- */ int ASYNC_LDS_DWORDS; /* 1 */ @@ -226,6 +225,10 @@ typedef struct rocke_gfx942_attention_tiled_3d_build_ctx rocke_value_t* num_seqs_p; /* I32 num_seqs */ rocke_value_t* bt_stride_p; /* I32 block_table_stride */ rocke_value_t* qq_bias_stride0_p; /* I32 qq_bias_stride_0 */ + /* RUNTIME_KERNARG_FIELDS params, in declaration order. */ + rocke_value_t* num_qh_p; /* I32 num_query_heads */ + rocke_value_t* num_kv_p; /* I32 num_kv_heads */ + rocke_value_t* sliding_window_p; /* I32 sliding_window */ /* ---------- grid ids + thread (lines 331-334) ---------- */ rocke_value_t* q_block_global_idx; /* block_id_x() */ @@ -258,7 +261,7 @@ typedef struct rocke_gfx942_attention_tiled_3d_build_ctx rocke_value_t* one_f; /* const_f32(1.0) */ rocke_value_t* rcp_ln2; /* const_f32(1.4426950408889634) */ rocke_value_t* qk_scale; /* scale_p * rcp_ln2 */ - rocke_value_t* sw_const; /* const_i32(SLIDING_WINDOW) */ + rocke_value_t* sw_const; /* sliding_window_p */ rocke_value_t* z8; /* zero_vec(dtype, 8) */ /* ---------- lane decode (lines 482-483) ---------- */ @@ -281,6 +284,8 @@ typedef struct rocke_gfx942_attention_tiled_3d_build_ctx rocke_value_t* big_bytes; /* const_i32(0x7FFF0000) */ rocke_value_t* key_rsrc; /* buffer_rsrc(key, big_bytes) */ rocke_value_t* value_rsrc; /* buffer_rsrc(value, big_bytes) */ + rocke_value_t* kv_stride_tok_b; /* num_kv_p * HD * KV_BYTES */ + rocke_value_t* kv_stride_blk_b; /* kv_stride_tok_b * BS */ rocke_value_t* lane_half_base; /* tid * HALVES_PER_LANE */ rocke_value_t* K_lds_addr; /* smem_addr_of(K_lds) */ rocke_value_t* V_lds_addr; /* smem_addr_of(V_lds) */ @@ -321,6 +326,7 @@ typedef struct rocke_gfx942_attention_tiled_3d_build_ctx rocke_value_t* seg_max; /* F32* readonly */ rocke_value_t* seg_l; /* F32* readonly (segm_expsum) */ rocke_value_t* red_seq_lens; /* I32* readonly (_seq_lens, unused body) */ + rocke_value_t* red_num_qh_p; /* I32 num_query_heads (RUNTIME_KERNARG_FIELDS) */ rocke_value_t* red_lse; /* F32* noalias writeonly (lse_ptr); NULL unless emit_lse */ /* ---------- grid ids (lines 1032-1034) ---------- */ @@ -403,8 +409,8 @@ rocke_value_t* rocke_gfx942_attention_tiled_3d_tile_base(rocke_gfx942_attention_tiled_3d_build_ctx_t* ctx, rocke_value_t* page_id, rocke_value_t* kv_tile_idx, - int page_stride, - int token_stride); + rocke_value_t* page_stride, + rocke_value_t* token_stride); /* _load_page_id_window(rel_page): lane l loads the id of the segment's page * rel_page + l (0 past the sequence's last page). */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx950_attention_tiled_2d_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx950_attention_tiled_2d_internal.h index 61901bdbd5ee..8665f4c206c4 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx950_attention_tiled_2d_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx950_attention_tiled_2d_internal.h @@ -87,6 +87,9 @@ extern "C" { #define ROCKE_GFX950_ATTN2D_MAX_ACCS 64 /* Online-softmax iter-args: 2*SOFTMAX_STATE_SLOTS (m,l per slot) + ACCS + cur_buf. */ #define ROCKE_GFX950_ATTN2D_MAX_ITER_ARGS 96 +/* Most pages per tile whose ids the KV loop carries; larger tiles read them in + * place (Python's PAGE_PREFETCH applies the same bound). */ +#define ROCKE_GFX950_ATTN2D_MAX_PREFETCH_PAGES 16 /* Per-lane register slots (REGS_PER_LANE): 4 (M=16), 8 (M=32), 16 (32x32). */ #define ROCKE_GFX950_ATTN2D_MAX_REGS_PER_LANE 16 /* QK / PV K-iters (HD/QK_K_STEP or T/PV_K_STEP); HD<=256, T<=128 -> <=16. */ @@ -117,7 +120,7 @@ typedef struct rocke_gfx950_attn2d_build_ctx /* ---- ALL-CAPS geometry constants (lines 754-847) ---- */ int HD, T, BS, N_BLOCKS_PER_TILE; - int BLOCK_M, BLOCK_Q, NQK, NUM_KV, NUM_QH; + int BLOCK_M, BLOCK_Q, NQK, NUM_KV; /* NUM_KV only for the fast-desc 64/8 pin */ int VALID_ROWS; /* BLOCK_Q * NQK -- rows that map to complete GQA groups */ int SLIDING_WINDOW; bool USE_SOFTCAP, USE_SINKS, USE_ALIBI, USE_QQ_BIAS; @@ -175,6 +178,10 @@ typedef struct rocke_gfx950_attn2d_build_ctx rocke_value_t* num_seqs_p; rocke_value_t* bt_stride_p; rocke_value_t* qq_bias_stride0_p; + /* RUNTIME_KERNARG_FIELDS params, in declaration order. */ + rocke_value_t* num_qh_p; + rocke_value_t* num_kv_p; + rocke_value_t* sliding_window_p; rocke_value_t* lse; /* lse_ptr (spec->emit_lse only, else NULL) */ /* ---- grid ids + wave decomposition ---- */ @@ -247,21 +254,41 @@ typedef struct rocke_gfx950_attn2d_build_ctx rocke_value_t* lane_rg_is2_v; /* ---- SSA constants ---- */ - rocke_value_t* c0; rocke_value_t* zero_f; rocke_value_t* neg_inf_v; /* b.const_f32(-inf) */ rocke_value_t* one_f_v; /* b.const_f32(1.0) */ rocke_value_t* rcp_ln2_v; /* b.const_f32(1.4426950408889634) */ rocke_value_t* qk_scale_v; /* derived from scale_p */ rocke_value_t* pv_fp8_scale_v; /* fdiv(v_scale, 240) when FP8_MFMA_PV (line 1149) */ - rocke_value_t* sw_const_v; /* b.const_i32(SLIDING_WINDOW) */ + rocke_value_t* sw_const_v; /* the sliding_window kernarg */ /* ---- paged-KV byte descriptor (full transform DAG, lines 1163-1630) ---- */ rocke_tensor_descriptor_t* q_desc; /* output/query [token,head,dim] */ rocke_tensor_descriptor_t* kv_desc; /* paged K/V byte descriptor */ rocke_value_t* seq_base; /* to_sgpr_u32(seq_idx*bt_stride) */ rocke_value_t* block_table_max_idx; /* to_sgpr_u32(num_seqs*bt_stride) */ - rocke_value_t* kv_block_bytes_c_v; /* const BS*NUM_KV*HD*KV_BYTES */ + rocke_value_t* kv_stride_tok_b_v; /* num_kv_p*HD*KV_BYTES */ + rocke_value_t* kv_stride_blk_b_v; /* kv_stride_tok_b_v*BS */ + rocke_value_t* kv_block_bytes_c_v; /* kv_stride_blk_b_v (one block) */ + /* bf16/fp16 loaders (!KV_FP8, !FAST_PAGED_KV_DESC): the lane's in-page byte + * offset, each copy's first-token offset in its page (indexed by that + * copy within the page, BS / copy tokens entries on the arena; [0] stays + * NULL), and the i64 page stride (I64_KV_ADDR). */ + rocke_value_t* lane_within_b_v; + rocke_value_t** copy_tok_b; + rocke_value_t* kv_stride_blk_i64_v; + /* Page ids the KV loop carries when page copies prefetch them (one per page + * of a tile), with the tile index value each set belongs to: the first tile + * and the one after it (prologue), the current and next tile, and the tile + * after next read one iteration ahead. */ + rocke_value_t* page_start_tile; + rocke_value_t* page_start[ROCKE_GFX950_ATTN2D_MAX_PREFETCH_PAGES]; + rocke_value_t* page_start_next[ROCKE_GFX950_ATTN2D_MAX_PREFETCH_PAGES]; + rocke_value_t* page_cur_tile; + rocke_value_t* page_cur[ROCKE_GFX950_ATTN2D_MAX_PREFETCH_PAGES]; + rocke_value_t* page_next_tile; + rocke_value_t* page_next[ROCKE_GFX950_ATTN2D_MAX_PREFETCH_PAGES]; + rocke_value_t* page_after_next[ROCKE_GFX950_ATTN2D_MAX_PREFETCH_PAGES]; rocke_value_t* lane_half_base_v; /* tid * KV_HALVES_PER_LANE */ rocke_value_t* zero_soff_v; /* const_i32(0) soffset */ rocke_value_t* wave_lds_off_i64_v; /* wave_lds_offset_i64 */ @@ -422,6 +449,13 @@ void rocke_gfx950_attn2d_issue_k_load_runtime(rocke_gfx950_attn2d_build_ctx_t* c void rocke_gfx950_attn2d_issue_v_load_runtime(rocke_gfx950_attn2d_build_ctx_t* ctx, rocke_value_t* kv_tile_idx, rocke_value_t* buf_idx); +/* True when the bf16/fp16 loaders use per-page copy offsets and the KV loop + * carries the page ids, reading each tile's ids one iteration ahead. */ +bool rocke_gfx950_attn2d_page_prefetch(const rocke_gfx950_attn2d_build_ctx_t* ctx); +/* The page ids of every page of a tile: one guarded block-table read each. */ +void rocke_gfx950_attn2d_tile_page_ids(rocke_gfx950_attn2d_build_ctx_t* ctx, + rocke_value_t* kv_tile_idx, + rocke_value_t** out); void rocke_gfx950_attn2d_issue_k(rocke_gfx950_attn2d_build_ctx_t* ctx, rocke_value_t* tile_idx, rocke_value_t* buf_idx); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx950_attention_tiled_3d.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx950_attention_tiled_3d.h index b674a5c2b2a3..53b11b0203fa 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx950_attention_tiled_3d.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx950_attention_tiled_3d.h @@ -283,7 +283,8 @@ int rocke_gfx950_unified_attention_3d_tiled_spec_tile_size( /* dtype_ir: F16 for "fp16", BF16 otherwise (returns a rocke_type_t*). */ const rocke_type_t* rocke_gfx950_unified_attention_3d_tiled_spec_dtype_ir( const rocke_unified_attention_3d_tiled_spec_t* s); -/* binary_search_iters: 32 if num_seqs<=0, else max(1, ceil(log2(num_seqs+1))) */ +/* binary_search_iters: 32 if num_seqs<=0, else max(1, ceil(log2(num_seqs+1))). + * Exported for existing callers; the kernels derive the search at runtime. */ int rocke_gfx950_unified_attention_3d_tiled_spec_binary_search_iters( const rocke_unified_attention_3d_tiled_spec_t* s); /* kernel_name(): kernel_name_join(...) into caller's buf (NUL-terminated, up to diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx950_attention_tiled_3d_internal.h b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx950_attention_tiled_3d_internal.h index 69228aa7003a..b1411537a82d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx950_attention_tiled_3d_internal.h +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/instance_gfx950_attention_tiled_3d_internal.h @@ -129,8 +129,6 @@ typedef struct rocke_gfx950_attn_tiled_3d_config int BLOCK_M; /* spec.block_m (== 16) */ int BLOCK_Q; /* spec.block_q */ int NQK; /* spec.num_queries_per_kv */ - int NUM_KV; /* spec.num_kv_heads */ - int NUM_QH; /* spec.num_query_heads */ int NUM_SEG; /* spec.num_segments */ int SLIDING_WINDOW; /* spec.sliding_window */ bool USE_SOFTCAP; /* spec.has_softcap */ @@ -153,7 +151,6 @@ typedef struct rocke_gfx950_attn_tiled_3d_config int PV_N_TILES; /* HD // MFMA_N */ int THREADS; /* 64 */ - int binary_search_iters; /* spec.binary_search_iters */ /* ---- async / fp8 KV feed geometry (lines 565-643) ---- */ int ASYNC_LDS_DWORDS; /* 4 (CDNA4) */ @@ -161,8 +158,6 @@ typedef struct rocke_gfx950_attn_tiled_3d_config int KV_HALVES_PER_CALL; /* THREADS * 8 */ int kv_calls_per_tile; /* (T*HD) // KV_HALVES_PER_CALL */ int bytes_per_call; /* KV_HALVES_PER_CALL * 2 */ - int kv_stride_blk_b; /* BS*NUM_KV*HD*KV_BYTES */ - int kv_stride_tok_b; /* NUM_KV*HD*KV_BYTES */ int kv_stride_h_b; /* HD*KV_BYTES */ int bytes_per_buf; /* T*HD*2 */ int N_BLOCKS_PER_TILE; /* T // BS (multi-page tile when > 1) */ @@ -252,6 +247,10 @@ typedef struct rocke_gfx950_attention_tiled_3d_build_ctx rocke_value_t* num_seqs_p; /* I32 num_seqs */ rocke_value_t* bt_stride_p; /* I32 block_table_stride */ rocke_value_t* qq_bias_stride0_p; /* I32 qq_bias_stride_0 */ + /* RUNTIME_KERNARG_FIELDS params, in declaration order. */ + rocke_value_t* num_qh_p; /* I32 num_query_heads */ + rocke_value_t* num_kv_p; /* I32 num_kv_heads */ + rocke_value_t* sliding_window_p; /* I32 sliding_window */ /* ---------- grid ids + thread (lines 360-363) ---------- */ rocke_value_t* q_block_global_idx; /* block_id_x() */ @@ -284,7 +283,7 @@ typedef struct rocke_gfx950_attention_tiled_3d_build_ctx rocke_value_t* one_f; /* const_f32(1.0) */ rocke_value_t* rcp_ln2; /* const_f32(1.4426950408889634) */ rocke_value_t* qk_scale; /* scale_p * rcp_ln2 */ - rocke_value_t* sw_const; /* const_i32(SLIDING_WINDOW) */ + rocke_value_t* sw_const; /* the sliding_window kernarg */ rocke_value_t* z8; /* zero_vec(dtype, 8) */ /* ---------- lane decode (lines 533-534) ---------- */ @@ -326,8 +325,14 @@ typedef struct rocke_gfx950_attention_tiled_3d_build_ctx rocke_value_t* first_page_ids[8]; /* tile_start's page ids (PAGE_IDS; N <= 8) */ rocke_value_t* seg_first_page; /* tile_start * N_BLOCKS_PER_TILE (WAVE_PAGE_ID_LOAD) */ rocke_value_t* first_page_id_window; /* the first 64-page id window (WAVE_PAGE_ID_LOAD) */ - rocke_value_t* - kv_block_bytes_c; /* const_i32(kv_stride_blk_b): 1-block buffer bound (i64 path) */ + /* K/V byte strides; they scale with the runtime kv head count. */ + rocke_value_t* kv_stride_tok_b; /* num_kv_p * HD * KV_BYTES */ + rocke_value_t* kv_stride_blk_b; /* kv_stride_tok_b * BS */ + rocke_value_t* kv_block_bytes_c; /* kv_stride_blk_b: 1-block buffer bound (i64 path) */ + /* Each call's token offset within a page (PAGE_IDS), indexed by + * tok0 / tokens_per_call; [0] stays NULL (no offset). */ + rocke_value_t* call_tok_b[64]; + rocke_value_t* kv_stride_blk_i64; /* zext(kv_stride_blk_b) (PAGE_IDS && I64_KV_ADDR) */ /* ---------- online-softmax loop init carry (lines 536-562, 717-718) ---------- * * m_init: the lane's query row (l starts at one_f). acc_inits: PV_N_TILES diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/_fmha_common.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/_fmha_common.cpp index df4baf078c9e..1d9364bb7e64 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/_fmha_common.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/_fmha_common.cpp @@ -8,6 +8,7 @@ */ #include "rocke/helper_rocke.instances.common._fmha_common.h" +#include #include #include "rocke/helper_rocke.helpers.io.h" /* rocke_b_io_ir_type */ @@ -100,6 +101,23 @@ rocke_fmha_common_spec_t rocke_fmha_common_spec_default(rocke_fmha_shape_t shape /* Set *out_reason (if non-NULL) to an arena-printf'd message, falling back to * `fallback` on OOM. Returns `ret` so callers can `return reject(...)`. */ +bool rocke_attention_check_head_counts(int num_query_heads, + int num_kv_heads, + char* reason, + size_t reason_cap) +{ + const char* fmt = NULL; + if(num_query_heads <= 0 || num_kv_heads <= 0) + fmt = "num_query_heads / num_kv_heads must be > 0 (got %d, %d)"; + else if(num_query_heads % num_kv_heads != 0) + fmt = "num_query_heads (%d) must be divisible by num_kv_heads (%d) for GQA / MQA"; + if(fmt == NULL) + return true; + if(reason != NULL && reason_cap > 0) + (void)snprintf(reason, reason_cap, fmt, num_query_heads, num_kv_heads); + return false; +} + static bool rocke_i_fmha_set_reason(rocke_arena_t* arena, const char** out_reason, bool ret, @@ -157,33 +175,18 @@ bool rocke_fmha_validate_common_spec(rocke_arena_t* arena, s->head_size), "head_size not in the supported set"); } - /* if s.num_query_heads <= 0 or s.num_kv_heads <= 0: */ - if(s->num_query_heads <= 0 || s->num_kv_heads <= 0) + /* ok, reason = check_head_counts(s.num_query_heads, s.num_kv_heads) */ { - return rocke_i_fmha_set_reason( - arena, - out_reason, - false, - rocke_arena_printf(arena, - "num_query_heads / num_kv_heads must be > 0 " - "(got %d, %d)", - s->num_query_heads, - s->num_kv_heads), - "num_query_heads / num_kv_heads must be > 0"); - } - /* if s.num_query_heads % s.num_kv_heads != 0: */ - if((s->num_query_heads % s->num_kv_heads) != 0) - { - return rocke_i_fmha_set_reason( - arena, - out_reason, - false, - rocke_arena_printf(arena, - "num_query_heads (%d) must be divisible by " - "num_kv_heads (%d) for GQA / MQA", - s->num_query_heads, - s->num_kv_heads), - "num_query_heads must be divisible by num_kv_heads"); + char why[160]; + if(!rocke_attention_check_head_counts( + s->num_query_heads, s->num_kv_heads, why, sizeof(why))) + { + return rocke_i_fmha_set_reason(arena, + out_reason, + false, + rocke_arena_printf(arena, "%s", why), + "num_query_heads / num_kv_heads not a valid GQA pair"); + } } /* if s.block_size_q not in (16, 32, 64, 128): */ if(s->block_size_q != 16 && s->block_size_q != 32 && s->block_size_q != 64 diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_ctx_prologue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_ctx_prologue.cpp index fd9c05df1dae..a221662cb4c0 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_ctx_prologue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_ctx_prologue.cpp @@ -398,6 +398,9 @@ bool rocke_gfx942_attn2d_build_ctx_init(rocke_gfx942_attn2d_build_ctx_t* ctx, ctx->num_seqs_p = rocke_b_param(b, "num_seqs", rocke_i32(), NULL); ctx->bt_stride_p = rocke_b_param(b, "block_table_stride", rocke_i32(), NULL); ctx->qq_bias_stride0_p = rocke_b_param(b, "qq_bias_stride_0", rocke_i32(), NULL); + ctx->num_qh_p = rocke_b_param(b, "num_query_heads", rocke_i32(), NULL); + ctx->num_kv_p = rocke_b_param(b, "num_kv_heads", rocke_i32(), NULL); + ctx->sliding_window_p = rocke_b_param(b, "sliding_window", rocke_i32(), NULL); /* Appended LAST and only when requested, so the LSE-off ABI is unchanged. */ if(spec->emit_lse) { @@ -439,14 +442,11 @@ bool rocke_gfx942_attn2d_build_ctx_init(rocke_gfx942_attn2d_build_ctx_t* ctx, } /* ---- seq lookup + Q-block geometry (lines 1390-1409) ---- */ - ctx->seq_idx - = rocke_binary_search_seq_idx(b, - ctx->cu_q, - ctx->q_block_global_idx, - ctx->num_seqs_p, - BLOCK_Q, - rocke_attention_tiled_2d_spec_binary_search_iters(spec), - false); + { + rocke_value_t* trip = rocke_binary_search_seq_idx_trip_count(b, ctx->num_seqs_p); + ctx->seq_idx = rocke_binary_search_seq_idx_dyn( + b, ctx->cu_q, ctx->q_block_global_idx, ctx->num_seqs_p, BLOCK_Q, trip, false); + } ctx->cu_q_start = rocke_b_global_load_i32(b, ctx->cu_q, ctx->seq_idx, 0); ctx->cu_q_stop = rocke_b_global_load_i32( b, ctx->cu_q, rocke_b_add(b, ctx->seq_idx, rocke_b_const_i32(b, 1)), 0); @@ -742,13 +742,9 @@ bool rocke_gfx942_attn2d_build_ctx_init(rocke_gfx942_attn2d_build_ctx_t* ctx, qk_scale = rocke_b_fmul(b, qk_scale, ctx->k_scale_p); ctx->qk_scale_v = qk_scale; } - /* sw_const = b.const_i32(int(SLIDING_WINDOW)) (line 1910). Python creates - * this once here in the constants block and REUSES it in the tile-bound - * (line 1994) and mask (lines 4111/4328/4403) regions. The previous port - * created an unused i32 0 here and a FRESH const in the loop scaffold, - * allocating an extra SSA value that shifted every later %N. Cache it. */ - ctx->sw_const_v = rocke_b_const_i32(b, ctx->SLIDING_WINDOW); - ctx->c0 = ctx->sw_const_v; + /* sw_const = rt["sliding_window"]. Python binds it once here in the + * constants block and REUSES it in the tile-bound and mask regions. */ + ctx->sw_const_v = ctx->sliding_window_p; /* ---- acc geometry the epilogue aliases (lines 2142-2143) ---- */ ctx->ACC_N_TILES = USE_MFMA_32X32 ? (HD / 32) : PV_N_TILES; @@ -1006,7 +1002,7 @@ void rocke_gfx942_attn2d_emit_licm_hoist(rocke_gfx942_attn2d_build_ctx_t* ctx) /* row_ok = (qp_r < q_len) && (qh_r < NUM_QH) [&& (row < VALID_ROWS)]. * Skip the VALID_ROWS guard when VALID_ROWS == BLOCK_M (NQK divides BLOCK_M). */ rocke_value_t* row_ok_pos = rocke_b_cmp_lt(b, qp_r, ctx->cur_batch_q_len); - rocke_value_t* row_ok_qh = rocke_b_cmp_lt(b, qh_r, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* row_ok_qh = rocke_b_cmp_lt(b, qh_r, ctx->num_qh_p); rocke_value_t* row_ok_inner = rocke_b_land(b, row_ok_pos, row_ok_qh); rocke_value_t* row_ok; if(ctx->VALID_ROWS < ctx->BLOCK_M) @@ -1034,7 +1030,7 @@ void rocke_gfx942_attn2d_emit_licm_hoist(rocke_gfx942_attn2d_build_ctx_t* ctx) for(reg = 0; reg < ctx->REGS_PER_LANE; ++reg) { rocke_value_t* qh_r = ctx->hoist_q_head[reg]; - rocke_value_t* qh_ok = rocke_b_cmp_lt(b, qh_r, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* qh_ok = rocke_b_cmp_lt(b, qh_r, ctx->num_qh_p); rocke_value_t* slope = rocke_b_masked_global_load( b, ctx->alibi_slopes_ptr, qh_r, qh_ok, rocke_b_const_f32(b, 0.0), f32, 4); /* The softmax bucket reads the per-reg slope from hoist_q_head. */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_epilogue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_epilogue.cpp index cf991e78e03d..1fb7c7b97637 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_epilogue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_epilogue.cpp @@ -58,7 +58,7 @@ static void epilogue_store_row_lse(rocke_gfx942_attn2d_build_ctx_t* ctx, rocke_value_t* qh_mod = rocke_b_mod(b, row, rocke_b_const_i32(b, ctx->NQK)); rocke_value_t* qh = rocke_b_add(b, qh_mul, qh_mod); rocke_value_t* ok_pos = rocke_b_cmp_lt(b, pos, ctx->cur_batch_q_len); - rocke_value_t* ok_qh = rocke_b_cmp_lt(b, qh, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* ok_qh = rocke_b_cmp_lt(b, qh, ctx->num_qh_p); rocke_value_t* ok = rocke_b_land(b, ok_pos, ok_qh); if(ctx->VALID_ROWS < ctx->BLOCK_M) { @@ -70,7 +70,7 @@ static void epilogue_store_row_lse(rocke_gfx942_attn2d_build_ctx_t* ctx, rocke_b_region_enter(b, iff.then_region); { rocke_value_t* tok = rocke_b_add(b, ctx->cu_q_start, pos); - rocke_value_t* tok_mul = rocke_b_mul(b, tok, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* tok_mul = rocke_b_mul(b, tok, ctx->num_qh_p); rocke_value_t* offset = rocke_b_add(b, tok_mul, qh); rocke_value_t* has_mass = rocke_lse_row_has_mass(b, l_final); rocke_lse_store(b, ctx->lse, offset, m_final, l_final, has_mass); @@ -99,7 +99,6 @@ rocke_kernel_def_t* rocke_gfx942_attn2d_emit_epilogue(rocke_gfx942_attn2d_build_ const int ACC_M_ATOMS = ctx->ACC_M_ATOMS; const int REGS_PER_LANE = ctx->REGS_PER_LANE; const int NQK = ctx->NQK; - const int NUM_QH = ctx->NUM_QH; const int BLOCK_M = ctx->BLOCK_M; const int THREADS = ctx->THREADS; const int MFMA_N = 16; /* module-level constant MFMA_N (attention_tiled_2d.py:81) */ @@ -173,7 +172,7 @@ rocke_kernel_def_t* rocke_gfx942_attn2d_emit_epilogue(rocke_gfx942_attn2d_build_ rocke_value_t* op_qh_mod = rocke_b_mod(b, q_row_t, rocke_b_const_i32(b, NQK)); rocke_value_t* op_qh_t = rocke_b_add(b, op_qh_mul, op_qh_mod); rocke_value_t* op_ok_pos = rocke_b_cmp_lt(b, op_pos_t, cur_batch_q_len); - rocke_value_t* op_ok_qh = rocke_b_cmp_lt(b, op_qh_t, rocke_b_const_i32(b, NUM_QH)); + rocke_value_t* op_ok_qh = rocke_b_cmp_lt(b, op_qh_t, ctx->num_qh_p); rocke_value_t* op_ok_inner = rocke_b_land(b, op_ok_pos, op_ok_qh); /* Skip the VALID_ROWS guard when VALID_ROWS == BLOCK_M (NQK divides BLOCK_M). */ rocke_value_t* op_mask_t; @@ -208,7 +207,7 @@ rocke_kernel_def_t* rocke_gfx942_attn2d_emit_epilogue(rocke_gfx942_attn2d_build_ rocke_b_region_enter(b, lse_if.then_region); { rocke_value_t* tok = rocke_b_add(b, cu_q_start, op_pos_t); - rocke_value_t* tok_mul = rocke_b_mul(b, tok, rocke_b_const_i32(b, NUM_QH)); + rocke_value_t* tok_mul = rocke_b_mul(b, tok, ctx->num_qh_p); rocke_value_t* offset = rocke_b_add(b, tok_mul, op_qh_t); rocke_value_t* has_mass = rocke_lse_row_has_mass(b, ctx->l_final[0]); rocke_lse_store(b, ctx->lse, offset, m_final[0], ctx->l_final[0], has_mass); @@ -269,7 +268,7 @@ rocke_kernel_def_t* rocke_gfx942_attn2d_emit_epilogue(rocke_gfx942_attn2d_build_ rocke_value_t* op_qh32_base = rocke_b_add(b, op_qh32_mul, op_qh32_mod); /* Python emits the pos cmp BEFORE the qh cmp, then AND's VALID_ROWS; bind in order. */ rocke_value_t* op_ok_pos32 = rocke_b_cmp_lt(b, op_pos32_base, cur_batch_q_len); - rocke_value_t* op_ok_qh32 = rocke_b_cmp_lt(b, op_qh32_base, rocke_b_const_i32(b, NUM_QH)); + rocke_value_t* op_ok_qh32 = rocke_b_cmp_lt(b, op_qh32_base, ctx->num_qh_p); rocke_value_t* op_ok_inner32 = rocke_b_land(b, op_ok_pos32, op_ok_qh32); /* Skip the VALID_ROWS guard when VALID_ROWS == BLOCK_M (NQK divides BLOCK_M). */ rocke_value_t* op_mask32_base; @@ -381,7 +380,7 @@ rocke_kernel_def_t* rocke_gfx942_attn2d_emit_epilogue(rocke_gfx942_attn2d_build_ rocke_value_t* op_qh_mod = rocke_b_mod(b, OUT_ROW_BASE, rocke_b_const_i32(b, NQK)); rocke_value_t* op_qh = rocke_b_add(b, op_qh_mul, op_qh_mod); rocke_value_t* op_mask_pos = rocke_b_cmp_lt(b, op_pos, cur_batch_q_len); - rocke_value_t* op_mask_qh = rocke_b_cmp_lt(b, op_qh, rocke_b_const_i32(b, NUM_QH)); + rocke_value_t* op_mask_qh = rocke_b_cmp_lt(b, op_qh, ctx->num_qh_p); rocke_value_t* op_mask_inner = rocke_b_land(b, op_mask_pos, op_mask_qh); /* Skip the VALID_ROWS guard when VALID_ROWS == BLOCK_M (NQK divides BLOCK_M). */ rocke_value_t* op_mask; diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_kv_body_qk_softmax.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_kv_body_qk_softmax.cpp index d087b7380a5d..e6742736461d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_kv_body_qk_softmax.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_kv_body_qk_softmax.cpp @@ -384,7 +384,7 @@ void rocke_gfx942_attn2d_emit_kv_body(rocke_gfx942_attn2d_build_ctx_t* ctx) /* row_ok = land(qpcur_batch_q_len); - rocke_value_t* st_ok_qh = rocke_b_cmp_lt(b, st_qh_iter, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* st_ok_qh = rocke_b_cmp_lt(b, st_qh_iter, ctx->num_qh_p); rocke_value_t* st_ok_inner = rocke_b_land(b, st_ok_qp, st_ok_qh); if(ctx->VALID_ROWS < ctx->BLOCK_M) { @@ -399,8 +399,7 @@ void rocke_gfx942_attn2d_emit_kv_body(rocke_gfx942_attn2d_build_ctx_t* ctx) st_causal_lim_iter = rocke_b_add(b, ctx->context_len, st_qp_iter); if(ctx->USE_ALIBI) { - rocke_value_t* st_qh_iter_ok - = rocke_b_cmp_lt(b, st_qh_iter, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* st_qh_iter_ok = rocke_b_cmp_lt(b, st_qh_iter, ctx->num_qh_p); st_alibi_slope_iter = rocke_b_masked_global_load(b, ctx->alibi_slopes_ptr, st_qh_iter, @@ -595,8 +594,7 @@ void rocke_gfx942_attn2d_emit_kv_body(rocke_gfx942_attn2d_build_ctx_t* ctx) /* row_ok = land(qpcur_batch_q_len); - rocke_value_t* ok_qh - = rocke_b_cmp_lt(b, qh_r, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* ok_qh = rocke_b_cmp_lt(b, qh_r, ctx->num_qh_p); rocke_value_t* ok_inner = rocke_b_land(b, ok_pos, ok_qh); if(ctx->VALID_ROWS < ctx->BLOCK_M) { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_kv_descriptors_loads.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_kv_descriptors_loads.cpp index 269d8cab0c4c..086f4d9a9047 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_kv_descriptors_loads.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_kv_descriptors_loads.cpp @@ -106,8 +106,12 @@ static rocke_value_t* rocke__kv_block_bytes_c(rocke_gfx942_attn2d_build_ctx_t* c { if(ctx->kv_block_bytes_c_v == NULL) { - int kv_stride_blk_b = ctx->BS * ctx->NUM_KV * ctx->HD * ctx->KV_BYTES; - ctx->kv_block_bytes_c_v = rocke_b_const_i32(ctx->b, kv_stride_blk_b); + /* The token and block strides scale with the runtime kv head count. */ + ctx->kv_stride_tok_b_v = rocke_b_mul( + ctx->b, ctx->num_kv_p, rocke_b_const_i32(ctx->b, ctx->HD * ctx->KV_BYTES)); + ctx->kv_stride_blk_b_v + = rocke_b_mul(ctx->b, ctx->kv_stride_tok_b_v, rocke_b_const_i32(ctx->b, ctx->BS)); + ctx->kv_block_bytes_c_v = ctx->kv_stride_blk_b_v; } return ctx->kv_block_bytes_c_v; } @@ -388,14 +392,11 @@ void rocke_gfx942_attn2d_emit_preloop(rocke_gfx942_attn2d_build_ctx_t* ctx) * lengths=[1<<24, BS, NUM_KV, HD], * strides=[kv_stride_blk_b, kv_stride_tok_b, kv_stride_h_b, KV_BYTES], * coord_names=("physical_block","token","kv_head","dim")). */ - int blk_b = ctx->BS * ctx->NUM_KV * ctx->HD * ctx->KV_BYTES; - int tok_b = ctx->NUM_KV * ctx->HD * ctx->KV_BYTES; - int h_b = ctx->HD * ctx->KV_BYTES; - int kv_lengths[4] = {1 << 24, ctx->BS, ctx->NUM_KV, ctx->HD}; - int kv_strides[4] = {blk_b, tok_b, h_b, ctx->KV_BYTES}; + int kv_strides[4] = {1, 1, ctx->HD * ctx->KV_BYTES, ctx->KV_BYTES}; static const char* const kv_coords[4] = {"physical_block", "token", "kv_head", "dim"}; - rocke_tensor_descriptor_t* kv_base = rocke_tensor_descriptor_naive( - b, "paged_kv_bytes", kv_lengths, 4, kv_strides, kv_coords, 4); + rocke_value_t* dyn[4] = {ctx->kv_stride_blk_b_v, ctx->kv_stride_tok_b_v, NULL, NULL}; + rocke_tensor_descriptor_t* kv_base = rocke_tensor_descriptor_naive_dynamic_mixed( + b, "paged_kv_bytes", kv_coords, 4, dyn, kv_strides); /* Single-block tile (N_BLOCKS_PER_TILE == 1, T == BS): * indirect(tile_idx -> physical_block) ; unmerge(linear_half -> token,dim). @@ -535,8 +536,9 @@ static void rocke__fast_paged_kv_voff_split(rocke_gfx942_attn2d_build_ctx_t* ctx (void)base_ptr_src; if(ctx->I64_KV_ADDR) { - rocke_value_t* base_i64 - = rocke_b_shl(b, rocke_b_zext(b, physical, rocke_i64()), rocke_b_const_i64(b, 15)); + /* zext before the shift constant, as Python evaluates them. */ + rocke_value_t* physical_i64 = rocke_b_zext(b, physical, rocke_i64()); + rocke_value_t* base_i64 = rocke_b_shl(b, physical_i64, rocke_b_const_i64(b, 15)); if(out_base_i64) *out_base_i64 = base_i64; if(out_voff) @@ -603,7 +605,6 @@ static bool rocke__tile_pages(rocke_gfx942_attn2d_build_ctx_t* ctx, snprintf(b->err, ROCKE_ERR_MSG_CAP, "out of memory for %d tile pages", n_pages); return false; } - int64_t kv_stride_blk_b = (int64_t)ctx->BS * ctx->NUM_KV * ctx->HD * ctx->KV_BYTES; rocke_value_t* seq_base = rocke__seq_base(ctx); rocke_value_t* max_idx = rocke__block_table_max_idx(ctx); rocke_value_t* tile_page0 = rocke_b_mul(b, kv_tile_idx, rocke_b_const_i32(b, n_pages)); @@ -616,7 +617,8 @@ static bool rocke__tile_pages(rocke_gfx942_attn2d_build_ctx_t* ctx, rocke_value_t* physical = rocke_b_masked_global_load(b, ctx->block_tables, idx, mask, def, rocke_i32(), 4); rocke_value_t* page_i64 = rocke_b_zext(b, rocke_b_to_sgpr_u32(b, physical), rocke_i64()); - rocke_value_t* base = rocke_b_mul(b, page_i64, rocke_b_const_i64(b, kv_stride_blk_b)); + rocke_value_t* stride64 = rocke_b_zext(b, ctx->kv_stride_blk_b_v, rocke_i64()); + rocke_value_t* base = rocke_b_mul(b, page_i64, stride64); ptrs[page] = rocke_b_global_ptr_add(b, src, base); } rocke_value_t* kv_block_bytes_c = rocke__kv_block_bytes_c(ctx); @@ -639,8 +641,7 @@ static rocke_value_t* rocke_value_t* half = rocke_b_add(b, call_half, rocke__lane_half_base(ctx)); rocke_value_t* token = rocke_b_div(b, half, rocke_b_const_i32(b, ctx->HD)); rocke_value_t* dim = rocke_b_mod(b, half, rocke_b_const_i32(b, ctx->HD)); - rocke_value_t* token_b - = rocke_b_mul(b, token, rocke_b_const_i32(b, ctx->NUM_KV * ctx->HD * ctx->KV_BYTES)); + rocke_value_t* token_b = rocke_b_mul(b, token, ctx->kv_stride_tok_b_v); rocke_value_t* token_head_b = rocke_b_add(b, token_b, kv_head_b); rocke_value_t* dim_b = rocke_b_mul(b, dim, rocke_b_const_i32(b, ctx->KV_BYTES)); return rocke_b_add(b, token_head_b, dim_b); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_loop_scaffold.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_loop_scaffold.cpp index 87e4c6755943..c3edf6420ac4 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_loop_scaffold.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_loop_scaffold.cpp @@ -90,7 +90,6 @@ void rocke_gfx942_attn2d_emit_loop_bounds_and_inits(rocke_gfx942_attn2d_build_ct const int BLOCK_M = ctx->BLOCK_M; const int NQK = ctx->NQK; const int SLIDING_WINDOW = ctx->SLIDING_WINDOW; - const int NUM_QH = ctx->NUM_QH; const int SOFTMAX_STATE_SLOTS = ctx->SOFTMAX_STATE_SLOTS; /* ---- max_seq_prefix_len (1982-1984) ---- */ @@ -173,9 +172,24 @@ void rocke_gfx942_attn2d_emit_loop_bounds_and_inits(rocke_gfx942_attn2d_build_ct rocke_value_t* m_inits[ROCKE_GFX942_ATTN2D_MAX_REGS_PER_LANE]; rocke_value_t* l_inits[ROCKE_GFX942_ATTN2D_MAX_REGS_PER_LANE]; - if(ctx->USE_SINKS) + if(ctx->USE_SINKS && ctx->USE_MFMA_32X32 && ctx->TRANSPOSED_QK_32X32 + && ctx->TRANSPOSED_SCALAR_STATE) { - /* Narrow gfx942 path (the transposed-scalar 32x32 branch is gfx950). */ + /* Transposed-scalar-state branch, reached by the gfx950 fastKV + * register-P twin: one state slot whose row is wave_row_base + + * lane_col32. */ + rocke_value_t* row = rocke_b_add(B, ctx->wave_row_base, ctx->lane_col32_v); + rocke_value_t* qh_mul = rocke_b_mul(B, ctx->kv_head_idx, rocke_b_const_i32(B, NQK)); + rocke_value_t* qh_mod = rocke_b_mod(B, row, rocke_b_const_i32(B, NQK)); + rocke_value_t* qh = rocke_b_add(B, qh_mul, qh_mod); + rocke_value_t* qh_in = rocke_b_cmp_lt(B, qh, ctx->num_qh_p); + rocke_value_t* sink_h = rocke_b_global_load(B, ctx->sinks, qh, ctx->dtype, 2); + rocke_value_t* sink_f = rocke_b_fmul(B, rocke_b_cast_to_f32(B, sink_h), rcp_ln2); + m_inits[0] = rocke_b_select(B, qh_in, sink_f, neg_inf); + } + else if(ctx->USE_SINKS) + { + /* Narrow gfx942 path. */ for(int r = 0; r < SOFTMAX_STATE_SLOTS; ++r) { rocke_value_t* row = rocke_b_add(B, ctx->wave_row_base, sc_state_row(ctx, lane_rg, r)); @@ -184,7 +198,7 @@ void rocke_gfx942_attn2d_emit_loop_bounds_and_inits(rocke_gfx942_attn2d_build_ct rocke_value_t* qh_mul = rocke_b_mul(B, ctx->kv_head_idx, rocke_b_const_i32(B, NQK)); rocke_value_t* qh_mod = rocke_b_mod(B, row, rocke_b_const_i32(B, NQK)); rocke_value_t* qh = rocke_b_add(B, qh_mul, qh_mod); - rocke_value_t* qh_in = rocke_b_cmp_lt(B, qh, rocke_b_const_i32(B, NUM_QH)); + rocke_value_t* qh_in = rocke_b_cmp_lt(B, qh, ctx->num_qh_p); rocke_value_t* sink_h = rocke_b_global_load(B, ctx->sinks, qh, ctx->dtype, 2); rocke_value_t* sink_f = rocke_b_fmul(B, rocke_b_cast_to_f32(B, sink_h), rcp_ln2); m_inits[r] = rocke_b_select(B, qh_in, sink_f, neg_inf); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_public_entry_glue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_public_entry_glue.cpp index 25da0bf46b41..79e3128f5fb6 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_public_entry_glue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_public_entry_glue.cpp @@ -40,7 +40,9 @@ #include "rocke/helper_helper_rocke.instances.common.attention_unified_selectors.h" /* rocke_unified_attn_tiled_2d_lds_bytes */ #include "rocke/helper_rocke.helpers.spec.h" /* rocke_kernel_name_join */ +#include "rocke/helper_rocke.instances.common._fmha_common.h" /* rocke_attention_check_head_counts */ +#include "rocke/arena.h" /* rocke_arena_strdup */ #include "rocke/error_boundary.hpp" /* ckc::guard_builder boundary shim */ #include #include @@ -514,7 +516,7 @@ static rocke_status_t /* Value-carrying optionals become conditional strings; value-less flags * are passed as positional parts (kernel_name_join drops empties), exactly * as the Python kernel_name does. */ - char d_buf[32], b_buf[32], t_buf[32], hkv_buf[64], kv_buf[64]; + char d_buf[32], b_buf[32], t_buf[32], hkv_buf[64], fhkv_buf[64], kv_buf[64]; char sw_buf[32], w_buf[32], wpe_buf[32], mw_buf[32], kvcp_buf[64]; char rd_buf[32], ks_buf[32]; int nqh, nkv; @@ -524,6 +526,9 @@ static rocke_status_t nqh = s->num_query_heads; nkv = s->num_kv_heads; + /* The name carries the GQA ratio; reject a spec it cannot be formed for. */ + if(nkv <= 0) + return ROCKE_ERR_VALUE; snprintf(d_buf, sizeof(d_buf), "d%d", s->head_size); snprintf(b_buf, sizeof(b_buf), "b%d", s->block_size); @@ -531,13 +536,20 @@ static rocke_status_t snprintf(t_buf, sizeof(t_buf), "t%d", rocke_attention_tiled_2d_spec_tile_size_eff(s)); else t_buf[0] = '\0'; - snprintf(hkv_buf, sizeof(hkv_buf), "h%dkv%d", nqh, nkv); + /* Head counts are kernargs; only the GQA ratio is baked. */ + snprintf(hkv_buf, sizeof(hkv_buf), "g%d", nqh / nkv); + /* Except on the fast paged-KV descriptor path, whose shifts bake them. */ + if(s->use_fast_paged_kv_desc) + snprintf(fhkv_buf, sizeof(fhkv_buf), "h%dkv%d", nqh, nkv); + else + fhkv_buf[0] = '\0'; if(s->kv_storage_dtype) snprintf(kv_buf, sizeof(kv_buf), "kv%s", s->kv_storage_dtype); else kv_buf[0] = '\0'; + /* The window size is a kernarg; only whether the body masks one is baked. */ if(s->sliding_window > 0) - snprintf(sw_buf, sizeof(sw_buf), "sw%d", s->sliding_window); + snprintf(sw_buf, sizeof(sw_buf), "sw"); else sw_buf[0] = '\0'; if(s->num_warps != 1) @@ -579,11 +591,14 @@ static rocke_status_t b_buf, t_buf, hkv_buf, + fhkv_buf, s->dtype ? s->dtype : "", kv_buf, /* Python emits "fnuz" under the *same* condition as the kv part, * not under a format test, so it keys off kv_buf being non-empty. */ kv_buf[0] ? "fnuz" : "", + /* i64 K/V addressing changes the loaders, so it is a distinct kernel. */ + s->use_i64_kv_addr ? "i64" : "", s->use_sinks ? "sinks" : "", sw_buf, s->has_softcap ? "softcap" : "", @@ -639,12 +654,21 @@ rocke_kernel_def_t* rocke_build_unified_attention_2d_tiled_scalar_new( { return ckc::guard_builder(b, [&]() -> rocke_kernel_def_t* { char kname[1024]; + char why[160]; if(b == NULL || spec == NULL) return NULL; - if(rocke__attn2d_kernel_name(spec, kname, sizeof(kname)) != ROCKE_OK) - return NULL; - if(rocke_ir_builder_init(b, kname) != ROCKE_OK) + /* Init before anything can fail: the lower convenience reads and frees + * `b` whatever this returns. */ + if(rocke_ir_builder_init(b, "") != ROCKE_OK) return NULL; + if(!rocke_attention_check_head_counts( + spec->num_query_heads, spec->num_kv_heads, why, sizeof(why))) + ckc::raise_status(ROCKE_ERR_VALUE, why); + if(rocke__attn2d_kernel_name(spec, kname, sizeof(kname)) != ROCKE_OK) + ckc::raise_status(ROCKE_ERR_VALUE, "tiled 2D kernel_name encode failed"); + b->kernel->name = rocke_arena_strdup(&b->arena, kname); + if(b->kernel->name == NULL) + ckc::raise_status(ROCKE_ERR_OOM, "tiled 2D kernel name OOM"); return rocke_build_unified_attention_2d_tiled_scalar(b, spec, arch); }); } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_q_load_gather.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_q_load_gather.cpp index 47e7b7be0afc..81135a4eb790 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_q_load_gather.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_2d_q_load_gather.cpp @@ -84,16 +84,16 @@ void rocke_gfx942_attn2d_emit_q_load(rocke_gfx942_attn2d_build_ctx_t* ctx) * recompute here for the staged-store fallback path. */ z8 = rocke_b_zero_vec(b, dtype, 8); - /* q_desc = TensorDescriptor.naive("Q", lengths=[1<<30, NUM_QH, HD], - * coord_names=("token","head","dim")). Stored on ctx for the - * downstream Q gather (32x32 direct-global) + the epilogue store. */ + /* q_desc over (token, head, dim), stored on ctx for the downstream Q gather + * (32x32 direct-global) + the epilogue store. The head count is a kernarg, + * so the token stride is too. */ { - int lengths[3]; static const char* const coord_names[3] = {"token", "head", "dim"}; - lengths[0] = 1 << 30; - lengths[1] = ctx->NUM_QH; - lengths[2] = ctx->HD; - ctx->q_desc = rocke_tensor_descriptor_naive(b, "Q", lengths, 3, NULL, coord_names, 3); + rocke_value_t* strides[3] + = {rocke_b_mul(b, ctx->num_qh_p, rocke_b_const_i32(b, ctx->HD)), NULL, NULL}; + const int static_strides[3] = {1, ctx->HD, 1}; + ctx->q_desc = rocke_tensor_descriptor_naive_dynamic_mixed( + b, "Q", coord_names, 3, strides, static_strides); } if(ctx->Q_DIRECT_GLOBAL) @@ -151,8 +151,7 @@ void rocke_gfx942_attn2d_emit_q_load(rocke_gfx942_attn2d_build_ctx_t* ctx) * guard is always true — omitting it avoids a redundant cmp+and on the hot path. */ { rocke_value_t* lt_pos = rocke_b_cmp_lt(b, q_pos_t, ctx->cur_batch_q_len); - rocke_value_t* lt_qh - = rocke_b_cmp_lt(b, qh_t, rocke_b_const_i32(b, (int64_t)ctx->NUM_QH)); + rocke_value_t* lt_qh = rocke_b_cmp_lt(b, qh_t, ctx->num_qh_p); rocke_value_t* and_pos_qh = rocke_b_land(b, lt_pos, lt_qh); if(ctx->VALID_ROWS < ctx->BLOCK_M) { @@ -424,10 +423,9 @@ void rocke_gfx942_attn2d_emit_q_gather(rocke_gfx942_attn2d_build_ctx_t* ctx) rocke_b_mul(b, ctx->kv_head_idx, rocke_b_const_i32(b, (int64_t)ctx->NQK)), rocke_b_mod(b, q32_row, rocke_b_const_i32(b, (int64_t)ctx->NQK))); /* q32_mask = (q32_poscur_batch_q_len), - rocke_b_cmp_lt(b, q32_h, rocke_b_const_i32(b, (int64_t)ctx->NUM_QH))); + q32_mask = rocke_b_land(b, + rocke_b_cmp_lt(b, q32_pos, ctx->cur_batch_q_len), + rocke_b_cmp_lt(b, q32_h, ctx->num_qh_p)); q32_pos_safe = rocke_b_select(b, q32_mask, q32_pos, rocke_b_const_i32(b, 0)); q32_h_safe = rocke_b_select(b, q32_mask, q32_h, rocke_b_const_i32(b, 0)); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_public_glue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_public_glue.cpp index e12a8548a223..5b6f049e7f4f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_public_glue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_public_glue.cpp @@ -8,7 +8,7 @@ * SCOPE (this TU only): * - The two by-value spec structs' _default() / _validate() (__post_init__) * and every @property accessor (num_queries_per_kv, block_m=16, block_q, - * tile_size, dtype_ir, binary_search_iters, kernel_name via + * tile_size, dtype_ir, kernel_name via * kernel_name_join). (Python lines 91-172, 977-999.) * - supports_tiled_3d coverage gate (pure host predicate, static reasons). * (Python lines 175-231.) @@ -51,6 +51,7 @@ #include "rocke/helper_rocke.helpers.atoms.h" /* rocke_mfma_atom, make_c_warp_dstr_encoding */ #include "rocke/helper_rocke.helpers.distribution.h" /* make_static_tile_distribution */ #include "rocke/helper_rocke.helpers.spec.h" /* rocke_kernel_name_join */ +#include "rocke/helper_rocke.instances.common._fmha_common.h" /* rocke_attention_check_head_counts */ #include "rocke/instance_gfx942_attention_tiled_3d.h" #include "rocke/instance_gfx942_attention_tiled_3d_internal.h" #include "rocke/ir.h" @@ -312,7 +313,12 @@ int rocke_unified_attention_3d_tiled_spec_kernel_name( snprintf(d_part, sizeof(d_part), "d%d", s->head_size); snprintf(b_part, sizeof(b_part), "b%d", s->block_size); - snprintf(h_part, sizeof(h_part), "h%dkv%d", s->num_query_heads, s->num_kv_heads); + /* Head counts are kernargs; only the GQA ratio is baked. */ + if(s->num_kv_heads <= 0) + { + return -1; + } + snprintf(h_part, sizeof(h_part), "g%d", s->num_query_heads / s->num_kv_heads); snprintf(seg_part, sizeof(seg_part), "seg%d", s->num_segments); parts[np++] = "rocke_uattn3d_tiled_gfx942"; @@ -333,9 +339,10 @@ int rocke_unified_attention_3d_tiled_spec_kernel_name( { parts[np++] = "sinks"; } + /* The window size is a kernarg; only whether the body masks one is baked. */ if(s->sliding_window > 0) { - snprintf(sw_part, sizeof(sw_part), "sw%d", s->sliding_window); + snprintf(sw_part, sizeof(sw_part), "sw"); parts[np++] = sw_part; } if(s->has_softcap) @@ -409,10 +416,9 @@ const rocke_type_t* rocke_unified_attention_reduce_tiled_spec_dtype_ir( int rocke_unified_attention_reduce_tiled_spec_kernel_name( const rocke_unified_attention_reduce_tiled_spec_t* s, char* buf, size_t cap) { - /* kernel_name_join("rocke_uattn_reduce_tiled_gfx942", d, h, - * seg, dtype[, "lse"]) */ + /* kernel_name_join("rocke_uattn_reduce_tiled_gfx942", d, seg, + * dtype[, "lse"]); the head count is a kernarg. */ char d_part[32]; - char h_part[32]; char seg_part[32]; const char* parts[8]; size_t np = 0; @@ -425,12 +431,10 @@ int rocke_unified_attention_reduce_tiled_spec_kernel_name( } snprintf(d_part, sizeof(d_part), "d%d", s->head_size); - snprintf(h_part, sizeof(h_part), "h%d", s->num_query_heads); snprintf(seg_part, sizeof(seg_part), "seg%d", s->num_segments); parts[np++] = "rocke_uattn_reduce_tiled_gfx942"; parts[np++] = d_part; - parts[np++] = h_part; parts[np++] = seg_part; parts[np++] = (s->dtype != NULL) ? s->dtype : ""; if(s->emit_lse) @@ -687,7 +691,6 @@ bool rocke_gfx942_attn_tiled_3d_config_from_spec( out->Q_IN_V_SLOT = out->HD == 192 && out->T >= out->BLOCK_M; out->THREADS = 64; - out->binary_search_iters = rocke_unified_attention_3d_tiled_spec_binary_search_iters(spec); if(out->PV_N_TILES < 0 || out->PV_N_TILES > 16) { @@ -1155,6 +1158,10 @@ rocke_status_t rocke_build_unified_attention_3d_tiled_gfx942_lower_to_llvm( return ROCKE_ERR_VALUE; } + /* The name carries the GQA ratio: refuse a head pair it cannot be formed + * for with the validator's reason, not the encoder's. */ + if(!rocke_attention_check_head_counts(spec->num_query_heads, spec->num_kv_heads, err, err_cap)) + return ROCKE_ERR_VALUE; if(rocke_unified_attention_3d_tiled_spec_kernel_name(spec, name, sizeof(name)) < 0) { rocke_attn3d_set_err_buf(err, err_cap, "lower_to_llvm: kernel_name encode failed"); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_reduce.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_reduce.cpp index ee2583d176dd..56e1fd522f61 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_reduce.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_reduce.cpp @@ -17,7 +17,7 @@ * Pure f32 load / exp2 / store; no MFMA, no async DMA. wave64_reduce_max / * wave64_reduce_sum come from helper_helper_rocke.helpers.attention.h. The * builder-call sequence is a byte-identical translation of the Python span - * (which is itself a byte-for-byte port of the gfx950 reduce). + * (itself the gfx950 reduce with a runtime query head count). * * Peers (segment-kernel phases, ctx init, config_from_spec) live in sibling * translation units and are called only via the internal header. This file @@ -135,7 +135,6 @@ void rocke_gfx942_attention_tiled_3d_reduce_declare_and_prologue( const rocke_gfx942_attn_tiled_3d_config_t* cfg = &ctx->cfg; rocke_param_opts_t opts; - int NUM_QH = cfg->NUM_QH; int NUM_SEG = cfg->NUM_SEG; int HD = cfg->HD; @@ -188,6 +187,7 @@ void rocke_gfx942_attention_tiled_3d_reduce_declare_and_prologue( /* seq_lens_ptr : ptr, readonly, align=4 (unused in body) */ ctx->red_seq_lens = rocke_b_param(b, "seq_lens_ptr", rocke_ptr_type(b, rocke_i32(), "global"), &opts); + ctx->red_num_qh_p = rocke_b_param(b, "num_query_heads", rocke_i32(), NULL); /* lse_ptr : ptr, noalias, writeonly, align=4 (emit_lse only) */ ctx->red_lse = NULL; @@ -213,23 +213,33 @@ void rocke_gfx942_attention_tiled_3d_reduce_declare_and_prologue( ctx->zero_f = rocke_b_const_f32(b, 0.0); /* ---- descriptors (lines 1039-1053) ---- */ + /* The head count is a kernarg, so each token stride is too. */ { - int ml_lengths[3] = {1 << 30, NUM_QH, NUM_SEG}; + rocke_value_t* strides[3] + = {rocke_b_mul(b, ctx->red_num_qh_p, rocke_b_const_i32(b, NUM_SEG)), NULL, NULL}; + const int static_strides[3] = {1, NUM_SEG, 1}; const char* ml_coords[3] = {"token", "head", "seg"}; - ctx->ml_desc_red - = rocke_tensor_descriptor_naive(b, "segm_ml", ml_lengths, 3, NULL, ml_coords, 3); + ctx->ml_desc_red = rocke_tensor_descriptor_naive_dynamic_mixed( + b, "segm_ml", ml_coords, 3, strides, static_strides); } { - int sa_lengths[4] = {1 << 30, NUM_QH, NUM_SEG, HD}; + rocke_value_t* strides[4] + = {rocke_b_mul(b, ctx->red_num_qh_p, rocke_b_const_i32(b, NUM_SEG * HD)), + NULL, + NULL, + NULL}; + const int static_strides[4] = {1, NUM_SEG * HD, HD, 1}; const char* sa_coords[4] = {"token", "head", "seg", "dim"}; - ctx->seg_acc_desc_red - = rocke_tensor_descriptor_naive(b, "segm_output", sa_lengths, 4, NULL, sa_coords, 4); + ctx->seg_acc_desc_red = rocke_tensor_descriptor_naive_dynamic_mixed( + b, "segm_output", sa_coords, 4, strides, static_strides); } { - int out_lengths[3] = {1 << 30, NUM_QH, HD}; + rocke_value_t* strides[3] + = {rocke_b_mul(b, ctx->red_num_qh_p, rocke_b_const_i32(b, HD)), NULL, NULL}; + const int static_strides[3] = {1, HD, 1}; const char* out_coords[3] = {"token", "head", "dim"}; - ctx->out_desc_red - = rocke_tensor_descriptor_naive(b, "out", out_lengths, 3, NULL, out_coords, 3); + ctx->out_desc_red = rocke_tensor_descriptor_naive_dynamic_mixed( + b, "out", out_coords, 3, strides, static_strides); } /* ---- base_ml = ml_desc_red.offset(token=q_token, head=q_head, seg=0) ---- @@ -536,7 +546,7 @@ void rocke_gfx942_attention_tiled_3d_reduce_normalize_pass( rocke_if_t guard = rocke_b_scf_if(b, is_writer); rocke_b_region_enter(b, guard.then_region); { - rocke_value_t* nqh = rocke_b_const_i32(b, cfg->NUM_QH); + rocke_value_t* nqh = ctx->red_num_qh_p; rocke_value_t* row = rocke_b_mul(b, ctx->q_token, nqh); rocke_value_t* lse_index = rocke_b_add(b, row, ctx->q_head); rocke_value_t* has_mass = rocke_lse_row_has_mass(b, ctx->overall_expsum); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_segment_compute.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_segment_compute.cpp index 2352528555bd..5198daf7c60d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_segment_compute.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_segment_compute.cpp @@ -101,7 +101,7 @@ void rocke_gfx942_attention_tiled_3d_emit_loop_init( rocke_value_t* qh_mul = rocke_b_mul(b, ctx->kv_head_idx, rocke_b_const_i32(b, cfg->NQK)); rocke_value_t* qh_mod = rocke_b_mod(b, ctx->lane_col, rocke_b_const_i32(b, cfg->NQK)); rocke_value_t* qh = rocke_b_add(b, qh_mul, qh_mod); - rocke_value_t* qh_in = rocke_b_cmp_lt(b, qh, rocke_b_const_i32(b, cfg->NUM_QH)); + rocke_value_t* qh_in = rocke_b_cmp_lt(b, qh, ctx->num_qh_p); rocke_value_t* sink_h = rocke_b_global_load(b, ctx->sinks, qh, dtype, 2); rocke_value_t* sink_f = rocke_b_fmul(b, rocke_b_cast_to_f32(b, sink_h), ctx->rcp_ln2); rocke_value_t* sink_with_mask = rocke_b_select(b, qh_in, sink_f, ctx->neg_inf); @@ -223,7 +223,7 @@ void rocke_gfx942_attention_tiled_3d_emit_softmax_loop( qh_mod = rocke_b_mod(b, ctx->lane_col, rocke_b_const_i32(b, cfg->NQK)); ctx->row_qh = rocke_b_add(b, qh_mul, qh_mod); ok_a = rocke_b_cmp_lt(b, ctx->row_qp, ctx->cur_batch_q_len); - ok_b = rocke_b_cmp_lt(b, ctx->row_qh, rocke_b_const_i32(b, cfg->NUM_QH)); + ok_b = rocke_b_cmp_lt(b, ctx->row_qh, ctx->num_qh_p); ctx->row_ok = rocke_b_land(b, ok_a, ok_b); /* Padding rows past BLOCK_Q*NQK alias the next q-block: no store. */ if(cfg->BLOCK_Q * cfg->NQK < cfg->BLOCK_M) @@ -236,8 +236,7 @@ void rocke_gfx942_attention_tiled_3d_emit_softmax_loop( ctx->row_alibi = NULL; if(cfg->USE_ALIBI) { - rocke_value_t* qh_ok - = rocke_b_cmp_lt(b, ctx->row_qh, rocke_b_const_i32(b, cfg->NUM_QH)); + rocke_value_t* qh_ok = rocke_b_cmp_lt(b, ctx->row_qh, ctx->num_qh_p); ctx->row_alibi = rocke_b_masked_global_load( b, ctx->alibi_slopes_ptr, ctx->row_qh, qh_ok, rocke_b_const_f32(b, 0.0), f32, 4); } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_segment_setup.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_segment_setup.cpp index 38220bb4ff4c..fc30292abe1a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_segment_setup.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx942/attention_tiled_3d_gfx942_attn_tiled_3d_segment_setup.cpp @@ -318,6 +318,9 @@ void rocke_gfx942_attention_tiled_3d_declare_params( ctx->num_seqs_p = rocke_b_param(B, "num_seqs", rocke_i32(), NULL); ctx->bt_stride_p = rocke_b_param(B, "block_table_stride", rocke_i32(), NULL); ctx->qq_bias_stride0_p = rocke_b_param(B, "qq_bias_stride_0", rocke_i32(), NULL); + ctx->num_qh_p = rocke_b_param(B, "num_query_heads", rocke_i32(), NULL); + ctx->num_kv_p = rocke_b_param(B, "num_kv_heads", rocke_i32(), NULL); + ctx->sliding_window_p = rocke_b_param(B, "sliding_window", rocke_i32(), NULL); if(ctx->strided_kv) { ctx->k_strides = rocke_declare_strided_kv_params(B, "k"); @@ -337,7 +340,6 @@ static void rocke__build_paged_kv_desc(rocke_gfx942_attention_tiled_3d_build_ctx const int HD = CFG.HD; const int T = CFG.T; const int BS = CFG.BS; - const int NUM_KV = CFG.NUM_KV; /* seq_base = seq_idx * bt_stride_p (line 569) */ if(ctx->strided_kv) @@ -351,12 +353,11 @@ static void rocke__build_paged_kv_desc(rocke_gfx942_attention_tiled_3d_build_ctx * strides=[kv_stride_blk_b, kv_stride_tok_b, kv_stride_h_b, KV_BYTES], * coord_names=("physical_block","token","kv_head","dim")) (lines 570-575) */ { - const int lengths[4] = {1 << 24, BS, NUM_KV, HD}; - const int strides[4] - = {CFG.kv_stride_blk_b, CFG.kv_stride_tok_b, CFG.kv_stride_h_b, CFG.KV_BYTES}; + rocke_value_t* strides[4] = {ctx->kv_stride_blk_b, ctx->kv_stride_tok_b, NULL, NULL}; + const int static_strides[4] = {1, 1, CFG.kv_stride_h_b, CFG.KV_BYTES}; const char* coords[4] = {"physical_block", "token", "kv_head", "dim"}; - ctx->kv_base_desc - = rocke_tensor_descriptor_naive(B, "paged_kv_bytes", lengths, 4, strides, coords, 4); + ctx->kv_base_desc = rocke_tensor_descriptor_naive_dynamic_mixed( + B, "paged_kv_bytes", coords, 4, strides, static_strides); } /* Transform chain (lines 576-602). */ @@ -414,7 +415,6 @@ void rocke_gfx942_attention_tiled_3d_emit_prologue(rocke_gfx942_attention_tiled_ const int T = CFG.T; const int BLOCK_M = CFG.BLOCK_M; const int BLOCK_Q = CFG.BLOCK_Q; - const int NUM_QH = CFG.NUM_QH; const int NUM_SEG = CFG.NUM_SEG; const rocke_type_t* dtype = CFG.dtype; @@ -433,13 +433,9 @@ void rocke_gfx942_attention_tiled_3d_emit_prologue(rocke_gfx942_attention_tiled_ } else { - ctx->seq_idx = rocke_binary_search_seq_idx(B, - ctx->cu_q, - ctx->q_block_global_idx, - ctx->num_seqs_p, - BLOCK_Q, - CFG.binary_search_iters, - false); + rocke_value_t* trip = rocke_binary_search_seq_idx_trip_count(B, ctx->num_seqs_p); + ctx->seq_idx = rocke_binary_search_seq_idx_dyn( + B, ctx->cu_q, ctx->q_block_global_idx, ctx->num_seqs_p, BLOCK_Q, trip, false); } /* ---- cu_q bounds / per-sequence geometry (lines 344-350) ---- */ @@ -481,22 +477,30 @@ void rocke_gfx942_attention_tiled_3d_emit_prologue(rocke_gfx942_attention_tiled_ } /* ---- descriptors (lines 359-373) ---- */ + /* The head count is a kernarg, so each token stride is too. */ { - const int ml_lengths[3] = {1 << 30, NUM_QH, NUM_SEG}; + rocke_value_t* strides[3] + = {rocke_b_mul(B, ctx->num_qh_p, rocke_b_const_i32(B, NUM_SEG)), NULL, NULL}; + const int static_strides[3] = {1, NUM_SEG, 1}; const char* ml_coords[3] = {"token", "head", "seg"}; - ctx->ml_desc - = rocke_tensor_descriptor_naive(B, "segm_ml", ml_lengths, 3, NULL, ml_coords, 3); + ctx->ml_desc = rocke_tensor_descriptor_naive_dynamic_mixed( + B, "segm_ml", ml_coords, 3, strides, static_strides); } { - const int sa_lengths[4] = {1 << 30, NUM_QH, NUM_SEG, HD}; + rocke_value_t* strides[4] + = {rocke_b_mul(B, ctx->num_qh_p, rocke_b_const_i32(B, NUM_SEG * HD)), NULL, NULL, NULL}; + const int static_strides[4] = {1, NUM_SEG * HD, HD, 1}; const char* sa_coords[4] = {"token", "head", "seg", "dim"}; - ctx->seg_acc_desc - = rocke_tensor_descriptor_naive(B, "segm_output", sa_lengths, 4, NULL, sa_coords, 4); + ctx->seg_acc_desc = rocke_tensor_descriptor_naive_dynamic_mixed( + B, "segm_output", sa_coords, 4, strides, static_strides); } { - const int q_lengths[3] = {1 << 30, NUM_QH, HD}; + rocke_value_t* strides[3] + = {rocke_b_mul(B, ctx->num_qh_p, rocke_b_const_i32(B, HD)), NULL, NULL}; + const int static_strides[3] = {1, HD, 1}; const char* q_coords[3] = {"token", "head", "dim"}; - ctx->q_desc = rocke_tensor_descriptor_naive(B, "Q", q_lengths, 3, NULL, q_coords, 3); + ctx->q_desc = rocke_tensor_descriptor_naive_dynamic_mixed( + B, "Q", q_coords, 3, strides, static_strides); } /* seg_start_tile_pos = seg_idx*tps*T (line 375) */ @@ -526,7 +530,7 @@ void rocke_gfx942_attention_tiled_3d_emit_prologue(rocke_gfx942_attention_tiled_ ctx->one_f = rocke_b_const_f32(B, 1.0); ctx->rcp_ln2 = rocke_b_const_f32(B, 1.4426950408889634); ctx->qk_scale = rocke_b_fmul(B, ctx->scale_p, ctx->rcp_ln2); - ctx->sw_const = rocke_b_const_i32(B, CFG.SLIDING_WINDOW); + ctx->sw_const = ctx->sliding_window_p; ctx->z8 = rocke_b_zero_vec(B, dtype, 8); /* ---- Q -> LDS (lines 437-467) ---- */ @@ -584,6 +588,10 @@ void rocke_gfx942_attention_tiled_3d_emit_async_infra( ctx->key_rsrc = rocke_b_buffer_rsrc(B, ctx->key, ctx->big_bytes); ctx->value_rsrc = rocke_b_buffer_rsrc(B, ctx->value, ctx->big_bytes); } + /* The token and page strides scale with the runtime kv head count. */ + ctx->kv_stride_tok_b + = rocke_b_mul(B, ctx->num_kv_p, rocke_b_const_i32(B, CFG.HD * CFG.KV_BYTES)); + ctx->kv_stride_blk_b = rocke_b_mul(B, ctx->kv_stride_tok_b, rocke_b_const_i32(B, CFG.BS)); ctx->lane_half_base = rocke_b_mul(B, ctx->tid, rocke_b_const_i32(B, CFG.HALVES_PER_LANE)); ctx->K_lds_addr = rocke_b_smem_addr_of(B, ctx->K_lds); ctx->V_lds_addr = rocke_b_smem_addr_of(B, ctx->V_lds); @@ -600,7 +608,6 @@ void rocke_gfx942_attention_tiled_3d_emit_early_zero_fill( rocke_gfx942_attention_tiled_3d_build_ctx_t* ctx) { const int NQK = CFG.NQK; - const int NUM_QH = CFG.NUM_QH; const int PV_N_TILES = CFG.PV_N_TILES; int n, reg; @@ -623,7 +630,7 @@ void rocke_gfx942_attention_tiled_3d_emit_early_zero_fill( rocke_value_t* qh_r_mod = rocke_b_mod(B, row, rocke_b_const_i32(B, NQK)); rocke_value_t* qh_r = rocke_b_add(B, qh_r_mul, qh_r_mod); rocke_value_t* row_ok_a = rocke_b_cmp_lt(B, qp_r, ctx->cur_batch_q_len); - rocke_value_t* row_ok_b = rocke_b_cmp_lt(B, qh_r, rocke_b_const_i32(B, NUM_QH)); + rocke_value_t* row_ok_b = rocke_b_cmp_lt(B, qh_r, ctx->num_qh_p); rocke_value_t* row_ok = rocke_b_land(B, row_ok_a, row_ok_b); /* Padding rows past BLOCK_Q*NQK alias the next q-block: no store. */ if(CFG.BLOCK_Q * NQK < CFG.BLOCK_M) @@ -663,7 +670,7 @@ void rocke_gfx942_attention_tiled_3d_emit_early_zero_fill( rocke_value_t* qh_r_mod = rocke_b_mod(B, row, rocke_b_const_i32(B, NQK)); rocke_value_t* qh_r = rocke_b_add(B, qh_r_mul, qh_r_mod); rocke_value_t* row_ok_a = rocke_b_cmp_lt(B, qp_r, ctx->cur_batch_q_len); - rocke_value_t* row_ok_b = rocke_b_cmp_lt(B, qh_r, rocke_b_const_i32(B, NUM_QH)); + rocke_value_t* row_ok_b = rocke_b_cmp_lt(B, qh_r, ctx->num_qh_p); rocke_value_t* row_ok = rocke_b_land(B, row_ok_a, row_ok_b); /* Padding rows past BLOCK_Q*NQK alias the next q-block: no store. */ if(CFG.BLOCK_Q * NQK < CFG.BLOCK_M) @@ -694,7 +701,6 @@ void rocke_gfx942_attention_tiled_3d_emit_early_zero_fill( void rocke_gfx942_attention_tiled_3d_emit_q_to_lds(rocke_gfx942_attention_tiled_3d_build_ctx_t* ctx) { const int NQK = CFG.NQK; - const int NUM_QH = CFG.NUM_QH; const int THREADS = CFG.THREADS; const int Q_VECS_PER_ROW = CFG.Q_VECS_PER_ROW; const int Q_VECS_PER_THREAD = CFG.Q_VECS_PER_THREAD; @@ -716,7 +722,7 @@ void rocke_gfx942_attention_tiled_3d_emit_q_to_lds(rocke_gfx942_attention_tiled_ rocke_value_t* qh_t_mod = rocke_b_mod(B, Q_row, rocke_b_const_i32(B, NQK)); rocke_value_t* qh_t = rocke_b_add(B, qh_t_mul, qh_t_mod); rocke_value_t* qmask_a = rocke_b_cmp_lt(B, q_pos_t, ctx->cur_batch_q_len); - rocke_value_t* qmask_b = rocke_b_cmp_lt(B, qh_t, rocke_b_const_i32(B, NUM_QH)); + rocke_value_t* qmask_b = rocke_b_cmp_lt(B, qh_t, ctx->num_qh_p); rocke_value_t* qmask_t = rocke_b_land(B, qmask_a, qmask_b); rocke_value_t* q_pos_safe = rocke_b_select(B, qmask_t, q_pos_t, rocke_b_const_i32(B, 0)); rocke_value_t* qh_safe = rocke_b_select(B, qmask_t, qh_t, rocke_b_const_i32(B, 0)); @@ -760,9 +766,10 @@ void rocke_gfx942_attention_tiled_3d_issue_k_load(rocke_gfx942_attention_tiled_3 /* The tile base goes into soffset; each copy adds its lane's offset within * the page. */ rocke_value_t* soff - = page_id != NULL ? rocke_gfx942_attention_tiled_3d_tile_base( - ctx, page_id, kv_tile_idx, CFG.kv_stride_blk_b, CFG.kv_stride_tok_b) - : ctx->zero_soff; + = page_id != NULL + ? rocke_gfx942_attention_tiled_3d_tile_base( + ctx, page_id, kv_tile_idx, ctx->kv_stride_blk_b, ctx->kv_stride_tok_b) + : ctx->zero_soff; rocke_value_t* buf_off_i32 = rocke_b_mul(B, buf_idx, rocke_b_const_i32(B, bytes_per_buf)); rocke_value_t* buf_off_i64 = rocke_b_zext(B, buf_off_i32, rocke_i64()); @@ -789,7 +796,7 @@ void rocke_gfx942_attention_tiled_3d_issue_k_load(rocke_gfx942_attention_tiled_3 rocke_value_t* src_half = rocke_gfx942_attention_tiled_3d_kv_src_linear(ctx, linear_half); rocke_value_t* tok = rocke_b_div(B, src_half, rocke_b_const_i32(B, CFG.HD)); - rocke_value_t* tok_off = rocke_b_mul(B, tok, rocke_b_const_i32(B, CFG.kv_stride_tok_b)); + rocke_value_t* tok_off = rocke_b_mul(B, tok, ctx->kv_stride_tok_b); rocke_value_t* head_off = rocke_b_mul(B, ctx->kv_head_idx, rocke_b_const_i32(B, CFG.kv_stride_h_b)); rocke_value_t* tok_head = rocke_b_add(B, tok_off, head_off); @@ -827,9 +834,10 @@ void rocke_gfx942_attention_tiled_3d_issue_v_load(rocke_gfx942_attention_tiled_3 /* The tile base goes into soffset; each copy adds its lane's offset within * the page. */ rocke_value_t* soff - = page_id != NULL ? rocke_gfx942_attention_tiled_3d_tile_base( - ctx, page_id, kv_tile_idx, CFG.kv_stride_blk_b, CFG.kv_stride_tok_b) - : ctx->zero_soff; + = page_id != NULL + ? rocke_gfx942_attention_tiled_3d_tile_base( + ctx, page_id, kv_tile_idx, ctx->kv_stride_blk_b, ctx->kv_stride_tok_b) + : ctx->zero_soff; rocke_value_t* buf_off_i32 = rocke_b_mul(B, buf_idx, rocke_b_const_i32(B, bytes_per_buf)); rocke_value_t* buf_off_i64 = rocke_b_zext(B, buf_off_i32, rocke_i64()); @@ -856,7 +864,7 @@ void rocke_gfx942_attention_tiled_3d_issue_v_load(rocke_gfx942_attention_tiled_3 rocke_value_t* src_half = rocke_gfx942_attention_tiled_3d_kv_src_linear(ctx, linear_half); rocke_value_t* tok = rocke_b_div(B, src_half, rocke_b_const_i32(B, CFG.HD)); - rocke_value_t* tok_off = rocke_b_mul(B, tok, rocke_b_const_i32(B, CFG.kv_stride_tok_b)); + rocke_value_t* tok_off = rocke_b_mul(B, tok, ctx->kv_stride_tok_b); rocke_value_t* head_off = rocke_b_mul(B, ctx->kv_head_idx, rocke_b_const_i32(B, CFG.kv_stride_h_b)); rocke_value_t* tok_head = rocke_b_add(B, tok_off, head_off); @@ -941,18 +949,19 @@ rocke_value_t* rocke_gfx942_attention_tiled_3d_tile_base(rocke_gfx942_attention_tiled_3d_build_ctx_t* ctx, rocke_value_t* page_id, rocke_value_t* kv_tile_idx, - int page_stride, - int token_stride) + rocke_value_t* page_stride, + rocke_value_t* token_stride) { - rocke_value_t* base = rocke_b_mul(B, page_id, rocke_b_const_i32(B, page_stride)); + rocke_value_t* base = rocke_b_mul(B, page_id, page_stride); rocke_value_t* in_page; + rocke_value_t* tile_stride; if(CFG.TILES_PER_PAGE == 1) { return base; } in_page = rocke_b_mod(B, kv_tile_idx, rocke_b_const_i32(B, CFG.TILES_PER_PAGE)); - return rocke_b_add( - B, base, rocke_b_mul(B, in_page, rocke_b_const_i32(B, CFG.T * token_stride))); + tile_stride = rocke_b_mul(B, token_stride, rocke_b_const_i32(B, CFG.T)); + return rocke_b_add(B, base, rocke_b_mul(B, in_page, tile_stride)); } /* _load_page_id_window(rel_page): lane l loads the id of the segment's page @@ -1057,9 +1066,12 @@ void rocke_gfx942_attention_tiled_3d_issue_wide_load( rocke_mem_opts_t opts = ROCKE_MEM_OPTS_INIT; if(page_id != NULL) { + /* Element strides (the wide loader indexes halves). */ + rocke_value_t* tok_e = rocke_b_mul(B, ctx->num_kv_p, rocke_b_const_i32(B, HD)); + rocke_value_t* blk_e = rocke_b_mul(B, tok_e, rocke_b_const_i32(B, CFG.BS)); rocke_value_t* page_base = rocke_gfx942_attention_tiled_3d_tile_base( - ctx, page_id, kv_tile_idx, CFG.BS * CFG.NUM_KV * HD, CFG.NUM_KV * HD); - rocke_value_t* row_off = rocke_b_mul(B, row, rocke_b_const_i32(B, CFG.NUM_KV * HD)); + ctx, page_id, kv_tile_idx, blk_e, tok_e); + rocke_value_t* row_off = rocke_b_mul(B, row, tok_e); rocke_value_t* head_off = rocke_b_mul(B, ctx->kv_head_idx, rocke_b_const_i32(B, HD)); rocke_value_t* in_page = rocke_b_add(B, rocke_b_add(B, row_off, head_off), col); elem_off = rocke_b_add(B, page_base, in_page); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_fastkv_regp_helpers.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_fastkv_regp_helpers.cpp index 0916cf1d0783..ad988823cddc 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_fastkv_regp_helpers.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_fastkv_regp_helpers.cpp @@ -9,6 +9,7 @@ #include "rocke/error.hpp" #include "rocke/helper_instance_gfx950_attention_tiled_2d_fastkv_regp.h" #include "rocke/helper_rocke.helpers.spec.h" /* rocke_kernel_name_join */ +#include "rocke/helper_rocke.instances.common._fmha_common.h" /* rocke_attention_check_head_counts */ #include "rocke/error_boundary.hpp" /* ckc::guard_builder boundary shim */ #include @@ -29,6 +30,19 @@ ckc::raise_status(st, msg ? msg : ""); } +/* Python _NOT_GFX950: the body uses gfx950-only wide MFMA atoms and transposed + * LDS reads. True (with the reason written) for any target but gfx950; NULL arch + * is the Python default "gfx950". */ +static bool rocke__fastkv_regp_not_gfx950(const char* arch, char* reason, size_t reason_cap) +{ + if(arch == NULL || strcmp(arch, "gfx950") == 0) + return false; + if(reason != NULL && reason_cap > 0) + snprintf( + reason, reason_cap, "fastKV register-P is a gfx950 kernel; %s is not supported", arch); + return true; +} + /* gfx950 UnifiedAttention2DTiledSpec.kernel_name() (rocke/instances/gfx950/ * attention_tiled_2d.py kernel_name). The gfx950 part list is SHORTER than the * gfx942 one (no wpe, no mfma32x8/qdir/kvcp/gldlds/qgrid/cfv/... parts), so the @@ -38,10 +52,11 @@ static rocke_status_t rocke__fastkv_gfx950_base_name(const rocke_attention_tiled char* out, size_t out_cap) { - char t_buf[32], hkv_buf[64], kv_buf[64], sw_buf[32], w_buf[32], mw_buf[32]; + char t_buf[32], hkv_buf[64], fhkv_buf[64], kv_buf[64], sw_buf[32], w_buf[32], mw_buf[32]; char d_buf[16], b_buf[16]; - if(s == NULL || out == NULL) + /* The name carries the GQA ratio; reject a spec it cannot be formed for. */ + if(s == NULL || out == NULL || s->num_kv_heads <= 0) return ROCKE_ERR_VALUE; snprintf(d_buf, sizeof(d_buf), "d%d", s->head_size); @@ -50,13 +65,20 @@ static rocke_status_t rocke__fastkv_gfx950_base_name(const rocke_attention_tiled snprintf(t_buf, sizeof(t_buf), "t%d", rocke_attention_tiled_2d_spec_tile_size_eff(s)); else t_buf[0] = '\0'; - snprintf(hkv_buf, sizeof(hkv_buf), "h%dkv%d", s->num_query_heads, s->num_kv_heads); + /* Head counts are kernargs; only the GQA ratio is baked. */ + snprintf(hkv_buf, sizeof(hkv_buf), "g%d", s->num_query_heads / s->num_kv_heads); + /* Except on the fast paged-KV descriptor path, whose shifts bake them. */ + if(s->use_fast_paged_kv_desc) + snprintf(fhkv_buf, sizeof(fhkv_buf), "h%dkv%d", s->num_query_heads, s->num_kv_heads); + else + fhkv_buf[0] = '\0'; if(s->kv_storage_dtype != NULL) snprintf(kv_buf, sizeof(kv_buf), "kv%s", s->kv_storage_dtype); else kv_buf[0] = '\0'; + /* The window size is a kernarg; only whether the body masks one is baked. */ if(s->sliding_window > 0) - snprintf(sw_buf, sizeof(sw_buf), "sw%d", s->sliding_window); + snprintf(sw_buf, sizeof(sw_buf), "sw"); else sw_buf[0] = '\0'; if(s->num_warps != 1) @@ -74,8 +96,11 @@ static rocke_status_t rocke__fastkv_gfx950_base_name(const rocke_attention_tiled b_buf, t_buf, hkv_buf, + fhkv_buf, s->dtype ? s->dtype : "", kv_buf, + /* i64 K/V addressing changes the loaders, so it is a distinct kernel. */ + s->use_i64_kv_addr ? "i64" : "", s->use_sinks ? "sinks" : "", sw_buf, s->has_softcap ? "softcap" : "", @@ -230,6 +255,11 @@ bool rocke_gfx950_supports_fastkv_register_p_2d( if(args == NULL) return false; + /* if arch != "gfx950": return False, _NOT_GFX950 -- supports_tiled_2d also + * admits gfx942, but this body is gfx950-only. NULL arch == Python "gfx950". */ + if(rocke__fastkv_regp_not_gfx950(args->arch, reason, reason_cap)) + return false; + /* ok, reason = supports_tiled_2d(... num_warps=4, kv_storage_dtype=None, * tile_size = tile_size if tile_size is not None else 2*block_size, * arch=arch) */ @@ -285,6 +315,16 @@ rocke_kernel_def_t* rocke_build_unified_attention_2d_fastkv_register_p( if(b->status != ROCKE_OK) /* dead builder: no-op */ return NULL; + /* if arch != "gfx950": raise ValueError(_NOT_GFX950) */ + { + char why[128]; + if(rocke__fastkv_regp_not_gfx950(arch, why, sizeof(why))) + { + rocke__fastkv_regp_set_err(b, ROCKE_ERR_VALUE, why); + return NULL; + } + } + /* if not spec.use_fast_paged_kv_desc: raise ValueError(...) */ if(!spec->use_fast_paged_kv_desc) { @@ -306,6 +346,18 @@ rocke_kernel_def_t* rocke_build_unified_attention_2d_fastkv_register_p( b, ROCKE_ERR_VALUE, "fastKV register-P experiment does not support FP8 KV cache"); return NULL; } + /* The tiled build reads spec.num_queries_per_kv, which raises ValueError + * with the head-count rule's text; check it here, before the shared + * scaffold divides by num_kv_heads. */ + { + char why[160]; + if(!rocke_attention_check_head_counts( + spec->num_query_heads, spec->num_kv_heads, why, sizeof(why))) + { + rocke__fastkv_regp_set_err(b, ROCKE_ERR_VALUE, why); + return NULL; + } + } /* return build_unified_attention_2d_tiled(_FastKvRegisterPProxy(spec), arch) * diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_kv_body_pv_epilogue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_kv_body_pv_epilogue.cpp index 2a16c0c13624..86feab10a348 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_kv_body_pv_epilogue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_kv_body_pv_epilogue.cpp @@ -658,6 +658,13 @@ void rocke_gfx950_attn2d_emit_pv_bucket(rocke_gfx950_attn2d_build_ctx_t* ctx, for(atom = 0; atom < ctx->ACC_M_ATOMS; ++atom) yields[yc++] = new_acc[n * ctx->ACC_M_ATOMS + atom]; yields[yc++] = ctx->GROUPED_KV2 ? in->cur_buf : ctx->nxt_buf_v; + if(rocke_gfx950_attn2d_page_prefetch(ctx)) + { + for(i = 0; i < ctx->N_BLOCKS_PER_TILE; ++i) + yields[yc++] = ctx->page_next[i]; + for(i = 0; i < ctx->N_BLOCKS_PER_TILE; ++i) + yields[yc++] = ctx->page_after_next[i]; + } /* Record the rewritten carry for callers that thread phases (the driver's * single-phase path reads this back via the loop results). */ @@ -713,6 +720,15 @@ rocke_for_t rocke_gfx950_attn2d_drive_kv_loop(rocke_gfx950_attn2d_build_ctx_t* c for(i = 0; i < num_accs; ++i) ctx->acc_cur[i] = kvloop.iter_vars[ml + i]; ctx->cur_buf = kvloop.iter_vars[ml + num_accs]; + if(rocke_gfx950_attn2d_page_prefetch(ctx)) + { + for(i = 0; i < ctx->N_BLOCKS_PER_TILE; ++i) + { + ctx->page_cur[i] = kvloop.iter_vars[ml + num_accs + 1 + i]; + ctx->page_next[i] = kvloop.iter_vars[ml + num_accs + 1 + ctx->N_BLOCKS_PER_TILE + i]; + } + ctx->page_cur_tile = kvloop.iv; + } ctx->skip_mask = false; rocke_gfx950_attn2d_emit_kv_body(ctx); @@ -785,8 +801,7 @@ rocke_kernel_def_t* rocke_gfx950_attn2d_emit_epilogue(rocke_gfx950_attn2d_build_ rocke_value_t* op_qh_mod = rocke_b_mod(b, q_row_t, rocke_b_const_i32(b, ctx->NQK)); rocke_value_t* op_qh_t = rocke_b_add(b, op_qh_mul, op_qh_mod); rocke_value_t* op_mask_pos = rocke_b_cmp_lt(b, op_pos_t, ctx->cur_batch_q_len); - rocke_value_t* op_mask_qh - = rocke_b_cmp_lt(b, op_qh_t, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* op_mask_qh = rocke_b_cmp_lt(b, op_qh_t, ctx->num_qh_p); rocke_value_t* op_mask_inner = rocke_b_land(b, op_mask_pos, op_mask_qh); rocke_value_t* op_mask_t; if(ctx->VALID_ROWS < ctx->BLOCK_M) @@ -828,7 +843,7 @@ rocke_kernel_def_t* rocke_gfx950_attn2d_emit_epilogue(rocke_gfx950_attn2d_build_ rocke_b_region_enter(b, lse_if.then_region); { rocke_value_t* tok = rocke_b_add(b, ctx->cu_q_start, op_pos_t); - rocke_value_t* nqh = rocke_b_const_i32(b, ctx->NUM_QH); + rocke_value_t* nqh = ctx->num_qh_p; rocke_value_t* tok_qh = rocke_b_mul(b, tok, nqh); rocke_value_t* off = rocke_b_add(b, tok_qh, op_qh_t); rocke_value_t* has_mass = rocke_lse_row_has_mass(b, ctx->l_final[0]); @@ -901,8 +916,7 @@ rocke_kernel_def_t* rocke_gfx950_attn2d_emit_epilogue(rocke_gfx950_attn2d_build_ } { rocke_value_t* mask_pos = rocke_b_cmp_lt(b, op_pos32_base, ctx->cur_batch_q_len); - rocke_value_t* mask_qh - = rocke_b_cmp_lt(b, op_qh32_base, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* mask_qh = rocke_b_cmp_lt(b, op_qh32_base, ctx->num_qh_p); rocke_value_t* mask_inner = rocke_b_land(b, mask_pos, mask_qh); if(ctx->VALID_ROWS < ctx->BLOCK_M) { @@ -937,7 +951,7 @@ rocke_kernel_def_t* rocke_gfx950_attn2d_emit_epilogue(rocke_gfx950_attn2d_build_ rocke_b_region_enter(b, lse_if.then_region); { rocke_value_t* tok = rocke_b_add(b, ctx->cu_q_start, ctx->hoist_q_pos[r]); - rocke_value_t* nqh = rocke_b_const_i32(b, ctx->NUM_QH); + rocke_value_t* nqh = ctx->num_qh_p; rocke_value_t* tok_qh = rocke_b_mul(b, tok, nqh); rocke_value_t* off = rocke_b_add(b, tok_qh, ctx->hoist_qh_r[r]); rocke_value_t* has_mass = rocke_lse_row_has_mass(b, ctx->l_final[r]); @@ -1043,7 +1057,7 @@ rocke_kernel_def_t* rocke_gfx950_attn2d_emit_epilogue(rocke_gfx950_attn2d_build_ } { rocke_value_t* mask_pos = rocke_b_cmp_lt(b, op_pos, ctx->cur_batch_q_len); - rocke_value_t* mask_qh = rocke_b_cmp_lt(b, op_qh, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* mask_qh = rocke_b_cmp_lt(b, op_qh, ctx->num_qh_p); rocke_value_t* mask_inner = rocke_b_land(b, mask_pos, mask_qh); if(ctx->VALID_ROWS < ctx->BLOCK_M) { @@ -1139,7 +1153,7 @@ rocke_kernel_def_t* rocke_gfx950_attn2d_emit_epilogue(rocke_gfx950_attn2d_build_ rocke_b_region_enter(b, lse_if.then_region); { rocke_value_t* tok = rocke_b_add(b, ctx->cu_q_start, ctx->hoist_q_pos[r]); - rocke_value_t* nqh = rocke_b_const_i32(b, ctx->NUM_QH); + rocke_value_t* nqh = ctx->num_qh_p; rocke_value_t* tok_qh = rocke_b_mul(b, tok, nqh); rocke_value_t* off = rocke_b_add(b, tok_qh, ctx->hoist_qh_r[r]); rocke_value_t* has_mass = rocke_lse_row_has_mass(b, ctx->l_final[r]); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_kv_body_qk_softmax.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_kv_body_qk_softmax.cpp index fb646752c9b8..20065f467d2a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_kv_body_qk_softmax.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_kv_body_qk_softmax.cpp @@ -243,7 +243,7 @@ void rocke_gfx950_attn2d_emit_licm_hoist(rocke_gfx950_attn2d_build_ctx_t* ctx) rocke_value_t* qh_mod = rocke_b_mod(b, row, rocke_b_const_i32(b, ctx->NQK)); rocke_value_t* qh_r = rocke_b_add(b, qh_mul, qh_mod); rocke_value_t* row_ok_pos = rocke_b_cmp_lt(b, qp_r, ctx->cur_batch_q_len); - rocke_value_t* row_ok_qh = rocke_b_cmp_lt(b, qh_r, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* row_ok_qh = rocke_b_cmp_lt(b, qh_r, ctx->num_qh_p); rocke_value_t* row_ok_inner = rocke_b_land(b, row_ok_pos, row_ok_qh); rocke_value_t* row_ok; if(ctx->VALID_ROWS < ctx->BLOCK_M) @@ -272,7 +272,7 @@ void rocke_gfx950_attn2d_emit_licm_hoist(rocke_gfx950_attn2d_build_ctx_t* ctx) for(reg = 0; reg < ctx->REGS_PER_LANE; ++reg) { rocke_value_t* qh_r = ctx->hoist_q_head[reg]; - rocke_value_t* qh_ok = rocke_b_cmp_lt(b, qh_r, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* qh_ok = rocke_b_cmp_lt(b, qh_r, ctx->num_qh_p); rocke_value_t* slope = rocke_b_masked_global_load( b, ctx->alibi_slopes_ptr, qh_r, qh_ok, rocke_b_const_f32(b, 0.0), f32, 4); ctx->hoist_q_head[reg] = slope; @@ -295,7 +295,7 @@ void rocke_gfx950_attn2d_emit_licm_hoist(rocke_gfx950_attn2d_build_ctx_t* ctx) rocke_value_t* st_qh_mod = rocke_b_mod(b, st_q_row, rocke_b_const_i32(b, ctx->NQK)); rocke_value_t* st_qh = rocke_b_add(b, st_qh_mul, st_qh_mod); rocke_value_t* st_ok_qp = rocke_b_cmp_lt(b, st_qp, ctx->cur_batch_q_len); - rocke_value_t* st_ok_qh = rocke_b_cmp_lt(b, st_qh, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* st_ok_qh = rocke_b_cmp_lt(b, st_qh, ctx->num_qh_p); rocke_value_t* st_ok_inner = rocke_b_land(b, st_ok_qp, st_ok_qh); ctx->st_qp_hoist = st_qp; ctx->st_qh_hoist = st_qh; @@ -312,7 +312,7 @@ void rocke_gfx950_attn2d_emit_licm_hoist(rocke_gfx950_attn2d_build_ctx_t* ctx) ctx->st_causal_lim_hoist = rocke_b_add(b, ctx->context_len, st_qp); if(ctx->USE_ALIBI) { - rocke_value_t* st_qh_ok = rocke_b_cmp_lt(b, st_qh, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* st_qh_ok = rocke_b_cmp_lt(b, st_qh, ctx->num_qh_p); ctx->st_alibi_slope_hoist = rocke_b_masked_global_load(b, ctx->alibi_slopes_ptr, st_qh, @@ -530,7 +530,7 @@ void rocke_gfx950_attn2d_emit_kv_body(rocke_gfx950_attn2d_build_ctx_t* ctx) rocke_value_t* st_qh_mod = rocke_b_mod(b, st_q_row_iter, rocke_b_const_i32(b, ctx->NQK)); rocke_value_t* st_qh_iter = rocke_b_add(b, st_qh_mul, st_qh_mod); rocke_value_t* st_ok_qp = rocke_b_cmp_lt(b, st_qp_iter, ctx->cur_batch_q_len); - rocke_value_t* st_ok_qh = rocke_b_cmp_lt(b, st_qh_iter, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* st_ok_qh = rocke_b_cmp_lt(b, st_qh_iter, ctx->num_qh_p); rocke_value_t* st_ok_inner = rocke_b_land(b, st_ok_qp, st_ok_qh); if(ctx->VALID_ROWS < ctx->BLOCK_M) { @@ -545,8 +545,7 @@ void rocke_gfx950_attn2d_emit_kv_body(rocke_gfx950_attn2d_build_ctx_t* ctx) st_causal_lim_iter = rocke_b_add(b, ctx->context_len, st_qp_iter); if(ctx->USE_ALIBI) { - rocke_value_t* st_qh_iter_ok - = rocke_b_cmp_lt(b, st_qh_iter, rocke_b_const_i32(b, ctx->NUM_QH)); + rocke_value_t* st_qh_iter_ok = rocke_b_cmp_lt(b, st_qh_iter, ctx->num_qh_p); st_alibi_slope_iter = rocke_b_masked_global_load(b, ctx->alibi_slopes_ptr, st_qh_iter, @@ -563,6 +562,8 @@ void rocke_gfx950_attn2d_emit_kv_body(rocke_gfx950_attn2d_build_ctx_t* ctx) rocke_value_t* in_range_next = rocke_b_cmp_lt(b, next_tile_iv_raw, ctx->tile_end); rocke_value_t* safe_next_tile = rocke_b_select(b, in_range_next, next_tile_iv_raw, ctx->kv_tile_iv); + if(rocke_gfx950_attn2d_page_prefetch(ctx)) + ctx->page_next_tile = safe_next_tile; /* softmax-derived state forwarded to the PV bucket (alpha, new_l, and * m_carry = the raw running max). m_new_out is the MFMA-32 branch's @@ -601,6 +602,19 @@ void rocke_gfx950_attn2d_emit_kv_body(rocke_gfx950_attn2d_build_ctx_t* ctx) && (ctx->SOFTMAX_INTERLEAVE_MODE == 0 || ctx->SOFTMAX_INTERLEAVE_MODE == 1)) rocke_b_iglp_opt(b, ctx->SOFTMAX_INTERLEAVE_MODE); + /* Read the page ids of the tile after next now, right after the iter-start + * drain, so they arrive long before next iteration's copies need them. That + * tile is next iteration's safe_next_tile. */ + if(rocke_gfx950_attn2d_page_prefetch(ctx)) + { + rocke_value_t* c_two = rocke_b_const_i32(b, 2); + rocke_value_t* after_next_raw = rocke_b_add(b, ctx->kv_tile_iv, c_two); + rocke_value_t* in_range_after = rocke_b_cmp_lt(b, after_next_raw, ctx->tile_end); + rocke_value_t* after_next + = rocke_b_select(b, in_range_after, after_next_raw, next_tile_iv_raw); + rocke_gfx950_attn2d_tile_page_ids(ctx, after_next, ctx->page_after_next); + } + /* Early-V schedule: V_lds is single-buffered and the iter-start full drain * guarantees the previous PV's V reads retired, so the current V copy can be * issued before QK to overlap with QK + softmax (Python 2566-2570). */ @@ -766,7 +780,7 @@ void rocke_gfx950_attn2d_emit_kv_body(rocke_gfx950_attn2d_build_ctx_t* ctx) qh_mod = rocke_b_mod(b, q_row_t, rocke_b_const_i32(b, ctx->NQK)); qh_r = rocke_b_add(b, qh_mul, qh_mod); row_ok_pos = rocke_b_cmp_lt(b, qp_r, ctx->cur_batch_q_len); - row_ok_qh = rocke_b_cmp_lt(b, qh_r, rocke_b_const_i32(b, ctx->NUM_QH)); + row_ok_qh = rocke_b_cmp_lt(b, qh_r, ctx->num_qh_p); rocke_value_t* row_ok_inner = rocke_b_land(b, row_ok_pos, row_ok_qh); if(ctx->VALID_ROWS < ctx->BLOCK_M) { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_kv_descriptors_loads.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_kv_descriptors_loads.cpp index 82129cb1b3bd..b613250ea548 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_kv_descriptors_loads.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_kv_descriptors_loads.cpp @@ -47,6 +47,10 @@ * Lifetime: every emitted node is arena-owned (rocke_ir_builder_t.arena). Nothing * is freed individually; the arena bulk-frees the whole graph. */ +#include /* snprintf */ +#include /* memset */ + +#include "rocke/arena.h" /* rocke_arena_calloc */ #include "rocke/instance_gfx950_attention_tiled_2d_internal.h" /* ========================================================================== @@ -88,16 +92,20 @@ static int rocke__wave_bytes(const rocke_gfx950_attn2d_build_ctx_t* ctx) return ctx->WAVE * 16; } -/* kv_block_bytes_c value = BS * NUM_KV * HD * KV_BYTES (one-block buffer bound, - * Python 1458). Python creates this const ONCE and reuses it in every loader; - * cache it on ctx so we do not allocate a duplicate const per loader (which would - * shift downstream %values). */ +/* kv_block_bytes_c = kv_stride_blk_b (one-block buffer bound). Python derives + * the token and block strides ONCE and reuses them in every loader; cache them + * on ctx so we do not emit a duplicate multiply per loader (which would shift + * downstream %values). */ static rocke_value_t* rocke__kv_block_bytes_c(rocke_gfx950_attn2d_build_ctx_t* ctx) { if(ctx->kv_block_bytes_c_v == NULL) { - int kv_stride_blk_b = ctx->BS * ctx->NUM_KV * ctx->HD * ctx->KV_BYTES; - ctx->kv_block_bytes_c_v = rocke_b_const_i32(ctx->b, kv_stride_blk_b); + /* The token and block strides scale with the runtime kv head count. */ + ctx->kv_stride_tok_b_v = rocke_b_mul( + ctx->b, ctx->num_kv_p, rocke_b_const_i32(ctx->b, ctx->HD * ctx->KV_BYTES)); + ctx->kv_stride_blk_b_v + = rocke_b_mul(ctx->b, ctx->kv_stride_tok_b_v, rocke_b_const_i32(ctx->b, ctx->BS)); + ctx->kv_block_bytes_c_v = ctx->kv_stride_blk_b_v; } return ctx->kv_block_bytes_c_v; } @@ -175,6 +183,167 @@ static rocke_value_t* rocke__wave_lds_offset_i64(rocke_gfx950_attn2d_build_ctx_t return ctx->wave_lds_off_i64_v; } +/* When a copy covers copy_tokens whole tokens and they fit in one page, the + * bf16/fp16 loaders address a copy as its page's base, its first token's + * offset in the page and the lane's own part. A copy that ends mid-row + * (head_size 192 with one warp: 512 halves are 2 2/3 rows) or spans pages + * keeps the paged-KV descriptor, which addresses each lane's own row. The token stride is a kernarg, so the lane part and the + * per-copy token offsets are built once, after the paged-KV descriptor, rather + * than multiplying the token index in every copy. */ +static int rocke__copy_tokens(const rocke_gfx950_attn2d_build_ctx_t* ctx) +{ + return rocke__kv_halves_per_call(ctx) / ctx->HD; +} + +static bool rocke__page_copies(const rocke_gfx950_attn2d_build_ctx_t* ctx) +{ + return !ctx->FAST_PAGED_KV_DESC && !ctx->KV_FP8 && rocke__kv_halves_per_call(ctx) % ctx->HD == 0 + && ctx->BS % rocke__copy_tokens(ctx) == 0; +} + +/* A zeroed table of `n` IR values on the builder arena, sized from the spec + * (the Python twins are dicts, so no fixed bound applies). NULL with the + * builder's error set on OOM. */ +static rocke_value_t** rocke__value_table(rocke_gfx950_attn2d_build_ctx_t* ctx, int n) +{ + rocke_ir_builder_t* b = ctx->b; + rocke_value_t** t = (rocke_value_t**)rocke_arena_calloc( + &b->arena, (size_t)(n > 0 ? n : 1) * sizeof(rocke_value_t*)); + if(t == NULL) + { + b->status = ROCKE_ERR_OOM; + snprintf(b->err, ROCKE_ERR_MSG_CAP, "out of memory for a %d-entry K/V table", n); + } + return t; +} + +static void rocke__emit_paged_copy_offsets(rocke_gfx950_attn2d_build_ctx_t* ctx) +{ + rocke_ir_builder_t* b = ctx->b; + const int copy_tokens = rocke__copy_tokens(ctx); + const int kv_calls_per_tile = rocke__kv_calls_per_tile(ctx); + rocke_value_t* lane_half_base = rocke__lane_half_base(ctx); + + const int copies_per_page = ctx->BS / copy_tokens; + + ctx->kv_stride_blk_i64_v = NULL; + ctx->copy_tok_b = rocke__value_table(ctx, copies_per_page); + if(ctx->copy_tok_b == NULL) + return; + rocke_value_t* c_hd0 = rocke_b_const_i32(b, ctx->HD); + rocke_value_t* lane_tok = rocke_b_div(b, lane_half_base, c_hd0); + rocke_value_t* c_hd1 = rocke_b_const_i32(b, ctx->HD); + rocke_value_t* lane_dim = rocke_b_mod(b, lane_half_base, c_hd1); + rocke_value_t* tok_b = rocke_b_mul(b, lane_tok, ctx->kv_stride_tok_b_v); + rocke_value_t* c_h = rocke_b_const_i32(b, ctx->HD * ctx->KV_BYTES); + rocke_value_t* head_b = rocke_b_mul(b, ctx->kv_head_idx, c_h); + rocke_value_t* tok_head = rocke_b_add(b, tok_b, head_b); + rocke_value_t* c_bytes = rocke_b_const_i32(b, ctx->KV_BYTES); + rocke_value_t* dim_b = rocke_b_mul(b, lane_dim, c_bytes); + ctx->lane_within_b_v = rocke_b_add(b, tok_head, dim_b); + for(int call = 0; call < kv_calls_per_tile; ++call) + { + const int tok_i = call % copies_per_page; + if(tok_i == 0 || ctx->copy_tok_b[tok_i] != NULL) + continue; + rocke_value_t* c_tok = rocke_b_const_i32(b, tok_i * copy_tokens); + ctx->copy_tok_b[tok_i] = rocke_b_mul(b, ctx->kv_stride_tok_b_v, c_tok); + } + if(ctx->I64_KV_ADDR) + ctx->kv_stride_blk_i64_v = rocke_b_zext(b, ctx->kv_stride_blk_b_v, rocke_i64()); +} + +/* The paged-KV descriptor's indirect lookup for page `blk` of a tile: one + * guarded block-table read, wave-uniform. */ +static rocke_value_t* + rocke__tile_page_id(rocke_gfx950_attn2d_build_ctx_t* ctx, rocke_value_t* kv_tile_idx, int blk) +{ + rocke_ir_builder_t* b = ctx->b; + rocke_value_t* tile_blk = kv_tile_idx; + if(ctx->N_BLOCKS_PER_TILE != 1) + { + rocke_value_t* c_nb = rocke_b_const_i32(b, ctx->N_BLOCKS_PER_TILE); + rocke_value_t* first = rocke_b_mul(b, kv_tile_idx, c_nb); + rocke_value_t* c_blk = rocke_b_const_i32(b, blk); + tile_blk = rocke_b_add(b, first, c_blk); + } + rocke_value_t* idx = rocke_b_add(b, rocke__seq_base(ctx), tile_blk); + rocke_value_t* mask = rocke_b_cmp_lt(b, idx, rocke__block_table_max_idx(ctx)); + rocke_value_t* zero = rocke_b_const_i32(b, 0); + rocke_value_t* page + = rocke_b_masked_global_load(b, ctx->block_tables, idx, mask, zero, rocke_i32(), 4); + return rocke_b_to_sgpr_u32(b, page); +} + +bool rocke_gfx950_attn2d_page_prefetch(const rocke_gfx950_attn2d_build_ctx_t* ctx) +{ + /* The C++ port has no deep K ring (kv_ring_depth != 2 is refused). */ + return rocke__page_copies(ctx) && !ctx->GROUPED_KV2 + && ctx->N_BLOCKS_PER_TILE <= ROCKE_GFX950_ATTN2D_MAX_PREFETCH_PAGES; +} + +void rocke_gfx950_attn2d_tile_page_ids(rocke_gfx950_attn2d_build_ctx_t* ctx, + rocke_value_t* kv_tile_idx, + rocke_value_t** out) +{ + for(int j = 0; j < ctx->N_BLOCKS_PER_TILE; ++j) + out[j] = rocke__tile_page_id(ctx, kv_tile_idx, j); +} + +/* The page ids the loop carries for `kv_tile_idx`, or NULL when it reads them + * in place. */ +static rocke_value_t* const* rocke__carried_pages(const rocke_gfx950_attn2d_build_ctx_t* ctx, + const rocke_value_t* kv_tile_idx) +{ + if(!rocke_gfx950_attn2d_page_prefetch(ctx)) + return NULL; + if(kv_tile_idx == ctx->page_cur_tile) + return ctx->page_cur; + if(kv_tile_idx == ctx->page_next_tile) + return ctx->page_next; + if(kv_tile_idx == ctx->page_start_tile) + return ctx->page_start; + return NULL; +} + +/* (rsrc, voffset, soffset) of one bf16/fp16 K/V copy. `pages` caches this + * tile's wave-uniform page ids by block within the tile. */ +static void rocke__paged_copy(rocke_gfx950_attn2d_build_ctx_t* ctx, + rocke_value_t* src, + rocke_value_t* src_rsrc, + rocke_value_t* kv_tile_idx, + int call, + rocke_value_t** pages, + rocke_value_t** out_rsrc, + rocke_value_t** out_voff, + rocke_value_t** out_soff) +{ + rocke_ir_builder_t* b = ctx->b; + const int copy_tokens = rocke__copy_tokens(ctx); + const int blk = (call * copy_tokens) / ctx->BS; + const int tok_i = call % (ctx->BS / copy_tokens); + rocke_value_t* const* carried = rocke__carried_pages(ctx, kv_tile_idx); + if(carried != NULL) + pages = (rocke_value_t**)carried; + else if(pages[blk] == NULL) + pages[blk] = rocke__tile_page_id(ctx, kv_tile_idx, blk); + rocke_value_t* voff = ctx->lane_within_b_v; + if(tok_i != 0) + voff = rocke_b_add(b, voff, ctx->copy_tok_b[tok_i]); + *out_voff = voff; + if(ctx->I64_KV_ADDR) + { + rocke_value_t* page_i64 = rocke_b_zext(b, pages[blk], rocke_i64()); + rocke_value_t* base_i64 = rocke_b_mul(b, page_i64, ctx->kv_stride_blk_i64_v); + rocke_value_t* ptr = rocke_b_global_ptr_add(b, src, base_i64); + *out_rsrc = rocke_b_buffer_rsrc(b, ptr, rocke__kv_block_bytes_c(ctx)); + *out_soff = rocke__zero_soff(ctx); + return; + } + *out_rsrc = src_rsrc; + *out_soff = rocke_b_mul(b, pages[blk], ctx->kv_stride_blk_b_v); +} + /* ========================================================================== * Pre-loop: K/V buffer resources, byte derivation, paged-KV descriptor * (Python lines 1163-1630, the K/V-descriptor parts). @@ -198,9 +367,9 @@ void rocke_gfx950_attn2d_emit_preloop(rocke_gfx950_attn2d_build_ctx_t* ctx) (void)((ctx->T * ctx->HD) % kv_halves_per_call); } - /* kv_block_bytes_c (1458): Python creates this one-block-bound const here, - * after the byte-stride derivation and BEFORE lane_half_base, and reuses it - * in every loader. Emit it now so its SSA value lands in source order. */ + /* kv_block_bytes_c (1458): Python derives the byte strides and this + * one-block bound here, BEFORE lane_half_base, and reuses them in every + * loader. Emit them now so their SSA values land in source order. */ (void)rocke__kv_block_bytes_c(ctx); /* Per-lane starting half index (1460), then K/V LDS base ptrtoints (1462- @@ -243,18 +412,15 @@ void rocke_gfx950_attn2d_emit_preloop(rocke_gfx950_attn2d_build_ctx_t* ctx) } /* ---- Paged KV byte descriptor (full transform DAG, 1591-1629) ---- - * _kv_base = naive("paged_kv_bytes", - * lengths=[1<<24, BS, NUM_KV, HD], - * strides=[kv_stride_blk_b, kv_stride_tok_b, kv_stride_h_b, KV_BYTES], - * coord_names=("physical_block","token","kv_head","dim")). */ - int blk_b = ctx->BS * ctx->NUM_KV * ctx->HD * ctx->KV_BYTES; - int tok_b = ctx->NUM_KV * ctx->HD * ctx->KV_BYTES; - int h_b = ctx->HD * ctx->KV_BYTES; - int kv_lengths[4] = {1 << 24, ctx->BS, ctx->NUM_KV, ctx->HD}; - int kv_strides[4] = {blk_b, tok_b, h_b, ctx->KV_BYTES}; + * _kv_base = DynamicTensorDescriptor.create("paged_kv_bytes", + * coord_names=("physical_block","token","kv_head","dim"), + * strides=(kv_stride_blk_b, kv_stride_tok_b, kv_stride_h_b, KV_BYTES)), + * the first two the runtime strides derived above. */ + int kv_strides[4] = {1, 1, ctx->HD * ctx->KV_BYTES, ctx->KV_BYTES}; + rocke_value_t* kv_dyn[4] = {ctx->kv_stride_blk_b_v, ctx->kv_stride_tok_b_v, NULL, NULL}; static const char* const kv_coords[4] = {"physical_block", "token", "kv_head", "dim"}; - rocke_tensor_descriptor_t* kv_base = rocke_tensor_descriptor_naive( - b, "paged_kv_bytes", kv_lengths, 4, kv_strides, kv_coords, 4); + rocke_tensor_descriptor_t* kv_base = rocke_tensor_descriptor_naive_dynamic_mixed( + b, "paged_kv_bytes", kv_coords, 4, kv_dyn, kv_strides); /* Single-block tile (N_BLOCKS_PER_TILE == 1, T == BS): * indirect(tile_idx -> physical_block) ; unmerge(linear_half -> token,dim). @@ -285,6 +451,8 @@ void rocke_gfx950_attn2d_emit_preloop(rocke_gfx950_attn2d_build_ctx_t* ctx) b, "linear_block_idx", "physical_block", ctx->block_tables, seq_base, max_idx, 0); ctx->kv_desc = rocke_tensor_descriptor_transform(b, kv_base, chain, 3); } + if(rocke__page_copies(ctx)) + rocke__emit_paged_copy_offsets(ctx); /* fp8 async-loader setup (Python 1785-1796), after the paged-KV * descriptor DAG. */ @@ -350,8 +518,9 @@ static void rocke__fast_paged_kv_voff_split(rocke_gfx950_attn2d_build_ctx_t* ctx if(ctx->I64_KV_ADDR) { /* base_i64 = zext(physical, i64) << 15 ; return (base_i64, within). */ - rocke_value_t* base_i64 - = rocke_b_shl(b, rocke_b_zext(b, physical, rocke_i64()), rocke_b_const_i64(b, 15)); + /* zext before the shift constant, as Python evaluates them. */ + rocke_value_t* physical_i64 = rocke_b_zext(b, physical, rocke_i64()); + rocke_value_t* base_i64 = rocke_b_shl(b, physical_i64, rocke_b_const_i64(b, 15)); if(out_base_i64) *out_base_i64 = base_i64; if(out_voff) @@ -410,10 +579,15 @@ void rocke_gfx950_attn2d_issue_k_load_runtime(rocke_gfx950_attn2d_build_ctx_t* c if(ctx->FAST_PAGED_KV_DESC) rocke_gfx950_attn2d_fast_paged_kv_blocks(ctx, kv_tile_idx, &fast_block0, &fast_block1); + /* This tile's page ids by block, read on first use. */ + rocke_value_t** pages = rocke__value_table(ctx, ctx->N_BLOCKS_PER_TILE); + if(pages == NULL) + return; for(int call = 0; call < kv_calls_per_tile; ++call) { rocke_value_t* k_rsrc = ctx->k_rsrc; rocke_value_t* voff = NULL; + rocke_value_t* soff = zero_soff; if(ctx->FAST_PAGED_KV_DESC) { rocke_value_t* base_i64 = NULL; @@ -424,6 +598,11 @@ void rocke_gfx950_attn2d_issue_k_load_runtime(rocke_gfx950_attn2d_build_ctx_t* c b, rocke_b_global_ptr_add(b, ctx->key, base_i64), kv_block_bytes_c); } } + else if(rocke__page_copies(ctx)) + { + rocke__paged_copy( + ctx, ctx->key, ctx->k_rsrc, kv_tile_idx, call, pages, &k_rsrc, &voff, &soff); + } else { /* Non-fast paged-KV (N_BLOCKS_PER_TILE==1 / multi-block): @@ -460,8 +639,7 @@ void rocke_gfx950_attn2d_issue_k_load_runtime(rocke_gfx950_attn2d_build_ctx_t* c rocke_value_t* k_dst = rocke_b_smem_ptr_add( b, K_wave_base, rocke_b_const_i64(b, (int64_t)call * bytes_per_call)); /* CACHE_STREAM (SLC): one-shot streaming load; dwords=4 (1687-1689). */ - rocke_b_async_buffer_load_lds_addr( - b, k_rsrc, k_dst, voff, zero_soff, 4, ROCKE_CACHE_STREAM); + rocke_b_async_buffer_load_lds_addr(b, k_rsrc, k_dst, voff, soff, 4, ROCKE_CACHE_STREAM); } } @@ -489,10 +667,15 @@ void rocke_gfx950_attn2d_issue_v_load_runtime(rocke_gfx950_attn2d_build_ctx_t* c if(ctx->FAST_PAGED_KV_DESC) rocke_gfx950_attn2d_fast_paged_kv_blocks(ctx, kv_tile_idx, &fast_block0, &fast_block1); + /* This tile's page ids by block, read on first use. */ + rocke_value_t** pages = rocke__value_table(ctx, ctx->N_BLOCKS_PER_TILE); + if(pages == NULL) + return; for(int call = 0; call < kv_calls_per_tile; ++call) { rocke_value_t* v_rsrc = ctx->v_rsrc; rocke_value_t* voff = NULL; + rocke_value_t* soff = zero_soff; if(ctx->FAST_PAGED_KV_DESC) { rocke_value_t* base_i64 = NULL; @@ -503,6 +686,11 @@ void rocke_gfx950_attn2d_issue_v_load_runtime(rocke_gfx950_attn2d_build_ctx_t* c b, rocke_b_global_ptr_add(b, ctx->value, base_i64), kv_block_bytes_c); } } + else if(rocke__page_copies(ctx)) + { + rocke__paged_copy( + ctx, ctx->value, ctx->v_rsrc, kv_tile_idx, call, pages, &v_rsrc, &voff, &soff); + } else { /* Non-fast paged-KV: voff = paged_kv_desc.offset(tile_idx=, @@ -538,8 +726,7 @@ void rocke_gfx950_attn2d_issue_v_load_runtime(rocke_gfx950_attn2d_build_ctx_t* c rocke_value_t* v_dst = rocke_b_smem_ptr_add( b, V_wave_base, rocke_b_const_i64(b, (int64_t)call * bytes_per_call)); /* CACHE_STREAM (SLC): V consumed once per iter; dwords=4 (1737-1739). */ - rocke_b_async_buffer_load_lds_addr( - b, v_rsrc, v_dst, voff, zero_soff, 4, ROCKE_CACHE_STREAM); + rocke_b_async_buffer_load_lds_addr(b, v_rsrc, v_dst, voff, soff, 4, ROCKE_CACHE_STREAM); } } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_public_entry_glue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_public_entry_glue.cpp index 390a1d1ae31f..3fd542998938 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_public_entry_glue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_public_entry_glue.cpp @@ -42,10 +42,12 @@ #include "rocke/instance_gfx950_attention_tiled_2d.h" #include "rocke/instance_gfx950_attention_tiled_2d_internal.h" +#include "rocke/arena.h" /* rocke_arena_strdup */ #include "rocke/error_boundary.hpp" /* ckc::guard_builder boundary shim */ #include "rocke/helper_helper_rocke.helpers.attention.h" /* binary_search_seq_idx */ #include "rocke/helper_helper_rocke.instances.gfx942.attention_tiled_2d.h" /* rocke__mfma_32x32_c_* */ #include "rocke/helper_rocke.helpers.spec.h" /* rocke_kernel_name_join */ +#include "rocke/helper_rocke.instances.common._fmha_common.h" /* rocke_attention_check_head_counts */ #include #include @@ -563,7 +565,6 @@ static bool rocke_gfx950_attn2d_build_ctx_init(rocke_gfx950_attn2d_build_ctx_t* const int BLOCK_Q = rocke_attention_tiled_2d_spec_block_q(spec); const int NQK = rocke_attention_tiled_2d_spec_num_queries_per_kv(spec); const int NUM_KV = spec->num_kv_heads; - const int NUM_QH = spec->num_query_heads; const int SLIDING_WINDOW = spec->sliding_window; ctx->HD = HD; @@ -575,7 +576,6 @@ static bool rocke_gfx950_attn2d_build_ctx_init(rocke_gfx950_attn2d_build_ctx_t* ctx->VALID_ROWS = BLOCK_Q * NQK; ctx->NQK = NQK; ctx->NUM_KV = NUM_KV; - ctx->NUM_QH = NUM_QH; ctx->SLIDING_WINDOW = SLIDING_WINDOW; ctx->USE_SOFTCAP = spec->has_softcap; ctx->USE_SINKS = spec->use_sinks; @@ -739,6 +739,9 @@ static bool rocke_gfx950_attn2d_build_ctx_init(rocke_gfx950_attn2d_build_ctx_t* ctx->num_seqs_p = rocke_b_param(b, "num_seqs", rocke_i32(), NULL); ctx->bt_stride_p = rocke_b_param(b, "block_table_stride", rocke_i32(), NULL); ctx->qq_bias_stride0_p = rocke_b_param(b, "qq_bias_stride_0", rocke_i32(), NULL); + ctx->num_qh_p = rocke_b_param(b, "num_query_heads", rocke_i32(), NULL); + ctx->num_kv_p = rocke_b_param(b, "num_kv_heads", rocke_i32(), NULL); + ctx->sliding_window_p = rocke_b_param(b, "sliding_window", rocke_i32(), NULL); /* Trailing LSE output (Python: after the optional additive-bias params, * which this twin does not declare). Off -> no param, ABI unchanged. */ if(spec->emit_lse) @@ -784,14 +787,11 @@ static bool rocke_gfx950_attn2d_build_ctx_init(rocke_gfx950_attn2d_build_ctx_t* } /* ---- seq lookup + Q-block geometry (Py906-925) ---- */ - ctx->seq_idx - = rocke_binary_search_seq_idx(b, - ctx->cu_q, - ctx->q_block_global_idx, - ctx->num_seqs_p, - BLOCK_Q, - rocke_attention_tiled_2d_spec_binary_search_iters(spec), - false); + { + rocke_value_t* trip = rocke_binary_search_seq_idx_trip_count(b, ctx->num_seqs_p); + ctx->seq_idx = rocke_binary_search_seq_idx_dyn( + b, ctx->cu_q, ctx->q_block_global_idx, ctx->num_seqs_p, BLOCK_Q, trip, false); + } ctx->cu_q_start = rocke_b_global_load_i32(b, ctx->cu_q, ctx->seq_idx, 0); ctx->cu_q_stop = rocke_b_global_load_i32( b, ctx->cu_q, rocke_b_add(b, ctx->seq_idx, rocke_b_const_i32(b, 1)), 0); @@ -935,8 +935,7 @@ static bool rocke_gfx950_attn2d_build_ctx_init(rocke_gfx950_attn2d_build_ctx_t* * here so the SSA id matches Python; the PV epilogue reuses this value. */ ctx->pv_fp8_scale_v = FP8_MFMA_PV ? rocke_b_fdiv(b, ctx->v_scale_p, rocke_b_const_f32(b, 240.0)) : NULL; - ctx->sw_const_v = rocke_b_const_i32(b, SLIDING_WINDOW); /* sw_const (1150) */ - ctx->c0 = ctx->sw_const_v; + ctx->sw_const_v = ctx->sliding_window_p; /* sw_const = rt["sliding_window"] */ /* ---- acc geometry the epilogue aliases (Py1381-1382) ---- */ ctx->ACC_N_TILES = USE_MFMA_32X32 ? (HD / 32) : PV_N_TILES; @@ -1046,7 +1045,7 @@ static rocke_status_t rocke_g950_attn2d_kernel_name(const rocke_attention_tiled_ char* out, size_t out_cap) { - char d_buf[32], b_buf[32], t_buf[32], hkv_buf[64], kv_buf[64]; + char d_buf[32], b_buf[32], t_buf[32], hkv_buf[64], fhkv_buf[64], kv_buf[64]; char sw_buf[32], w_buf[32], mw_buf[32], schedb_buf[32], smxil_buf[32]; int nqh, nkv; @@ -1055,6 +1054,9 @@ static rocke_status_t rocke_g950_attn2d_kernel_name(const rocke_attention_tiled_ nqh = s->num_query_heads; nkv = s->num_kv_heads; + /* The name carries the GQA ratio; reject a spec it cannot be formed for. */ + if(nkv <= 0) + return ROCKE_ERR_VALUE; snprintf(d_buf, sizeof(d_buf), "d%d", s->head_size); snprintf(b_buf, sizeof(b_buf), "b%d", s->block_size); @@ -1062,13 +1064,20 @@ static rocke_status_t rocke_g950_attn2d_kernel_name(const rocke_attention_tiled_ snprintf(t_buf, sizeof(t_buf), "t%d", rocke_attention_tiled_2d_spec_tile_size_eff(s)); else t_buf[0] = '\0'; - snprintf(hkv_buf, sizeof(hkv_buf), "h%dkv%d", nqh, nkv); + /* Head counts are kernargs; only the GQA ratio is baked. */ + snprintf(hkv_buf, sizeof(hkv_buf), "g%d", nqh / nkv); + /* Except on the fast paged-KV descriptor path, whose shifts bake them. */ + if(s->use_fast_paged_kv_desc) + snprintf(fhkv_buf, sizeof(fhkv_buf), "h%dkv%d", nqh, nkv); + else + fhkv_buf[0] = '\0'; if(s->kv_storage_dtype) snprintf(kv_buf, sizeof(kv_buf), "kv%s", s->kv_storage_dtype); else kv_buf[0] = '\0'; + /* The window size is a kernarg; only whether the body masks one is baked. */ if(s->sliding_window > 0) - snprintf(sw_buf, sizeof(sw_buf), "sw%d", s->sliding_window); + snprintf(sw_buf, sizeof(sw_buf), "sw"); else sw_buf[0] = '\0'; if(s->num_warps != 1) @@ -1104,8 +1113,11 @@ static rocke_status_t rocke_g950_attn2d_kernel_name(const rocke_attention_tiled_ b_buf, t_buf, hkv_buf, + fhkv_buf, s->dtype ? s->dtype : "", kv_buf, + /* i64 K/V addressing changes the loaders, so it is a distinct kernel. */ + s->use_i64_kv_addr ? "i64" : "", s->use_sinks ? "sinks" : "", sw_buf, s->has_softcap ? "softcap" : "", @@ -1150,14 +1162,25 @@ static rocke_status_t rocke_g950_attn2d_kernel_name(const rocke_attention_tiled_ rocke_kernel_def_t* rocke_gfx950_build_unified_attention_2d_tiled_new( rocke_ir_builder_t* b, const rocke_attention_tiled_2d_spec_t* spec, const char* arch) { - char kname[1024]; - if(b == NULL || spec == NULL) - return NULL; - if(rocke_g950_attn2d_kernel_name(spec, kname, sizeof(kname)) != ROCKE_OK) - return NULL; - if(rocke_ir_builder_init(b, kname) != ROCKE_OK) - return NULL; - return rocke_gfx950_build_unified_attention_2d_tiled(b, spec, arch); + return ckc::guard_builder(b, [&]() -> rocke_kernel_def_t* { + char kname[1024]; + char why[160]; + if(b == NULL || spec == NULL) + return NULL; + /* Init before anything can fail: the lower convenience reads and frees + * `b` whatever this returns. */ + if(rocke_ir_builder_init(b, "") != ROCKE_OK) + return NULL; + if(!rocke_attention_check_head_counts( + spec->num_query_heads, spec->num_kv_heads, why, sizeof(why))) + ckc::raise_status(ROCKE_ERR_VALUE, why); + if(rocke_g950_attn2d_kernel_name(spec, kname, sizeof(kname)) != ROCKE_OK) + ckc::raise_status(ROCKE_ERR_VALUE, "tiled 2D kernel_name encode failed"); + b->kernel->name = rocke_arena_strdup(&b->arena, kname); + if(b->kernel->name == NULL) + ckc::raise_status(ROCKE_ERR_OOM, "tiled 2D kernel name OOM"); + return rocke_gfx950_build_unified_attention_2d_tiled(b, spec, arch); + }); } /* ===================================================================== * diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_q_load_gather.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_q_load_gather.cpp index 81b1e09a4da5..ae3d0609cf0f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_q_load_gather.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_2d_q_load_gather.cpp @@ -110,16 +110,16 @@ void rocke_gfx950_attn2d_emit_q_load(rocke_gfx950_attn2d_build_ctx_t* ctx) Q_VECS_PER_ROW = ctx->HD / 8; Q_VECS_PER_THREAD = (ctx->BLOCK_M * Q_VECS_PER_ROW) / ctx->THREADS; - /* q_desc = TensorDescriptor.naive("Q", lengths=[1<<30, NUM_QH, HD], - * coord_names=("token","head","dim")). Stored on ctx for the - * downstream Q gather + the epilogue output store. */ + /* q_desc over (token, head, dim), stored on ctx for the downstream Q gather + * + the epilogue output store. The head count is a kernarg, so the token + * stride is too. */ { - int lengths[3]; static const char* const coord_names[3] = {"token", "head", "dim"}; - lengths[0] = 1 << 30; - lengths[1] = ctx->NUM_QH; - lengths[2] = ctx->HD; - ctx->q_desc = rocke_tensor_descriptor_naive(b, "Q", lengths, 3, NULL, coord_names, 3); + rocke_value_t* strides[3] + = {rocke_b_mul(b, ctx->num_qh_p, rocke_b_const_i32(b, ctx->HD)), NULL, NULL}; + const int static_strides[3] = {1, ctx->HD, 1}; + ctx->q_desc = rocke_tensor_descriptor_naive_dynamic_mixed( + b, "Q", coord_names, 3, strides, static_strides); } for(li = 0; li < Q_VECS_PER_THREAD; ++li) @@ -164,12 +164,11 @@ void rocke_gfx950_attn2d_emit_q_load(rocke_gfx950_attn2d_build_ctx_t* ctx) rocke_value_t* qh_mod = rocke_b_mod(b, Q_row, rocke_b_const_i32(b, (int64_t)ctx->NQK)); qh_t = rocke_b_add(b, qh_mul, qh_mod); } - /* qmask_t = (q_pos_t < cur_batch_q_len) && (qh_t < NUM_QH) [&& (Q_row < VALID_ROWS)]. + /* qmask_t = (q_pos_t < cur_batch_q_len) && (qh_t < num_query_heads) [&& (Q_row < VALID_ROWS)]. * Skip the VALID_ROWS guard when VALID_ROWS == BLOCK_M (NQK divides BLOCK_M). */ { rocke_value_t* lt_pos = rocke_b_cmp_lt(b, q_pos_t, ctx->cur_batch_q_len); - rocke_value_t* lt_qh - = rocke_b_cmp_lt(b, qh_t, rocke_b_const_i32(b, (int64_t)ctx->NUM_QH)); + rocke_value_t* lt_qh = rocke_b_cmp_lt(b, qh_t, ctx->num_qh_p); rocke_value_t* and_pos_qh = rocke_b_land(b, lt_pos, lt_qh); if(ctx->VALID_ROWS < ctx->BLOCK_M) { @@ -248,7 +247,6 @@ void rocke_gfx950_attn2d_emit_loop_bounds_and_inits(rocke_gfx950_attn2d_build_ct const int BLOCK_M = ctx->BLOCK_M; const int NQK = ctx->NQK; const int SLIDING_WINDOW = ctx->SLIDING_WINDOW; - const int NUM_QH = ctx->NUM_QH; const int SOFTMAX_STATE_SLOTS = ctx->SOFTMAX_STATE_SLOTS; /* ---- max_seq_prefix_len (1221-1223) ---- */ @@ -334,7 +332,7 @@ void rocke_gfx950_attn2d_emit_loop_bounds_and_inits(rocke_gfx950_attn2d_build_ct rocke_value_t* qh_mul = rocke_b_mul(B, ctx->kv_head_idx, rocke_b_const_i32(B, NQK)); rocke_value_t* qh_mod = rocke_b_mod(B, row, rocke_b_const_i32(B, NQK)); rocke_value_t* qh = rocke_b_add(B, qh_mul, qh_mod); - rocke_value_t* qh_in = rocke_b_cmp_lt(B, qh, rocke_b_const_i32(B, NUM_QH)); + rocke_value_t* qh_in = rocke_b_cmp_lt(B, qh, ctx->num_qh_p); rocke_value_t* sink_h = rocke_b_global_load(B, ctx->sinks, qh, ctx->dtype, 2); rocke_value_t* sink_f = rocke_b_fmul(B, rocke_b_cast_to_f32(B, sink_h), rcp_ln2); m_inits[0] = rocke_b_select(B, qh_in, sink_f, neg_inf); @@ -349,7 +347,7 @@ void rocke_gfx950_attn2d_emit_loop_bounds_and_inits(rocke_gfx950_attn2d_build_ct rocke_value_t* qh_mul = rocke_b_mul(B, ctx->kv_head_idx, rocke_b_const_i32(B, NQK)); rocke_value_t* qh_mod = rocke_b_mod(B, row, rocke_b_const_i32(B, NQK)); rocke_value_t* qh = rocke_b_add(B, qh_mul, qh_mod); - rocke_value_t* qh_in = rocke_b_cmp_lt(B, qh, rocke_b_const_i32(B, NUM_QH)); + rocke_value_t* qh_in = rocke_b_cmp_lt(B, qh, ctx->num_qh_p); rocke_value_t* sink_h = rocke_b_global_load(B, ctx->sinks, qh, ctx->dtype, 2); rocke_value_t* sink_f = rocke_b_fmul(B, rocke_b_cast_to_f32(B, sink_h), rcp_ln2); m_inits[r] = rocke_b_select(B, qh_in, sink_f, neg_inf); @@ -626,6 +624,20 @@ void rocke_gfx950_attn2d_emit_preloop_prefetch(rocke_gfx950_attn2d_build_ctx_t* if(ctx == NULL) return; + /* Page ids of the first tile and of the one after it, carried into the + * loop; iteration i then reads the ids of tile i + 2. */ + const bool page_prefetch = rocke_gfx950_attn2d_page_prefetch(ctx); + if(page_prefetch) + { + rocke_value_t* c_one = rocke_b_const_i32(B, 1); + rocke_value_t* start_next_raw = rocke_b_add(B, ctx->tile_start, c_one); + rocke_value_t* in_range = rocke_b_cmp_lt(B, start_next_raw, ctx->tile_end); + rocke_value_t* start_next = rocke_b_select(B, in_range, start_next_raw, ctx->tile_start); + rocke_gfx950_attn2d_tile_page_ids(ctx, ctx->tile_start, ctx->page_start); + rocke_gfx950_attn2d_tile_page_ids(ctx, start_next, ctx->page_start_next); + ctx->page_start_tile = ctx->tile_start; + } + /* Prefetch tile_start's K into buffer 0 BEFORE the loop (2384). */ rocke_gfx950_attn2d_issue_k(ctx, ctx->tile_start, rocke_b_const_i32(B, 0)); @@ -636,6 +648,36 @@ void rocke_gfx950_attn2d_emit_preloop_prefetch(rocke_gfx950_attn2d_build_ctx_t* ctx->iter_args_names[idx] = ctx->iter_args_name_buf[idx]; ctx->iter_args[idx] = rocke_b_const_i32(B, 0); ctx->iter_args_count = idx + 1; + if(page_prefetch + && ctx->iter_args_count + 2 * ctx->N_BLOCKS_PER_TILE > ROCKE_GFX950_ATTN2D_MAX_ITER_ARGS) + { + B->status = ROCKE_ERR_VALUE; + snprintf(B->err, + ROCKE_ERR_MSG_CAP, + "%d loop carries exceed the %d-entry iter_args table", + ctx->iter_args_count + 2 * ctx->N_BLOCKS_PER_TILE, + ROCKE_GFX950_ATTN2D_MAX_ITER_ARGS); + return; + } + if(page_prefetch) + { + for(int j = 0; j < ctx->N_BLOCKS_PER_TILE; ++j) + { + idx = ctx->iter_args_count++; + snprintf( + ctx->iter_args_name_buf[idx], sizeof(ctx->iter_args_name_buf[0]), "page_cur%d", j); + ctx->iter_args_names[idx] = ctx->iter_args_name_buf[idx]; + ctx->iter_args[idx] = ctx->page_start[j]; + } + for(int j = 0; j < ctx->N_BLOCKS_PER_TILE; ++j) + { + idx = ctx->iter_args_count++; + snprintf( + ctx->iter_args_name_buf[idx], sizeof(ctx->iter_args_name_buf[0]), "page_next%d", j); + ctx->iter_args_names[idx] = ctx->iter_args_name_buf[idx]; + ctx->iter_args[idx] = ctx->page_start_next[j]; + } + } /* kv_step (2482) is emitted by the caller AFTER emit_licm_hoist to keep the * const in Python emission order; see rocke_gfx950_attn2d_emit_kv_step. */ diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_public_glue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_public_glue.cpp index 939d46c55b52..80fe704bcddc 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_public_glue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_public_glue.cpp @@ -9,7 +9,7 @@ * SCOPE (this TU only): * - The shared spec PODs' _default() / _validate() (__post_init__) and every * @property accessor (num_queries_per_kv, block_m=16, block_q, tile_size, - * dtype_ir, binary_search_iters, kernel_name via kernel_name_join). These + * dtype_ir, kernel_name via kernel_name_join). These * spec struct/default definitions are GUARDED against the gfx942 3D header's * duplicate definitions (the two arch ports share the identical frozen * dataclass shape -- see the SHARED SPEC POD note in @@ -72,6 +72,7 @@ * _default() definitions below -- the gfx942 TU owns them. The structs are * byte-identical so the single shared definition is correct for both. */ #include "rocke/error_boundary.hpp" /* ckc::guard_builder boundary shim */ +#include "rocke/helper_rocke.instances.common._fmha_common.h" /* rocke_attention_check_head_counts */ #include "rocke/instance_gfx942_attention_tiled_3d.h" #include "rocke/instance_gfx950_attention_tiled_3d.h" #include "rocke/instance_gfx950_attention_tiled_3d_internal.h" @@ -427,7 +428,12 @@ int rocke_gfx950_unified_attention_3d_tiled_spec_kernel_name( snprintf(d_part, sizeof(d_part), "d%d", s->head_size); snprintf(b_part, sizeof(b_part), "b%d", s->block_size); - snprintf(h_part, sizeof(h_part), "h%dkv%d", s->num_query_heads, s->num_kv_heads); + /* Head counts are kernargs; only the GQA ratio is baked. */ + if(s->num_kv_heads <= 0) + { + return -1; + } + snprintf(h_part, sizeof(h_part), "g%d", s->num_query_heads / s->num_kv_heads); snprintf(seg_part, sizeof(seg_part), "seg%d", s->num_segments); parts[np++] = "rocke_uattn3d_tiled"; @@ -457,9 +463,10 @@ int rocke_gfx950_unified_attention_3d_tiled_spec_kernel_name( { parts[np++] = "sinks"; } + /* The window size is a kernarg; only whether the body masks one is baked. */ if(s->sliding_window > 0) { - snprintf(sw_part, sizeof(sw_part), "sw%d", s->sliding_window); + snprintf(sw_part, sizeof(sw_part), "sw"); parts[np++] = sw_part; } if(s->has_softcap) @@ -517,10 +524,9 @@ const rocke_type_t* rocke_gfx950_unified_attention_reduce_tiled_spec_dtype_ir( int rocke_gfx950_unified_attention_reduce_tiled_spec_kernel_name( const rocke_unified_attention_reduce_tiled_spec_t* s, char* buf, size_t cap) { - /* kernel_name_join("rocke_uattn_reduce_tiled", d, h, - * seg, dtype[, "lse"]) */ + /* kernel_name_join("rocke_uattn_reduce_tiled", d, seg, dtype + * [, "lse"]); the head count is a kernarg. */ char d_part[32]; - char h_part[32]; char seg_part[32]; const char* parts[8]; size_t np = 0; @@ -533,12 +539,10 @@ int rocke_gfx950_unified_attention_reduce_tiled_spec_kernel_name( } snprintf(d_part, sizeof(d_part), "d%d", s->head_size); - snprintf(h_part, sizeof(h_part), "h%d", s->num_query_heads); snprintf(seg_part, sizeof(seg_part), "seg%d", s->num_segments); parts[np++] = "rocke_uattn_reduce_tiled"; parts[np++] = d_part; - parts[np++] = h_part; parts[np++] = seg_part; parts[np++] = (s->dtype != NULL) ? s->dtype : ""; if(s->emit_lse) @@ -723,8 +727,6 @@ bool rocke_gfx950_attn_tiled_3d_config_from_spec( out->BLOCK_M = rocke_gfx950_unified_attention_3d_tiled_spec_block_m(spec); out->BLOCK_Q = rocke_gfx950_unified_attention_3d_tiled_spec_block_q(spec); out->NQK = rocke_gfx950_unified_attention_3d_tiled_spec_num_queries_per_kv(spec); - out->NUM_KV = spec->num_kv_heads; - out->NUM_QH = spec->num_query_heads; out->NUM_SEG = spec->num_segments; out->SLIDING_WINDOW = spec->sliding_window; out->USE_SOFTCAP = spec->has_softcap; @@ -758,8 +760,6 @@ bool rocke_gfx950_attn_tiled_3d_config_from_spec( out->PV_N_TILES = out->HD / ROCKE_ATTN3D950_MFMA_N; out->THREADS = ROCKE_ATTN3D950_SEGMENT_THREADS; - out->binary_search_iters - = rocke_gfx950_unified_attention_3d_tiled_spec_binary_search_iters(spec); if(out->PV_N_TILES < 0 || out->PV_N_TILES > 16) { @@ -786,8 +786,6 @@ bool rocke_gfx950_attn_tiled_3d_config_from_spec( } out->kv_calls_per_tile = (out->T * out->HD) / out->KV_HALVES_PER_CALL; out->bytes_per_call = out->KV_HALVES_PER_CALL * 2; - out->kv_stride_blk_b = out->BS * out->NUM_KV * out->HD * out->KV_BYTES; - out->kv_stride_tok_b = out->NUM_KV * out->HD * out->KV_BYTES; out->kv_stride_h_b = out->HD * out->KV_BYTES; out->bytes_per_buf = out->T * out->HD * 2; out->N_BLOCKS_PER_TILE = out->BS > 0 ? out->T / out->BS : 0; @@ -812,6 +810,15 @@ bool rocke_gfx950_attn_tiled_3d_config_from_spec( return false; } out->calls_per_block = out->BS / out->tokens_per_call; + /* The prologue caches each call's in-page token offset in call_tok_b. */ + if(out->calls_per_block > 64) + { + rocke_i_set_err(b, + ROCKE_ERR_VALUE, + "%d calls per page exceed the 64-entry offset table", + out->calls_per_block); + return false; + } } out->fp8_elems_per_chunk = 8; @@ -872,8 +879,6 @@ bool rocke_gfx950_attn_tiled_3d_reduce_config_from_spec( out->HD = spec->head_size; out->NUM_SEG = spec->num_segments; - out->NUM_QH = spec->num_query_heads; - out->NUM_KV = spec->num_kv_heads; out->dtype = rocke_gfx950_unified_attention_reduce_tiled_spec_dtype_ir(spec); out->THREADS = 64; @@ -1241,6 +1246,10 @@ rocke_status_t rocke_build_unified_attention_3d_tiled_gfx950_lower_to_llvm( return ROCKE_ERR_VALUE; } + /* The name carries the GQA ratio: refuse a head pair it cannot be formed + * for with the validator's reason, not the encoder's. */ + if(!rocke_attention_check_head_counts(spec->num_query_heads, spec->num_kv_heads, err, err_cap)) + return ROCKE_ERR_VALUE; if(rocke_gfx950_unified_attention_3d_tiled_spec_kernel_name(spec, name, sizeof(name)) < 0) { rocke_attn3d950_set_err_buf(err, err_cap, "lower_to_llvm: kernel_name encode failed"); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_reduce.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_reduce.cpp index 91f4a05f1728..14a506cf91aa 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_reduce.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_reduce.cpp @@ -136,7 +136,6 @@ void rocke_gfx950_attention_tiled_3d_reduce_declare_and_prologue( const rocke_gfx950_attn_tiled_3d_config_t* cfg = &ctx->cfg; rocke_param_opts_t opts; - int NUM_QH = cfg->NUM_QH; int NUM_SEG = cfg->NUM_SEG; int HD = cfg->HD; @@ -190,6 +189,9 @@ void rocke_gfx950_attention_tiled_3d_reduce_declare_and_prologue( ctx->red_seq_lens = rocke_b_param(b, "seq_lens_ptr", rocke_ptr_type(b, rocke_i32(), "global"), &opts); + /* RUNTIME_KERNARG_FIELDS : the query head count */ + ctx->num_qh_p = rocke_b_param(b, "num_query_heads", rocke_i32(), NULL); + /* lse_ptr : ptr, noalias, writeonly, align=4 (emit_lse only) */ ctx->red_lse = NULL; if(spec != NULL && spec->emit_lse) @@ -213,24 +215,31 @@ void rocke_gfx950_attention_tiled_3d_reduce_declare_and_prologue( ctx->neg_inf = rocke_b_const_f32(b, -INFINITY); ctx->zero_f = rocke_b_const_f32(b, 0.0); - /* ---- descriptors (lines 1042-1056) ---- */ + /* ---- descriptors (lines 1042-1056) ---- + * The head count is a kernarg, so each token stride is too. */ { - int ml_lengths[3] = {1 << 30, NUM_QH, NUM_SEG}; - const char* ml_coords[3] = {"token", "head", "seg"}; - ctx->ml_desc_red - = rocke_tensor_descriptor_naive(b, "segm_ml", ml_lengths, 3, NULL, ml_coords, 3); + static const char* const ml_coords[3] = {"token", "head", "seg"}; + rocke_value_t* ml_dyn[3] + = {rocke_b_mul(b, ctx->num_qh_p, rocke_b_const_i32(b, NUM_SEG)), NULL, NULL}; + const int ml_static[3] = {1, NUM_SEG, 1}; + ctx->ml_desc_red = rocke_tensor_descriptor_naive_dynamic_mixed( + b, "segm_ml", ml_coords, 3, ml_dyn, ml_static); } { - int sa_lengths[4] = {1 << 30, NUM_QH, NUM_SEG, HD}; - const char* sa_coords[4] = {"token", "head", "seg", "dim"}; - ctx->seg_acc_desc_red - = rocke_tensor_descriptor_naive(b, "segm_output", sa_lengths, 4, NULL, sa_coords, 4); + static const char* const sa_coords[4] = {"token", "head", "seg", "dim"}; + rocke_value_t* sa_dyn[4] + = {rocke_b_mul(b, ctx->num_qh_p, rocke_b_const_i32(b, NUM_SEG * HD)), NULL, NULL, NULL}; + const int sa_static[4] = {1, NUM_SEG * HD, HD, 1}; + ctx->seg_acc_desc_red = rocke_tensor_descriptor_naive_dynamic_mixed( + b, "segm_output", sa_coords, 4, sa_dyn, sa_static); } { - int out_lengths[3] = {1 << 30, NUM_QH, HD}; - const char* out_coords[3] = {"token", "head", "dim"}; - ctx->out_desc_red - = rocke_tensor_descriptor_naive(b, "out", out_lengths, 3, NULL, out_coords, 3); + static const char* const out_coords[3] = {"token", "head", "dim"}; + rocke_value_t* out_dyn[3] + = {rocke_b_mul(b, ctx->num_qh_p, rocke_b_const_i32(b, HD)), NULL, NULL}; + const int out_static[3] = {1, HD, 1}; + ctx->out_desc_red = rocke_tensor_descriptor_naive_dynamic_mixed( + b, "out", out_coords, 3, out_dyn, out_static); } /* ---- base_ml = ml_desc_red.offset(token=q_token, head=q_head, seg=0) ---- @@ -535,7 +544,7 @@ void rocke_gfx950_attention_tiled_3d_reduce_normalize_pass( rocke_if_t guard = rocke_b_scf_if(b, is_writer); rocke_b_region_enter(b, guard.then_region); { - rocke_value_t* nqh = rocke_b_const_i32(b, cfg->NUM_QH); + rocke_value_t* nqh = ctx->num_qh_p; rocke_value_t* row = rocke_b_mul(b, ctx->q_token, nqh); rocke_value_t* lse_index = rocke_b_add(b, row, ctx->q_head); rocke_value_t* has_mass = rocke_lse_row_has_mass(b, ctx->overall_expsum); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_segment_loop_epilogue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_segment_loop_epilogue.cpp index b500627c1806..1e5c3c0be231 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_segment_loop_epilogue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_segment_loop_epilogue.cpp @@ -116,7 +116,7 @@ void rocke_gfx950_attention_tiled_3d_emit_loop_init( rocke_value_t* qh_mul = rocke_b_mul(b, ctx->kv_head_idx, rocke_b_const_i32(b, cfg->NQK)); rocke_value_t* qh_mod = rocke_b_mod(b, ctx->lane_col, rocke_b_const_i32(b, cfg->NQK)); rocke_value_t* qh = rocke_b_add(b, qh_mul, qh_mod); - rocke_value_t* qh_in = rocke_b_cmp_lt(b, qh, rocke_b_const_i32(b, cfg->NUM_QH)); + rocke_value_t* qh_in = rocke_b_cmp_lt(b, qh, ctx->num_qh_p); rocke_value_t* sink_h = rocke_b_global_load(b, ctx->sinks, qh, dtype, 2); rocke_value_t* sink_f = rocke_b_fmul(b, rocke_b_cast_to_f32(b, sink_h), ctx->rcp_ln2); rocke_value_t* sink_with_mask = rocke_b_select(b, qh_in, sink_f, ctx->neg_inf); @@ -248,7 +248,7 @@ void rocke_gfx950_attention_tiled_3d_emit_softmax_loop( qh_mod = rocke_b_mod(b, ctx->lane_col, rocke_b_const_i32(b, cfg->NQK)); ctx->row_qh = rocke_b_add(b, qh_mul, qh_mod); ok_a = rocke_b_cmp_lt(b, ctx->row_qp, ctx->cur_batch_q_len); - ok_b = rocke_b_cmp_lt(b, ctx->row_qh, rocke_b_const_i32(b, cfg->NUM_QH)); + ok_b = rocke_b_cmp_lt(b, ctx->row_qh, ctx->num_qh_p); ctx->row_ok = rocke_b_land(b, ok_a, ok_b); /* Padding rows past BLOCK_Q*NQK alias the next q-block: no store. */ if(cfg->BLOCK_Q * cfg->NQK < cfg->BLOCK_M) @@ -261,8 +261,7 @@ void rocke_gfx950_attention_tiled_3d_emit_softmax_loop( ctx->row_alibi = NULL; if(cfg->USE_ALIBI) { - rocke_value_t* qh_ok - = rocke_b_cmp_lt(b, ctx->row_qh, rocke_b_const_i32(b, cfg->NUM_QH)); + rocke_value_t* qh_ok = rocke_b_cmp_lt(b, ctx->row_qh, ctx->num_qh_p); ctx->row_alibi = rocke_b_masked_global_load( b, ctx->alibi_slopes_ptr, ctx->row_qh, qh_ok, rocke_b_const_f32(b, 0.0), f32, 4); } diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_segment_prologue.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_segment_prologue.cpp index 632dd5449411..691050c530e7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_segment_prologue.cpp +++ b/dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/gfx950/attention_tiled_3d_segment_prologue.cpp @@ -328,6 +328,9 @@ void rocke_gfx950_attention_tiled_3d_declare_params( ctx->num_seqs_p = rocke_b_param(B, "num_seqs", rocke_i32(), NULL); ctx->bt_stride_p = rocke_b_param(B, "block_table_stride", rocke_i32(), NULL); ctx->qq_bias_stride0_p = rocke_b_param(B, "qq_bias_stride_0", rocke_i32(), NULL); + ctx->num_qh_p = rocke_b_param(B, "num_query_heads", rocke_i32(), NULL); + ctx->num_kv_p = rocke_b_param(B, "num_kv_heads", rocke_i32(), NULL); + ctx->sliding_window_p = rocke_b_param(B, "sliding_window", rocke_i32(), NULL); if(ctx->strided_kv) { ctx->k_strides = rocke_declare_strided_kv_params(B, "k"); @@ -344,7 +347,6 @@ void rocke_gfx950_attention_tiled_3d_emit_prologue(rocke_gfx950_attention_tiled_ const int T = CFG.T; const int BLOCK_M = CFG.BLOCK_M; const int BLOCK_Q = CFG.BLOCK_Q; - const int NUM_QH = CFG.NUM_QH; const int NUM_SEG = CFG.NUM_SEG; const rocke_type_t* dtype = CFG.dtype; @@ -363,13 +365,9 @@ void rocke_gfx950_attention_tiled_3d_emit_prologue(rocke_gfx950_attention_tiled_ } else { - ctx->seq_idx = rocke_binary_search_seq_idx(B, - ctx->cu_q, - ctx->q_block_global_idx, - ctx->num_seqs_p, - BLOCK_Q, - CFG.binary_search_iters, - false); + rocke_value_t* trip = rocke_binary_search_seq_idx_trip_count(B, ctx->num_seqs_p); + ctx->seq_idx = rocke_binary_search_seq_idx_dyn( + B, ctx->cu_q, ctx->q_block_global_idx, ctx->num_seqs_p, BLOCK_Q, trip, false); } /* ---- cu_q bounds / per-sequence geometry (lines 373-379) ---- */ @@ -410,23 +408,31 @@ void rocke_gfx950_attention_tiled_3d_emit_prologue(rocke_gfx950_attention_tiled_ ctx->tps = rocke_b_div(B, tps_num, rocke_b_const_i32(B, NUM_SEG * T)); } - /* ---- descriptors (lines 399-413) ---- */ + /* ---- descriptors (lines 399-413) ---- + * The head count is a kernarg, so each token stride is too. */ { - const int ml_lengths[3] = {1 << 30, NUM_QH, NUM_SEG}; - const char* ml_coords[3] = {"token", "head", "seg"}; - ctx->ml_desc - = rocke_tensor_descriptor_naive(B, "segm_ml", ml_lengths, 3, NULL, ml_coords, 3); + static const char* const ml_coords[3] = {"token", "head", "seg"}; + rocke_value_t* ml_dyn[3] + = {rocke_b_mul(B, ctx->num_qh_p, rocke_b_const_i32(B, NUM_SEG)), NULL, NULL}; + const int ml_static[3] = {1, NUM_SEG, 1}; + ctx->ml_desc = rocke_tensor_descriptor_naive_dynamic_mixed( + B, "segm_ml", ml_coords, 3, ml_dyn, ml_static); } { - const int sa_lengths[4] = {1 << 30, NUM_QH, NUM_SEG, HD}; - const char* sa_coords[4] = {"token", "head", "seg", "dim"}; - ctx->seg_acc_desc - = rocke_tensor_descriptor_naive(B, "segm_output", sa_lengths, 4, NULL, sa_coords, 4); + static const char* const sa_coords[4] = {"token", "head", "seg", "dim"}; + rocke_value_t* sa_dyn[4] + = {rocke_b_mul(B, ctx->num_qh_p, rocke_b_const_i32(B, NUM_SEG * HD)), NULL, NULL, NULL}; + const int sa_static[4] = {1, NUM_SEG * HD, HD, 1}; + ctx->seg_acc_desc = rocke_tensor_descriptor_naive_dynamic_mixed( + B, "segm_output", sa_coords, 4, sa_dyn, sa_static); } { - const int q_lengths[3] = {1 << 30, NUM_QH, HD}; - const char* q_coords[3] = {"token", "head", "dim"}; - ctx->q_desc = rocke_tensor_descriptor_naive(B, "Q", q_lengths, 3, NULL, q_coords, 3); + static const char* const q_coords[3] = {"token", "head", "dim"}; + rocke_value_t* q_dyn[3] + = {rocke_b_mul(B, ctx->num_qh_p, rocke_b_const_i32(B, HD)), NULL, NULL}; + const int q_static[3] = {1, HD, 1}; + ctx->q_desc + = rocke_tensor_descriptor_naive_dynamic_mixed(B, "Q", q_coords, 3, q_dyn, q_static); } /* seg_start_tile_pos = seg_idx*tps*T (line 415) */ @@ -466,7 +472,7 @@ void rocke_gfx950_attention_tiled_3d_emit_prologue(rocke_gfx950_attention_tiled_ ctx->one_f = rocke_b_const_f32(B, 1.0); ctx->rcp_ln2 = rocke_b_const_f32(B, 1.4426950408889634); ctx->qk_scale = rocke_b_fmul(B, ctx->scale_p, ctx->rcp_ln2); - ctx->sw_const = rocke_b_const_i32(B, CFG.SLIDING_WINDOW); + ctx->sw_const = ctx->sliding_window_p; ctx->z8 = rocke_b_zero_vec(B, dtype, 8); /* ---- Q -> LDS (lines 482-512) ---- */ @@ -519,7 +525,6 @@ void rocke_gfx950_attention_tiled_3d_emit_async_infra( { const int HD = CFG.HD; const int T = CFG.T; - const int NUM_KV = CFG.NUM_KV; /* ---- async DMA infra (lines 565-585) ---- */ ctx->big_bytes = rocke_b_const_i32(B, 0x7FFF0000); @@ -535,10 +540,12 @@ void rocke_gfx950_attention_tiled_3d_emit_async_infra( ctx->key_rsrc = rocke_b_buffer_rsrc(B, ctx->key, ctx->big_bytes); ctx->value_rsrc = rocke_b_buffer_rsrc(B, ctx->value, ctx->big_bytes); } - /* kv_block_bytes_c = const_i32(kv_stride_blk_b): one-block buffer bound for - * the i64-addressing path (Python creates it unconditionally right after the - * byte strides; unused in the i32 path, where it is DCE'd). */ - ctx->kv_block_bytes_c = rocke_b_const_i32(B, CFG.kv_stride_blk_b); + /* The token and page byte strides scale with the runtime kv head count. + * kv_block_bytes_c is the page stride: the one-block buffer bound for the + * i64-addressing path (unused in the i32 path, where it is DCE'd). */ + ctx->kv_stride_tok_b = rocke_b_mul(B, ctx->num_kv_p, rocke_b_const_i32(B, HD * CFG.KV_BYTES)); + ctx->kv_stride_blk_b = rocke_b_mul(B, ctx->kv_stride_tok_b, rocke_b_const_i32(B, CFG.BS)); + ctx->kv_block_bytes_c = ctx->kv_stride_blk_b; ctx->lane_half_base = rocke_b_mul(B, ctx->tid, rocke_b_const_i32(B, CFG.HALVES_PER_LANE)); ctx->K_lds_addr = rocke_b_smem_addr_of(B, ctx->K_lds); ctx->V_lds_addr = rocke_b_smem_addr_of(B, ctx->V_lds); @@ -546,6 +553,8 @@ void rocke_gfx950_attention_tiled_3d_emit_async_infra( ctx->seq_pages = NULL; memset(ctx->lane_within_b, 0, sizeof(ctx->lane_within_b)); + memset(ctx->call_tok_b, 0, sizeof(ctx->call_tok_b)); + ctx->kv_stride_blk_i64 = NULL; ctx->paged_kv_desc = NULL; /* seq_base = seq_idx * bt_stride_p (line 592) */ if(ctx->strided_kv) @@ -555,15 +564,14 @@ void rocke_gfx950_attention_tiled_3d_emit_async_infra( ctx->seq_base = rocke_b_mul(B, ctx->seq_idx, ctx->bt_stride_p); if(!CFG.PAGE_IDS) { - /* paged_kv_desc = naive("paged_kv_bytes", [1<<24, T, NUM_KV, HD], byte + /* paged_kv_desc = DynamicTensorDescriptor("paged_kv_bytes", byte * strides).transform(indirect("tile_idx" -> "physical_block", * base=seq_base), unmerge("linear_half" -> ("token","dim"), (T, HD))) */ - const int lengths[4] = {1 << 24, T, NUM_KV, HD}; - const int strides[4] - = {CFG.kv_stride_blk_b, CFG.kv_stride_tok_b, CFG.kv_stride_h_b, CFG.KV_BYTES}; - const char* coords[4] = {"physical_block", "token", "kv_head", "dim"}; - rocke_tensor_descriptor_t* base - = rocke_tensor_descriptor_naive(B, "paged_kv_bytes", lengths, 4, strides, coords, 4); + const int strides[4] = {1, 1, CFG.kv_stride_h_b, CFG.KV_BYTES}; + rocke_value_t* dyn[4] = {ctx->kv_stride_blk_b, ctx->kv_stride_tok_b, NULL, NULL}; + static const char* const coords[4] = {"physical_block", "token", "kv_head", "dim"}; + rocke_tensor_descriptor_t* base = rocke_tensor_descriptor_naive_dynamic_mixed( + B, "paged_kv_bytes", coords, 4, dyn, strides); const rocke_transform_t* chain[2]; const char* into_td[2] = {"token", "dim"}; const int dims_td[2] = {T, HD}; @@ -587,7 +595,7 @@ void rocke_gfx950_attention_tiled_3d_emit_async_infra( { rocke_value_t* lane_tok = rocke_b_div(B, ctx->lane_half_base, rocke_b_const_i32(B, HD)); rocke_value_t* lane_dim = rocke_b_mod(B, ctx->lane_half_base, rocke_b_const_i32(B, HD)); - rocke_value_t* tok_b = rocke_b_mul(B, lane_tok, rocke_b_const_i32(B, CFG.kv_stride_tok_b)); + rocke_value_t* tok_b = rocke_b_mul(B, lane_tok, ctx->kv_stride_tok_b); rocke_value_t* head_b = rocke_b_mul(B, ctx->kv_head_idx, rocke_b_const_i32(B, CFG.kv_stride_h_b)); rocke_value_t* tok_head = rocke_b_add(B, tok_b, head_b); @@ -616,6 +624,24 @@ void rocke_gfx950_attention_tiled_3d_emit_async_infra( rocke_value_t* dim_b = rocke_b_mul(B, lane_dim, rocke_b_const_i32(B, CFG.KV_BYTES)); ctx->lane_within_b[0] = rocke_b_add(B, tok_head, dim_b); } + /* The token stride is a kernarg, so each call's token offset within the + * page is a multiply. They are wave-uniform and loop-invariant: build + * them once here rather than in every tile's copies. Calls that cover a + * partial row (head_size 192) compute their own offsets instead. */ + { + int call; + const int n_calls = CFG.KV_HALVES_PER_CALL % CFG.HD == 0 ? CFG.kv_calls_per_tile : 0; + for(call = 0; call < n_calls; ++call) + { + const int tok_i = call % CFG.calls_per_block; + if(tok_i == 0 || ctx->call_tok_b[tok_i] != NULL) + continue; + ctx->call_tok_b[tok_i] = rocke_b_mul( + B, ctx->kv_stride_tok_b, rocke_b_const_i32(B, tok_i * CFG.tokens_per_call)); + } + } + if(CFG.I64_KV_ADDR) + ctx->kv_stride_blk_i64 = rocke_b_zext(B, ctx->kv_stride_blk_b, rocke_i64()); } } @@ -775,7 +801,7 @@ static void rocke__issue_paged_tile(rocke_gfx950_attention_tiled_3d_build_ctx_t* for(call = 0; call < CFG.kv_calls_per_tile; ++call) { const int blk = call / CFG.calls_per_block; - const int tok0 = (call % CFG.calls_per_block) * CFG.tokens_per_call; + const int tok_i = call % CFG.calls_per_block; const int ph_i = CFG.KV_SWZ ? (call % (CFG.KV_CHUNKS_PER_ROW / CFG.tokens_per_call)) : 0; rocke_value_t* voff; if(CFG.KV_HALVES_PER_CALL % CFG.HD != 0) @@ -785,7 +811,7 @@ static void rocke__issue_paged_tile(rocke_gfx950_attention_tiled_3d_build_ctx_t* rocke_value_t* lin = rocke_b_add( B, rocke_b_const_i32(B, call * CFG.KV_HALVES_PER_CALL), ctx->lane_half_base); rocke_value_t* row = rocke_b_div(B, lin, rocke_b_const_i32(B, CFG.HD)); - rocke_value_t* tok_b = rocke_b_mul(B, row, rocke_b_const_i32(B, CFG.kv_stride_tok_b)); + rocke_value_t* tok_b = rocke_b_mul(B, row, ctx->kv_stride_tok_b); rocke_value_t* head_b = rocke_b_mul(B, ctx->kv_head_idx, rocke_b_const_i32(B, CFG.kv_stride_h_b)); rocke_value_t* dim = rocke_b_mod(B, lin, rocke_b_const_i32(B, CFG.HD)); @@ -794,16 +820,16 @@ static void rocke__issue_paged_tile(rocke_gfx950_attention_tiled_3d_build_ctx_t* } else { - voff = rocke_b_add( - B, ctx->lane_within_b[ph_i], rocke_b_const_i32(B, tok0 * CFG.kv_stride_tok_b)); + voff = ctx->lane_within_b[ph_i]; + if(tok_i != 0) + voff = rocke_b_add(B, voff, ctx->call_tok_b[tok_i]); } rocke_value_t* call_rsrc; rocke_value_t* soff; if(CFG.I64_KV_ADDR) { rocke_value_t* pid_i64 = rocke_b_zext(B, page_ids[blk], rocke_i64()); - rocke_value_t* base_i64 - = rocke_b_mul(B, pid_i64, rocke_b_const_i64(B, (int64_t)CFG.kv_stride_blk_b)); + rocke_value_t* base_i64 = rocke_b_mul(B, pid_i64, ctx->kv_stride_blk_i64); call_rsrc = rocke_b_buffer_rsrc( B, rocke_b_global_ptr_add(B, src, base_i64), ctx->kv_block_bytes_c); soff = ctx->zero_soff; @@ -811,7 +837,7 @@ static void rocke__issue_paged_tile(rocke_gfx950_attention_tiled_3d_build_ctx_t* else { call_rsrc = src_rsrc; - soff = rocke_b_mul(B, page_ids[blk], rocke_b_const_i32(B, CFG.kv_stride_blk_b)); + soff = rocke_b_mul(B, page_ids[blk], ctx->kv_stride_blk_b); } rocke_value_t* dst = rocke_b_smem_ptr_add( B, buf_base, rocke_b_const_i64(B, (int64_t)call * CFG.bytes_per_call)); @@ -827,7 +853,6 @@ void rocke_gfx950_attention_tiled_3d_emit_early_zero_fill( rocke_gfx950_attention_tiled_3d_build_ctx_t* ctx) { const int NQK = CFG.NQK; - const int NUM_QH = CFG.NUM_QH; const int PV_N_TILES = CFG.PV_N_TILES; int n, reg; @@ -850,7 +875,7 @@ void rocke_gfx950_attention_tiled_3d_emit_early_zero_fill( rocke_value_t* qh_r_mod = rocke_b_mod(B, row, rocke_b_const_i32(B, NQK)); rocke_value_t* qh_r = rocke_b_add(B, qh_r_mul, qh_r_mod); rocke_value_t* row_ok_a = rocke_b_cmp_lt(B, qp_r, ctx->cur_batch_q_len); - rocke_value_t* row_ok_b = rocke_b_cmp_lt(B, qh_r, rocke_b_const_i32(B, NUM_QH)); + rocke_value_t* row_ok_b = rocke_b_cmp_lt(B, qh_r, ctx->num_qh_p); rocke_value_t* row_ok = rocke_b_land(B, row_ok_a, row_ok_b); /* Padding rows past BLOCK_Q*NQK alias the next q-block: no store. */ if(CFG.BLOCK_Q * NQK < CFG.BLOCK_M) @@ -890,7 +915,7 @@ void rocke_gfx950_attention_tiled_3d_emit_early_zero_fill( rocke_value_t* qh_r_mod = rocke_b_mod(B, row, rocke_b_const_i32(B, NQK)); rocke_value_t* qh_r = rocke_b_add(B, qh_r_mul, qh_r_mod); rocke_value_t* row_ok_a = rocke_b_cmp_lt(B, qp_r, ctx->cur_batch_q_len); - rocke_value_t* row_ok_b = rocke_b_cmp_lt(B, qh_r, rocke_b_const_i32(B, NUM_QH)); + rocke_value_t* row_ok_b = rocke_b_cmp_lt(B, qh_r, ctx->num_qh_p); rocke_value_t* row_ok = rocke_b_land(B, row_ok_a, row_ok_b); /* Padding rows past BLOCK_Q*NQK alias the next q-block: no store. */ if(CFG.BLOCK_Q * NQK < CFG.BLOCK_M) @@ -921,7 +946,6 @@ void rocke_gfx950_attention_tiled_3d_emit_early_zero_fill( void rocke_gfx950_attention_tiled_3d_emit_q_to_lds(rocke_gfx950_attention_tiled_3d_build_ctx_t* ctx) { const int NQK = CFG.NQK; - const int NUM_QH = CFG.NUM_QH; const int THREADS = CFG.THREADS; const int Q_VECS_PER_ROW = CFG.Q_VECS_PER_ROW; const int Q_VECS_PER_THREAD = CFG.Q_VECS_PER_THREAD; @@ -943,7 +967,7 @@ void rocke_gfx950_attention_tiled_3d_emit_q_to_lds(rocke_gfx950_attention_tiled_ rocke_value_t* qh_t_mod = rocke_b_mod(B, Q_row, rocke_b_const_i32(B, NQK)); rocke_value_t* qh_t = rocke_b_add(B, qh_t_mul, qh_t_mod); rocke_value_t* qmask_a = rocke_b_cmp_lt(B, q_pos_t, ctx->cur_batch_q_len); - rocke_value_t* qmask_b = rocke_b_cmp_lt(B, qh_t, rocke_b_const_i32(B, NUM_QH)); + rocke_value_t* qmask_b = rocke_b_cmp_lt(B, qh_t, ctx->num_qh_p); rocke_value_t* qmask_t = rocke_b_land(B, qmask_a, qmask_b); rocke_value_t* q_pos_safe = rocke_b_select(B, qmask_t, q_pos_t, rocke_b_const_i32(B, 0)); rocke_value_t* qh_safe = rocke_b_select(B, qmask_t, qh_t, rocke_b_const_i32(B, 0)); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/file_index.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/file_index.md index 9dd67c23d790..b0ffbfdfa224 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/file_index.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/file_index.md @@ -55,7 +55,7 @@ A by-file map of the `rocke` package. Symbols listed are the primary contents (p | `helpers/schedule.py` | `SchedulePolicy`. `sched_group_barrier`, `s_setprio`, named schedule flavors. | | `helpers/pipeline.py` | `SoftwarePipeline.run_ping_pong(...)`. | | `helpers/epilogues.py` | `DirectEpilogue`, `CShuffleEpilogue`. | -| `helpers/attention.py` | `Attention2DConfig`, `Attention3DConfig`, `OnlineSoftmaxState`, `PagedKvDescriptor`, `apply_softcap_log2`, `apply_softcap_scalar`, `binary_search_seq_idx`, `binary_search_seq_idx_iters`, `causal_mask`, `sliding_window_mask`, `mfma_16x16x16_for_dtype`, `mfma_16x16x32_for_dtype`, `select_2d_config`, `select_3d_config`, `use_2d_kernel`, `warp_xor_reduce_max`, `warp_xor_reduce_sum`. | +| `helpers/attention.py` | `Attention2DConfig`, `Attention3DConfig`, `OnlineSoftmaxState`, `PagedKvDescriptor`, `apply_softcap_log2`, `apply_softcap_scalar`, `binary_search_seq_idx`, `binary_search_seq_idx_iters`, `binary_search_seq_idx_trip_count`, `causal_mask`, `sliding_window_mask`, `mfma_16x16x16_for_dtype`, `mfma_16x16x32_for_dtype`, `select_2d_config`, `select_3d_config`, `use_2d_kernel`, `warp_xor_reduce_max`, `warp_xor_reduce_sum`. | | `helpers/compile.py` | `compile_kernel`, `KernelArtifact`. | | `helpers/manifest.py` | `MANIFEST_SCHEMA = "ck.dsl.example.manifest/v1"`, `attention_args_signature`, `gemm_args_signature`, `make_attention_manifest`, `make_conv_manifest`, `make_gemm_manifest`, `make_simple_op_manifest`, `write_artifact`. | | `helpers/tensor_view.py` | `TensorDescriptor`, `TensorView`, `TileWindow`, `TensorCoordinate`, `BufferResource`; constructors `make_global_view`, `make_lds_view`, `make_buffer_resource`, `make_buffer_view`, `make_naive_tensor_descriptor_packed`, `make_naive_tensor_view_packed`, `make_tile_window`, `make_tensor_coordinate`, `move_tensor_coordinate`, `view_from_transforms_descriptor`. | diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/__init__.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/__init__.py index a9e24bee5e9f..dc4523422258 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/__init__.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/__init__.py @@ -111,6 +111,7 @@ apply_softcap_scalar, binary_search_seq_idx, binary_search_seq_idx_iters, + binary_search_seq_idx_trip_count, apply_attention_mask, causal_mask, mfma_16x16x16_for_dtype, @@ -442,6 +443,7 @@ "BINARY_SEARCH_FALLBACK_ITERS", "binary_search_seq_idx", "binary_search_seq_idx_iters", + "binary_search_seq_idx_trip_count", "apply_attention_mask", "causal_mask", # MFMA GEMM inner helper diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/attention.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/attention.py index a882b6c9b120..77b2c8e1be90 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/attention.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/attention.py @@ -718,6 +718,19 @@ def binary_search_seq_idx_iters(num_seqs: int, *, floor: int = 1) -> int: return max(floor, int(math.ceil(math.log2(num_seqs + 1)))) +def binary_search_seq_idx_trip_count(b: IRBuilder, num_seqs: Value) -> Value: + """Runtime trip count for :func:`binary_search_seq_idx`: ``bit_length(num_seqs)``, + at least 1, read off the exponent of ``float(num_seqs)``. + + Equals :func:`binary_search_seq_idx_iters` below 2**24 sequences. Past that the + conversion may round up a power of two and add one iteration, which the + search's done guard turns into a no-op. Lets one kernel serve every batch size. + """ + bits = b.bitcast(b.sitofp_f32(num_seqs), I32) + exponent = b.lshr(bits, b.const_i32(23)) + return b.smax(b.sub(exponent, b.const_i32(126)), b.const_i32(1)) + + def binary_search_seq_idx( b: IRBuilder, cu_q: Value, @@ -725,7 +738,7 @@ def binary_search_seq_idx( num_seqs: Value, *, block_q: int, - iterations: int, + iterations: int | Value, per_token: bool = False, ) -> Value: """Triton-style binary search for the seq_idx for this q_block. @@ -735,7 +748,8 @@ def binary_search_seq_idx( invariant is ``cu_q[i] // BLOCK_Q + i <= target`` (i.e. the cumulative Q-block count up to sequence ``i``). The caller specializes ``iterations`` from the known problem batch size with - :func:`binary_search_seq_idx_iters`. + :func:`binary_search_seq_idx_iters`, or passes the runtime bound from + :func:`binary_search_seq_idx_trip_count`. P49: ``per_token=True`` switches to per-token mode (``cu_q[s] <= q_token < cu_q[s+1]``) used by ``fmha_paged_prefill``, @@ -748,7 +762,7 @@ def binary_search_seq_idx( bq = b.const_i32(block_q) loop = b.scf_for_iter( b.const_i32(0), - b.const_i32(iterations), + iterations if isinstance(iterations, Value) else b.const_i32(iterations), b.const_i32(1), [("left", b.const_i32(0)), ("right", num_seqs)], iv_name="bs_i", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/transforms.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/transforms.py index bf4d34487075..95589cc8406f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/transforms.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/helpers/transforms.py @@ -69,8 +69,9 @@ from __future__ import annotations +import operator from dataclasses import dataclass, replace -from typing import Any, Dict, List, Optional, Sequence, Tuple +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union from ..core.ir import F16, I32, I64, IRBuilder, Type, Value from . import magic_division @@ -1542,6 +1543,18 @@ def _run_chain( remaining = next_remaining return coords + def _term(self, b: IRBuilder, i: int, value: Value) -> Value: + """Base coord ``i``'s i32 offset term: ``value`` times its stride, with no + multiply for a stride of 1. :class:`DynamicTensorDescriptor` overrides + the stride source; every offset/move loop goes through this hook.""" + stride = self.base_strides[i] + return value if stride == 1 else b.mul(value, b.const_i32(stride)) + + def _term_i64(self, b: IRBuilder, i: int, wide: Value) -> Value: + """Base coord ``i``'s i64 term: the already-widened ``wide`` times its + stride (always multiplied, as the i64 paths do).""" + return b.mul(wide, b.const_i64(int(self.base_strides[i]))) + def offset( self, b: IRBuilder, @@ -1557,17 +1570,14 @@ def offset( coords = self._run_chain(b, upper_values) offset: Optional[Value] = None valid: Optional[Value] = None - for name, stride in zip(self.base_names, self.base_strides): + for i, name in enumerate(self.base_names): if name not in coords: raise ValueError( f"after chain, base coord {name!r} not in {sorted(coords.keys())}" ) c = coords[name] valid = _and(b, valid, c.valid) - if stride == 1: - term = c.value - else: - term = b.mul(c.value, b.const_i32(stride)) + term = self._term(b, i, c.value) offset = term if offset is None else b.add(offset, term) if offset is None: offset = b.const_i32(0) @@ -1592,7 +1602,7 @@ def offset_i64_split( base_i64: Optional[Value] = None within: Optional[Value] = None valid: Optional[Value] = None - for name, stride in zip(self.base_names, self.base_strides): + for i, name in enumerate(self.base_names): if name not in coords: raise ValueError( f"after chain, base coord {name!r} not in {sorted(coords.keys())}" @@ -1604,9 +1614,9 @@ def offset_i64_split( # descriptor (uniform per wave), so pin the (wave-uniform) # block id to an SGPR before widening. base_val = b.to_sgpr_u32(c.value) - base_i64 = b.mul(b.zext(base_val, I64), b.const_i64(int(stride))) + base_i64 = self._term_i64(b, i, b.zext(base_val, I64)) else: - term = c.value if stride == 1 else b.mul(c.value, b.const_i32(stride)) + term = self._term(b, i, c.value) within = term if within is None else b.add(within, term) if base_i64 is None: raise ValueError( @@ -1634,7 +1644,7 @@ def offset_i64( coords = self._run_chain(b, upper_values) off: Optional[Value] = None valid: Optional[Value] = None - for name, stride in zip(self.base_names, self.base_strides): + for i, name in enumerate(self.base_names): if name not in coords: raise ValueError( f"after chain, base coord {name!r} not in {sorted(coords.keys())}" @@ -1642,10 +1652,9 @@ def offset_i64( c = coords[name] valid = _and(b, valid, c.valid) if name == i64_coord: - term = b.mul(b.zext(c.value, I64), b.const_i64(int(stride))) + term = self._term_i64(b, i, b.zext(c.value, I64)) else: - t32 = c.value if stride == 1 else b.mul(c.value, b.const_i32(stride)) - term = b.zext(t32, I64) + term = b.zext(self._term(b, i, c.value), I64) off = term if off is None else b.add(off, term) if off is None: off = b.const_i64(0) @@ -1736,14 +1745,13 @@ def move( delta, new = self._run_chain_delta(b, old_values, delta_values) off_delta: Optional[Value] = None valid: Optional[Value] = None - for name, stride in zip(self.base_names, self.base_strides): + for i, name in enumerate(self.base_names): if name not in delta: raise ValueError( f"after chain, base coord {name!r} not in {sorted(delta.keys())}" ) valid = _and(b, valid, new[name].valid) - dv = delta[name].value - term = dv if stride == 1 else b.mul(dv, b.const_i32(stride)) + term = self._term(b, i, delta[name].value) off_delta = term if off_delta is None else b.add(off_delta, term) if off_delta is None: return prev_offset, valid @@ -1752,40 +1760,31 @@ def move( @dataclass class DynamicTensorDescriptor(TensorDescriptor): - """A :class:`TensorDescriptor` whose base strides are runtime SSA Values. + """A :class:`TensorDescriptor` whose base strides may be runtime SSA Values. Constructed via :meth:`DynamicTensorDescriptor.create`. The transform - chain and ``offset()`` work identically to the static descriptor except - the final stride-multiply in ``offset()`` uses runtime Values instead of - ``b.const_i32(stride)``. + chain works identically to the static descriptor; the final stride + reduction multiplies a base coord by its runtime ``Value`` stride, and an + ``int`` stride follows the static rule (no multiply for 1). - ``dynamic_strides`` holds one ``Value`` per base coord, in the same - order as ``base_names``. ``base_strides`` is set to all-ones (unused - sentinel) so the parent's ``offset()`` is not called directly. + ``dynamic_strides`` holds one entry per base coord, in the same order as + ``base_names``: the runtime ``Value``, or ``None`` where ``base_strides`` + holds the compile-time stride (1 at every runtime position). """ - dynamic_strides: Tuple[Value, ...] = () + dynamic_strides: Tuple[Optional[Value], ...] = () - def offset( - self, - b: IRBuilder, - **upper_values: Value, - ) -> Tuple[Value, Optional[Value]]: - coords = self._run_chain(b, upper_values) - off: Optional[Value] = None - valid: Optional[Value] = None - for name, stride_v in zip(self.base_names, self.dynamic_strides): - if name not in coords: - raise ValueError( - f"after chain, base coord {name!r} not in {sorted(coords.keys())}" - ) - c = coords[name] - valid = _and(b, valid, c.valid) - term = b.mul(c.value, stride_v) - off = term if off is None else b.add(off, term) - if off is None: - off = b.const_i32(0) - return off, valid + def _term(self, b: IRBuilder, i: int, value: Value) -> Value: + stride_v = self.dynamic_strides[i] + if stride_v is not None: + return b.mul(value, stride_v) + return super()._term(b, i, value) + + def _term_i64(self, b: IRBuilder, i: int, wide: Value) -> Value: + stride_v = self.dynamic_strides[i] + if stride_v is not None: + return b.mul(wide, b.zext(stride_v, I64)) + return super()._term_i64(b, i, wide) @classmethod def create( @@ -1793,30 +1792,39 @@ def create( name: str, *, coord_names: Sequence[str], - strides: Sequence[Value], + strides: Sequence[Union[Value, int]], ) -> "DynamicTensorDescriptor": - """Build a dynamic descriptor from runtime SSA strides. - - ``coord_names`` and ``strides`` must have the same length. - ``strides`` are i32 SSA Values representing the row-major strides - (in elements) for each coordinate, computed host-side and passed - as kernel arguments. + """Build a descriptor from per-coord strides, each a runtime i32 SSA + ``Value`` (in elements, usually from kernel arguments) or an ``int``. - The descriptor can then be extended with ``.transform(...)`` just - like a static descriptor. + ``coord_names`` and ``strides`` must have the same length. The + descriptor can then be extended with ``.transform(...)`` just like a + static descriptor. """ if len(coord_names) != len(strides): raise ValueError("coord_names and strides length mismatch") coord_names = tuple(coord_names) - strides = tuple(strides) + dynamic = tuple(s if isinstance(s, Value) else None for s in strides) + static = [] + for i, s in enumerate(strides): + if isinstance(s, Value): + static.append(1) + continue + # operator.index rejects a float or string stride instead of truncating it. + try: + static.append(operator.index(s)) + except TypeError: + raise TypeError( + f"strides[{i}] must be a Value or an int, got {type(s).__name__}" + ) from None return cls( name=name, base_names=coord_names, base_lengths=tuple(0 for _ in coord_names), - base_strides=tuple(1 for _ in coord_names), + base_strides=tuple(static), chain=(), upper_names=coord_names, - dynamic_strides=strides, + dynamic_strides=dynamic, ) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/cmake/InstanceTests.cmake b/dnn-providers/hip-kernel-provider/rocke/platform/tests/cmake/InstanceTests.cmake index b1278a892ef4..47210d22e3f5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/cmake/InstanceTests.cmake +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/cmake/InstanceTests.cmake @@ -24,6 +24,28 @@ if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/instances/tiled_attention_2d_reentrancy.c rocke_tiled_attention_2d_reentrancy) endif() +# The C tiled-attention entries refuse bad head counts, and fastKV register-P +# on gfx942, with the Python reason text (host only, no GPU). +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/instances/tiled_attention_entry_guards.cpp") + add_executable(rocke_tiled_attention_entry_guards + instances/tiled_attention_entry_guards.cpp) + target_link_libraries(rocke_tiled_attention_entry_guards PRIVATE rocke_core) + if(UNIX) + target_link_libraries(rocke_tiled_attention_entry_guards PRIVATE m) + endif() + add_test(NAME rocke_tiled_attention_entry_guards + COMMAND rocke_tiled_attention_entry_guards) + set_tests_properties(rocke_tiled_attention_entry_guards + PROPERTIES LABELS "ckc;unit;host") + rocke_cxx_test_installed_name(_rocke_installed_name rocke_tiled_attention_entry_guards) + set_target_properties(rocke_tiled_attention_entry_guards + PROPERTIES OUTPUT_NAME "${_rocke_installed_name}") + install(TARGETS rocke_tiled_attention_entry_guards + RUNTIME DESTINATION ${ROCKE_CXX_TEST_INSTALL_DIR}) + set_property(GLOBAL APPEND PROPERTY ROCKE_INSTALL_TEST_TARGETS + rocke_tiled_attention_entry_guards) +endif() + # The three scalar-attention _lower_to_llvm wrappers take no target, so they # lower for a fixed one while the lowerer itself rejects a NULL arch. Nothing # else calls them, so this is the only thing that notices when they stop diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_transforms_dynamic.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_transforms_dynamic.py index d0284324d7e7..5b90ba8edfe8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_transforms_dynamic.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/core/test_transforms_dynamic.py @@ -505,6 +505,18 @@ def test_no_bounds_no_predicate(self): # --------------------------------------------------------------------------- +def _depends_on(value, source) -> bool: + """True when ``source`` feeds ``value`` through some chain of ops.""" + stack = [value] + while stack: + v = stack.pop() + if v is source: + return True + if getattr(v, "op", None) is not None: + stack.extend(v.op.operands) + return False + + class TestDynamicTensorDescriptor: def test_offset_multiplies_by_runtime_strides(self): b = _builder() @@ -605,6 +617,15 @@ def test_missing_chain_input_rejected(self): with pytest.raises(ValueError, match="missing upper coords"): desc.offset(b, x=x) + @pytest.mark.parametrize("bad", [3.75, "3"]) + def test_create_rejects_a_non_integer_stride(self, bad): + b = _builder() + (s0,) = _params(b, "s0") + with pytest.raises(TypeError, match=r"strides\[1\]"): + DynamicTensorDescriptor.create( + "t", coord_names=("a", "b"), strides=(s0, bad) + ) + def test_create_length_mismatch_rejected(self): b = _builder() (s0,) = _params(b, "s0") @@ -618,3 +639,69 @@ def test_strides_are_values(self): "t", coord_names=("a", "b"), strides=(s0, s1) ) assert all(isinstance(s, Value) for s in desc.dynamic_strides) + + def test_int_strides_follow_the_static_rule(self): + b = _builder() + a, c, d, s = _params(b, "a", "c", "d", "s") + desc = DynamicTensorDescriptor.create( + "t", coord_names=("a", "c", "d"), strides=(s, 3, 1) + ) + assert desc.dynamic_strides == (s, None, None) + start = len(b.kernel.body.ops) + off, valid = desc.offset(b, a=a, c=c, d=d) + ops = _new_ops(b, start) + muls = [op for op in ops if op.name == "arith.mul"] + # The Value stride and int stride 3 multiply; int stride 1 does not. + assert len(muls) == 2 + assert muls[0].operands == [a, s] + assert muls[1].operands[0] is c + assert muls[1].operands[1].op.attrs["value"] == 3 + env = {id(a): 5, id(c): 7, id(d): 11, id(s): 13} + _evaluate(ops, env) + assert env[id(off)] == 5 * 13 + 7 * 3 + 11 + assert valid is None + + def test_offset_i64_split_widens_a_runtime_base_stride(self): + b = _builder() + blk, tok, d, s_blk, s_tok = _params(b, "blk", "tok", "d", "s_blk", "s_tok") + desc = DynamicTensorDescriptor.create( + "kv", coord_names=("blk", "tok", "d"), strides=(s_blk, s_tok, 2) + ) + base, within, _ = desc.offset_i64_split(b, "blk", blk=blk, tok=tok, d=d) + # base = zext(to_sgpr(blk)) * zext(s_blk), in i64. + wide_blk, wide_stride = base.op.operands + assert wide_stride.op.operands == [s_blk] + assert _depends_on(wide_blk, blk) + # within = tok * s_tok + d * 2, in i32. + tok_term, d_term = within.op.operands + assert tok_term.op.operands == [tok, s_tok] + assert d_term.op.operands[0] is d + assert d_term.op.operands[1].op.attrs["value"] == 2 + + def test_offset_i64_widens_a_runtime_stride(self): + b = _builder() + blk, d, s_blk = _params(b, "blk", "d", "s_blk") + desc = DynamicTensorDescriptor.create( + "kv", coord_names=("blk", "d"), strides=(s_blk, 1) + ) + off, _ = desc.offset_i64(b, "blk", blk=blk, d=d) + blk_term, d_term = off.op.operands + assert blk_term.op.operands[0].op.operands == [blk] + assert blk_term.op.operands[1].op.operands == [s_blk] + # The int stride-1 term is the coord itself, widened. + assert d_term.op.operands == [d] + + def test_move_multiplies_deltas_by_runtime_strides(self): + b = _builder() + a, c, da, dc, s = _params(b, "a", "c", "da", "dc", "s") + desc = DynamicTensorDescriptor.create( + "t", coord_names=("a", "c"), strides=(s, 3) + ) + start = len(b.kernel.body.ops) + prev, _ = desc.offset(b, a=a, c=c) + off, valid = desc.move(b, prev, {"a": a, "c": c}, {"a": da, "c": dc}) + assert valid is None + env = {id(a): 5, id(c): 7, id(da): 2, id(dc): 4, id(s): 13} + _evaluate(_new_ops(b, start), env) + # The runtime stride scales the delta, not the 1 held in base_strides. + assert env[id(off)] == (5 + 2) * 13 + (7 + 4) * 3 diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256/attention.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256/attention.json index b51eb99d2312..8c552e956736 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256/attention.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256/attention.json @@ -5,306 +5,306 @@ "cases": { "attention/gfx942/2d_bf16_d128_b16_decode_i64": { "arch": "gfx942", - "bytes": 464846, + "bytes": 466413, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b16_t64_h32kv8_bf16_wpe2_regpv", - "sha256": "2f88ce53cd77456946d05366db2fcf99abe21659a066a77fa6a9280ca5689b77", + "kernel_name": "rocke_uattn2d_tiled_d128_b16_t64_g4_bf16_i64_wpe2_regpv", + "sha256": "749e1d93638da6ab37bb118e288b67dd31b58aabdf2dffdaa2b154cc78ebab9e", "status": "ok" }, "attention/gfx942/2d_bf16_d128_b64": { "arch": "gfx942", - "bytes": 394746, + "bytes": 396253, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16", - "sha256": "370007a4987b8659cc375563ac01f8980f2d8d5d5899e796ae34af086846efa5", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16", + "sha256": "9587ff1ec02c46045ef6a07575c3ec7fe761c230484dbec9b0dc66ba1ee0296f", "status": "ok" }, "attention/gfx942/2d_fp16_d128_b64": { "arch": "gfx942", - "bytes": 383076, + "bytes": 384583, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16", - "sha256": "555c9ac36b8a1ba6153446281cc29eb180e8c635503d933fd4facb6dd2be7004", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16", + "sha256": "034ae62e0aac3c46e917a6846205287c3f0897f6fd2f20036fa8568f58556083", "status": "ok" }, "attention/gfx942/2d_lse_mfma32_bf16_d128": { "arch": "gfx942", - "bytes": 529371, + "bytes": 530921, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32x8_lse", - "sha256": "76789686526efcedd42778e95ce53c8b8607447397d1e0306c3243079f35bc26", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32x8_lse", + "sha256": "5589e93bb87e307ad96de0e99520e5e7bbe6552ec3a02e6d7aa3de193dc29fb7", "status": "ok" }, "attention/gfx942/2d_lse_striped16_bf16_d128": { "arch": "gfx942", - "bytes": 400480, + "bytes": 402107, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_lse", - "sha256": "01c9c68efc860179b91f1d66a95545ed72fe003b2cd391030935d2b5e11c493a", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_lse", + "sha256": "d582f3f33f67afba12a36e312e57568f2b3432fa554584ad2484d4ec91e25694", "status": "ok" }, "attention/gfx942/2d_lse_transposed32_bf16_d128": { "arch": "gfx942", - "bytes": 488248, + "bytes": 489899, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32x8_stqk_lse", - "sha256": "f4dfde43b327a95cdcc569ef1dac4f72a9cf978ebf23ad3641f64ee265dfb6a5", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32x8_stqk_lse", + "sha256": "4911beddd44ceb1625e2d9496477b1523869506a14b54b64d16b51a13bd54be0", "status": "ok" }, "attention/gfx942/3d_bf16_d128_b64": { "arch": "gfx942", - "bytes": 355976, + "bytes": 358681, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_h4kv2_seg4_bf16", - "sha256": "e4128a6e834957e3eaefa368f1409d69125dfca44ffa4633a684ab378f0f5bb2", + "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_g2_seg4_bf16", + "sha256": "9c14d2a299f5bf54629246deed221e8aec0198bfb80804c8ca21fce67f63d5c4", "status": "ok" }, "attention/gfx942/3d_fp16_d128_b64": { "arch": "gfx942", - "bytes": 343932, + "bytes": 346637, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_h4kv2_seg4_fp16", - "sha256": "02a8532f212341b8ac080bf4f838859e782d18423c6edb799d5048157970414c", + "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_g2_seg4_fp16", + "sha256": "8891d9a78aad2540771b5b1b3253683be81401088ce9f6ca7b7e01e5840013fd", "status": "ok" }, "attention/gfx942/reduce_bf16_d128": { "arch": "gfx942", - "bytes": 9617, + "bytes": 9773, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_bf16", - "sha256": "5ecc6f410ad0e6b674e4ea99868463538a75fe5c7c81e294f3db697e6f134eba", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_bf16", + "sha256": "f978e1889a9b57c2c544a31e393a75ffd729837a967d44c4dd6361582459870e", "status": "ok" }, "attention/gfx942/reduce_bf16_d128_own_module": { "arch": "gfx942", - "bytes": 9617, + "bytes": 9773, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_bf16", - "sha256": "5ecc6f410ad0e6b674e4ea99868463538a75fe5c7c81e294f3db697e6f134eba", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_bf16", + "sha256": "f978e1889a9b57c2c544a31e393a75ffd729837a967d44c4dd6361582459870e", "status": "ok" }, "attention/gfx942/reduce_fp16_d128": { "arch": "gfx942", - "bytes": 9605, + "bytes": 9761, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_fp16", - "sha256": "9793be461b2096fe71e56defe36858dd58257d0f8243d9dece3e53f4d6123207", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_fp16", + "sha256": "afa3c4ede02acabade859cd09cc57a94fc27a7c6e893976954489f68dd932a28", "status": "ok" }, "attention/gfx942/reduce_lse_bf16_d128": { "arch": "gfx942", - "bytes": 10583, + "bytes": 10754, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_bf16_lse", - "sha256": "eeb8ba17bd6b9f664e9a02113b53bb913d9b8fdb8d59179df0a64166ff79bd18", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_bf16_lse", + "sha256": "f488eb1aa9d6f799194600142fe095bdd57fd75e40c2999fa23325bf21cd4ecc", "status": "ok" }, "attention/gfx950/2d_bf16_d128_b64": { "arch": "gfx950", - "bytes": 168283, + "bytes": 140771, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16", - "sha256": "7dd8456e579e54c8d40d1eeee84d07e44df7d677bdd4a50ec87a2673ee060487", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16", + "sha256": "802f5613a30d409b2c3490f37a788c009eddd6c6652ea25cb3f57cebea5a8f8c", "status": "ok" }, "attention/gfx950/2d_bf16_d64_b32_gqa8_sinks_combo": { "arch": "gfx950", - "bytes": 116996, + "bytes": 117791, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_bf16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim_fastkvdesc", - "sha256": "f3944a91e9f9593a8ffcc91ed2f2212eafa3941211ba267114f392106ff6a269", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_h64kv8_bf16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim_fastkvdesc", + "sha256": "40192b8a73a7474443fcaf49d937a2009fdf45fb2cbcab597c7238992e14ba25", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64": { "arch": "gfx950", - "bytes": 166706, + "bytes": 139194, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16", - "sha256": "557bc508e1e359369b2984a95d7ca89213270bf8506ccbc5fcd3d12b77a09358", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16", + "sha256": "11f96787c1f380f1f2e90b45df3228940eb02f4bf5fd406b7fc5f7c3f1bfd44d", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_alibi": { "arch": "gfx950", - "bytes": 171881, + "bytes": 144424, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_alibi", - "sha256": "f412b87ccb96abb3946f6e35f524c4b47564a3cfcdfe9b67537098cd56afdf62", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_alibi", + "sha256": "4f23312907a9829bbb86bcc48f298bef35af73574375805a0353c96602907886", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_left_only_sw64": { "arch": "gfx950", - "bytes": 168175, + "bytes": 140751, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_sw64_tl_rbu", - "sha256": "a577c1d43bf33ace3aa30bfa9d60c5408a7c1cfdabdc25b7175396d98b5de5b9", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_sw_tl_rbu", + "sha256": "3039635b72d32b6e075ed06db9e8556b43b9d1d63db91e5e8d8a6e083613ec2a", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_no_mask": { "arch": "gfx950", - "bytes": 165338, + "bytes": 137822, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_rbu", - "sha256": "91ba6bc50e7274ea217fc6b55be17e747df4d7bd52d9f6dcb8a95c94efbb1e03", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_rbu", + "sha256": "4fa2281badb6157c8ec7ad734faae2dd7ac35d5efc4a729dc04ae43c8aec9daa", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_rb16": { "arch": "gfx950", - "bytes": 167485, + "bytes": 140004, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_rb16", - "sha256": "a48169e65689de983de22447a923776cb3543613e21b24095d42ad811aa539b3", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_rb16", + "sha256": "4b1e5ed42b11c77e6f98f2224889c920290e5c8f57057a696313de43dbb71ffa", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_softcap": { "arch": "gfx950", - "bytes": 174738, + "bytes": 147190, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_softcap", - "sha256": "05c851c52bf38a36ea4bdb8bfedd189ff1ca603e51d0bd4a43aa68b149c1e200", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_softcap", + "sha256": "5e00a5a538f82cf394f013d855fb5daf8bc65f403f616a315c4f6fa75fd2ebf4", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_top_left": { "arch": "gfx950", - "bytes": 166746, + "bytes": 139234, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_tl", - "sha256": "9c0387e58fd3bbea8e94cf7e2b54be03dd4b60a54b0d8347e23ab1dd55d5298f", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_tl", + "sha256": "4ea8ac780b59d9454d4d6a43fa6cd2fe60537066ecae9260f16349365b93fc05", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_top_left_sw64": { "arch": "gfx950", - "bytes": 169536, + "bytes": 142098, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_sw64_tl", - "sha256": "badbbc59a834d0dda208861440b8693d5df02442949c03e17fc09e06521c8ece", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_sw_tl", + "sha256": "0f969967ef9ad7d13d2d66ab5e880a426c0c29adf1f6f43c0c06864066e4ab72", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_two_sided_sw64_rb16": { "arch": "gfx950", - "bytes": 170353, + "bytes": 142937, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_sw64_tl_rb16", - "sha256": "158ebe15410fbc2f2af509e09c3785b79e1ff9cf2aa743bcf0e245443d6d4bc7", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_sw_tl_rb16", + "sha256": "5db294eab5599422dcabe50dd0256367613b7616fe262ed01503f6beef08dcd3", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_combo_no_mask": { "arch": "gfx950", - "bytes": 115747, + "bytes": 115143, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_rbu_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "5285a546ecdcfb5d0099402fa1e68291970d2fdaf847cf1a33594150db103439", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_rbu_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "c5db679abecb9546a77cb3b294a54d18d3561157c754ffd90cb835e5b2af4a2c", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_combo_rb16": { "arch": "gfx950", - "bytes": 116083, + "bytes": 115496, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_rb16_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "2ae870996e8a1e31da1a4ede84e055111ecb20f21d6c1397e7c2bab7cd9c327d", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_rb16_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "06f10b291046a6f6079d3e073bc12d0f122df7a0b911def9f63b8318cf177d3c", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_combo_top_left": { "arch": "gfx950", - "bytes": 115875, + "bytes": 115274, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_tl_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "c14c34d57dfbe739143a61578908d69372c33698922c50539d00f0781c31e0f1", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_tl_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "4a2317fa3664c45fef4cb139214b4c8c0a9289c85e8882eb07f9495e415cc928", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_sinks_combo": { "arch": "gfx950", - "bytes": 116554, + "bytes": 115995, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "bb09632c2225d223d2daec4f586e9fa9a662502b11d1cb8bfb31a77b83a11e29", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "d5845a417190c278ec214528e56ad209ba7f685a4cce554469082087d9b96526", "status": "ok" }, "attention/gfx950/2d_lse_mfma32_bf16_d128": { "arch": "gfx950", - "bytes": 314684, + "bytes": 287645, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32_lse", - "sha256": "87f12b31db45cc10dc85df48281501e10721f500259b6a55b5dda86ab6eac9eb", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32_lse", + "sha256": "deb63f352fa8ec8fe2728723e688a24a93cd0d13fad2e9be331dfcec33124435", "status": "ok" }, "attention/gfx950/2d_lse_striped16_bf16_d128": { "arch": "gfx950", - "bytes": 172253, + "bytes": 144801, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_lse", - "sha256": "f02250f16da02526125c82a9ba982b2e4c59a4458fc70c85c458f60ae3ae3266", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_lse", + "sha256": "397dd4a2303940c05f3d61b53b0e40185eebb2f9b1ca6e9e902520acbf4986e1", "status": "ok" }, "attention/gfx950/2d_lse_transposed32_bf16_d128": { "arch": "gfx950", - "bytes": 352854, + "bytes": 326192, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32_stqk_s1_lse", - "sha256": "332dc60ad88293938e57dc4194c2483c21e12721603ac37be74d816eabf7e64e", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32_stqk_s1_lse", + "sha256": "38b1d7f491d94811aaa3896b9f9af52514ca18b0ea5f0e44907bfeb9f8eaaf21", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64": { "arch": "gfx950", - "bytes": 161417, + "bytes": 163351, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16", - "sha256": "68dcecb4f369b8bcab80ed8cc7d7c664fbd12ca2dd4c8a21b1f1d2533dd0f52b", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16", + "sha256": "ecf4ac152e550ad5aaeeb6ac76bf69c7846fea33e71e18a1ffdf98e790fa9c01", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64_no_mask": { "arch": "gfx950", - "bytes": 159936, + "bytes": 161870, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16_rbu", - "sha256": "29722169fae8ba2f9cf874a8aebd5eef91bd5323e23a6d63f8be393c9d16f8b3", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16_rbu", + "sha256": "ae54fba61b58bfac4ba0b3785085990844912143a73aa4c2a47d1472c17d8126", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64_rb16": { "arch": "gfx950", - "bytes": 162096, + "bytes": 164030, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16_rb16", - "sha256": "60c24d2b8e9a7c62c626e9b6e253dd618ee942d0ee612cf34e162a43f2200421", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16_rb16", + "sha256": "6f44b77f8b70a7dce8bd1169f1c955080a8de140c34cb7e5d34659e57e36d1a0", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64_top_left": { "arch": "gfx950", - "bytes": 161412, + "bytes": 163346, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16_tl", - "sha256": "55f1c89711ab6956d3c1730aa0f680af09f02f72f114546d37ae4cefec583911", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16_tl", + "sha256": "a22800436a8303278d75a142e84c6ab26e60d436e24a0494f5dfc2d4f56249d4", "status": "ok" }, "attention/gfx950/3d_fp16_d128_b64": { "arch": "gfx950", - "bytes": 159456, + "bytes": 161390, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_fp16", - "sha256": "a9de23680e469af7b7c240b5ceba497e9ab4a1d096ac522bb3de5f03b8debd02", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_fp16", + "sha256": "a72ec1595bf9f2688997ed5dba8b2aef5ad42c4f9f5e7e5ccc8f9eaae50904f8", "status": "ok" }, "attention/gfx950/reduce_bf16_d128": { "arch": "gfx950", - "bytes": 9582, + "bytes": 9738, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_d128_h4_seg4_bf16", - "sha256": "ffd2175efd452e81fae6d9a0e673054b8a983b39655e2799cc62c45ea00f500c", + "kernel_name": "rocke_uattn_reduce_tiled_d128_seg4_bf16", + "sha256": "a3e43426dbbeb00409c2c52791998a12a05aa4664c42c26ef77649ce7d65e4ff", "status": "ok" }, "attention/gfx950/reduce_fp16_d128": { "arch": "gfx950", - "bytes": 9570, + "bytes": 9726, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_d128_h4_seg4_fp16", - "sha256": "beabc28143983d8b7435934ad4ac4c8f9b21ee351d15132153f37b0a15272a97", + "kernel_name": "rocke_uattn_reduce_tiled_d128_seg4_fp16", + "sha256": "2f121f51da067142e50d33d27ae2d7a82d58792f119ad4a3d9bd7248718d2db7", "status": "ok" }, "attention/gfx950/reduce_lse_bf16_d128": { "arch": "gfx950", - "bytes": 10548, + "bytes": 10719, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_d128_h4_seg4_bf16_lse", - "sha256": "3e28886dc22e9f20989a4ecaedcc388a6a38a60d8822f5a7f39d303f5515d97f", + "kernel_name": "rocke_uattn_reduce_tiled_d128_seg4_bf16_lse", + "sha256": "a699fff63f43aa6b3723d28b7f489d0e05b6ea0d82fe1c3b0c940bbbbc9bd2ca", "status": "ok" }, "attention/gfx950/scalar2d_fp16_d128_no_mask": { @@ -386,306 +386,306 @@ "cases": { "attention/gfx942/2d_bf16_d128_b16_decode_i64": { "arch": "gfx942", - "bytes": 465458, + "bytes": 467025, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b16_t64_h32kv8_bf16_wpe2_regpv", - "sha256": "e76ba89008060a838f26f485302891dadfab7e40ad6d4557481854ce2aa51ae1", + "kernel_name": "rocke_uattn2d_tiled_d128_b16_t64_g4_bf16_i64_wpe2_regpv", + "sha256": "dc94853ae5351d5d1b20ad86e4ce2fac67f72d6c7b44adf379f576462692e323", "status": "ok" }, "attention/gfx942/2d_bf16_d128_b64": { "arch": "gfx942", - "bytes": 394842, + "bytes": 396349, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16", - "sha256": "2170701a2a07d593102a376b3971907d1a753e39a6e41bd092df2bcb92370062", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16", + "sha256": "dad984c5e78bda96c774b8dbec8301b6b3f45a300c7b3cbd752bcfd28be933f7", "status": "ok" }, "attention/gfx942/2d_fp16_d128_b64": { "arch": "gfx942", - "bytes": 383172, + "bytes": 384679, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16", - "sha256": "1e163d35d59bacf3ab52922e747c369d16481f8dd464607f7bfb3ee77c8bd6bf", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16", + "sha256": "b3f16be414bab20b81c804dcd3a4f14a914542d048527e24c1edeb532ae59423", "status": "ok" }, "attention/gfx942/2d_lse_mfma32_bf16_d128": { "arch": "gfx942", - "bytes": 529471, + "bytes": 531021, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32x8_lse", - "sha256": "eacec37309a870aa88f00bd154b0fd7bd289918bf09fc83c9cebdb0ca71ef3ec", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32x8_lse", + "sha256": "65048192c48bfcdc484e5eeb3c9f212833791acb496558ef80af8cb80e26ee8b", "status": "ok" }, "attention/gfx942/2d_lse_striped16_bf16_d128": { "arch": "gfx942", - "bytes": 400576, + "bytes": 402203, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_lse", - "sha256": "855aa198f3f4c819fa4581dbd88ce6db9eda921737e77aaaa3df665abdb956d0", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_lse", + "sha256": "76190c1916c9c3d61ebbe516370a67c243fa1a677ff56b3d339066b3fd4c6a44", "status": "ok" }, "attention/gfx942/2d_lse_transposed32_bf16_d128": { "arch": "gfx942", - "bytes": 488348, + "bytes": 489999, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32x8_stqk_lse", - "sha256": "28964d416187a795bc8ffa73136cee4beba4399c0671e0e84a8d949acd61348e", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32x8_stqk_lse", + "sha256": "706b2c69142c6a22c21d0ffba51cab6367622f14692d2676e4edf4ebbc975ca8", "status": "ok" }, "attention/gfx942/3d_bf16_d128_b64": { "arch": "gfx942", - "bytes": 356076, + "bytes": 358781, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_h4kv2_seg4_bf16", - "sha256": "38f97d98aa14759dda281c4952e6571d4e1c6ff21ac08274473091c0e0bb0146", + "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_g2_seg4_bf16", + "sha256": "fad6de65bd50402a69c278452eaf61951e7fe4db3a6eba8dafece7d108a6d1ab", "status": "ok" }, "attention/gfx942/3d_fp16_d128_b64": { "arch": "gfx942", - "bytes": 344028, + "bytes": 346733, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_h4kv2_seg4_fp16", - "sha256": "2264bce69fae37533413945b5a0fa9778737b197ee8f3791043dbd13865736e4", + "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_g2_seg4_fp16", + "sha256": "902f7f7dc7eb0b2a5a21292899e44e816fb7112c844b6e199e9a087125df11b1", "status": "ok" }, "attention/gfx942/reduce_bf16_d128": { "arch": "gfx942", - "bytes": 9624, + "bytes": 9780, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_bf16", - "sha256": "7f3bea52ba6bedf52e5219615a0988da1f6870faba69d9d02cfc81cffca477f5", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_bf16", + "sha256": "be513de59d3fde4a6d0a372effed327b37d2d5de4b48161a6de5186dc692d721", "status": "ok" }, "attention/gfx942/reduce_bf16_d128_own_module": { "arch": "gfx942", - "bytes": 9624, + "bytes": 9780, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_bf16", - "sha256": "7f3bea52ba6bedf52e5219615a0988da1f6870faba69d9d02cfc81cffca477f5", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_bf16", + "sha256": "be513de59d3fde4a6d0a372effed327b37d2d5de4b48161a6de5186dc692d721", "status": "ok" }, "attention/gfx942/reduce_fp16_d128": { "arch": "gfx942", - "bytes": 9612, + "bytes": 9768, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_fp16", - "sha256": "d6db7088aa420dd3e8f81576f4ccf2d5a35f9bb211695ec3b9c2c38ff33139f1", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_fp16", + "sha256": "3d7c19d0b0422eeb6a1e558bf0c90ba458dfc659b5c49712b706960d39998a1e", "status": "ok" }, "attention/gfx942/reduce_lse_bf16_d128": { "arch": "gfx942", - "bytes": 10590, + "bytes": 10761, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_bf16_lse", - "sha256": "9f208dd3810d1e9712135bd2d022106e1bb7352290c18e8eb21d11bbc76fbb4f", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_bf16_lse", + "sha256": "14f4b73e8d658f8318f1af0baf18fe086fafc6d6c024f8e42630e4a5135d6313", "status": "ok" }, "attention/gfx950/2d_bf16_d128_b64": { "arch": "gfx950", - "bytes": 168379, + "bytes": 140867, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16", - "sha256": "658820521cee8af1a60b7b7553873cefb40757d405de47ff702989bfff0dea22", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16", + "sha256": "3b09194ba11e2285d27667e18993ae4b09058ce857af1a92f1078ff65969eac1", "status": "ok" }, "attention/gfx950/2d_bf16_d64_b32_gqa8_sinks_combo": { "arch": "gfx950", - "bytes": 117092, + "bytes": 117887, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_bf16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim_fastkvdesc", - "sha256": "5094d1472b6fce2f4a40b866fa8bcf69b622b2ff03c042b8873b9000e2f03722", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_h64kv8_bf16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim_fastkvdesc", + "sha256": "ce82af853fce8157326230c4de2a1c8c0e27206a01c04f290177a4a4409df90a", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64": { "arch": "gfx950", - "bytes": 166802, + "bytes": 139290, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16", - "sha256": "98e71ae61d6c1f8ff6ca59e35ed44d279551b113383990fcb5ea153e3f479220", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16", + "sha256": "63e0c0f37ee0100577efc5b4007e9c6f29605757d4cfdc76be1b593dadc445db", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_alibi": { "arch": "gfx950", - "bytes": 171977, + "bytes": 144520, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_alibi", - "sha256": "7318fe1d06c917d4e569b0360ff9e877230aba01e82da75a5e1df1c1f9a83541", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_alibi", + "sha256": "1d843b4c77015ce4c31e3941663d2bca0fa3ea881e80db0664fefc7935aadf8c", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_left_only_sw64": { "arch": "gfx950", - "bytes": 168271, + "bytes": 140847, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_sw64_tl_rbu", - "sha256": "205a2046b4ef0ab4da00a62dfce7993048f26dc92e31ea57747199e4adf60c6e", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_sw_tl_rbu", + "sha256": "bad4836aa613aced15382430e69d4c832fbd1bd97c4118d60cdaa3efdbca674d", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_no_mask": { "arch": "gfx950", - "bytes": 165434, + "bytes": 137918, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_rbu", - "sha256": "dc4dfa0ec9c67c5355ea17a9b030dcc7e3d27740adba4aa40af3821c03cdab32", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_rbu", + "sha256": "a7f3f644696d86dbd5cb353d40c0adea071f66771ab4275775b67f517b7ae0b1", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_rb16": { "arch": "gfx950", - "bytes": 167581, + "bytes": 140100, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_rb16", - "sha256": "32e13d494042664fc72ab10a7fa8141225a35e05ce20e6230209cfdf9ee636ed", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_rb16", + "sha256": "49a4dae2f6559943288e0b60af507d5311898b417f72c55eee8a0c687d39886a", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_softcap": { "arch": "gfx950", - "bytes": 174834, + "bytes": 147286, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_softcap", - "sha256": "7aec1d7dac73479d17b7cb6641549fba0a64c47e13f63b49a0ddd047d08ef316", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_softcap", + "sha256": "207c6572d45923fa94268e47e7aa7abdaf633640a14d2838fb2c0c4ea3f54add", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_top_left": { "arch": "gfx950", - "bytes": 166842, + "bytes": 139330, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_tl", - "sha256": "f7add7931bf1e95062559404fd3a0f0d8d095fe1a4a8a39dfade489d9a3056b2", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_tl", + "sha256": "01f49969b29bd992aa56b5956812b8a1838aca4762fdbc6b5be96fc07ca571b2", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_top_left_sw64": { "arch": "gfx950", - "bytes": 169632, + "bytes": 142194, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_sw64_tl", - "sha256": "e6fd4cde9164c0528c97ba34ee5281ba95fd705a1b9e9a645fb03ed48e3c3ac3", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_sw_tl", + "sha256": "01be693271ef19911ad8a1cccf7c716b0cda51c9c2833b3bed81254ce6e11ebf", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_two_sided_sw64_rb16": { "arch": "gfx950", - "bytes": 170449, + "bytes": 143033, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_sw64_tl_rb16", - "sha256": "5cf5e7f8b13f5a07203844a9a63955943a3aaae3945403b45dade3542e3c95b5", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_sw_tl_rb16", + "sha256": "0dcee701d234f0686f571cd03bd9195aaa16b3e8e43f633fd14151e63677a41b", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_combo_no_mask": { "arch": "gfx950", - "bytes": 115843, + "bytes": 115239, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_rbu_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "e21dbee0e998d27b69ce8193386583eb1e6371f2822ef93aa99faa2cf75104cd", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_rbu_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "316957a721e2577c746ae50dfdb0f90836bbb1beb8eec92d5af0368a20cdef6b", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_combo_rb16": { "arch": "gfx950", - "bytes": 116179, + "bytes": 115592, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_rb16_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "d42c49b313f04bf6b6a1294c5cdd0c849133c15692755a9daeccb3566e3362b3", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_rb16_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "4e7530d5858b9a88c14b39586de76cc1462bbe5c765f4a8c763ef9aa3c4b2696", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_combo_top_left": { "arch": "gfx950", - "bytes": 115971, + "bytes": 115370, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_tl_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "56b13921b15ff66e8d3521054478012060ad759047d5bc100825593f9f8b4e66", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_tl_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "0eba15e54a6783dc45a48d273168b12a7d63be2adad24e097fe26b6fdc56014a", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_sinks_combo": { "arch": "gfx950", - "bytes": 116650, + "bytes": 116091, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "caf014e9bad532cde9c3820ed188cdc6ff56df20ed3f8d1796a1a81fa14f210f", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "231f98b8b174d680c2cb4e1f73086ab67e99c73e2c67503fd2f184d25d8e0e51", "status": "ok" }, "attention/gfx950/2d_lse_mfma32_bf16_d128": { "arch": "gfx950", - "bytes": 314780, + "bytes": 287741, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32_lse", - "sha256": "1b02b801e28881de8dd40b0994d533573d780ebb91738de7ce07d02ed6e201f5", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32_lse", + "sha256": "45170f0d7ccb4d2f6217df49907b17b8af3301fb12e1b65ee9086c2b9e09c357", "status": "ok" }, "attention/gfx950/2d_lse_striped16_bf16_d128": { "arch": "gfx950", - "bytes": 172349, + "bytes": 144897, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_lse", - "sha256": "be5dfe6ba0fc542d3cc1a7490cef8bc3951fe502bea4c74f8e4f877c4df52534", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_lse", + "sha256": "0b856a126c3f0d34dc1496434f2159cb8d090a99df116f2624d38509d7021295", "status": "ok" }, "attention/gfx950/2d_lse_transposed32_bf16_d128": { "arch": "gfx950", - "bytes": 352950, + "bytes": 326288, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32_stqk_s1_lse", - "sha256": "8083406611e0cce60ef09da21e842a7b12a46a20830e7b4ed30d2075d33e3753", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32_stqk_s1_lse", + "sha256": "93bd2edc732ca53cd5e117784d69a00f546afde218e90ebfa4ea10d620cb33ba", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64": { "arch": "gfx950", - "bytes": 161513, + "bytes": 163447, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16", - "sha256": "fc2317de951975a2b4e8cb60cc366ba9c5a247c8ca095615e936d14a1acd4f56", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16", + "sha256": "050a52b77e77931513af83a3ab5c69f088a3d7483a21cfcdba82a1a9bbb5fe8f", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64_no_mask": { "arch": "gfx950", - "bytes": 160032, + "bytes": 161966, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16_rbu", - "sha256": "0995eea05db416b61c1e1cb41687c37c263710a0f67c3976b836bf17d9af66a3", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16_rbu", + "sha256": "c4d124595b1970e86378eb29358bfd18f1813bad78226e81e25b52c2dbdda5b7", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64_rb16": { "arch": "gfx950", - "bytes": 162192, + "bytes": 164126, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16_rb16", - "sha256": "2fc3b55e07054ec1ba3fedfd0daa9e85ea56d4df60424550814481b8f9319f7d", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16_rb16", + "sha256": "bc7f23ff8c8c30262bd7e6725c2bbc3e021bbf6413eb5ae4378e861864364cc6", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64_top_left": { "arch": "gfx950", - "bytes": 161508, + "bytes": 163442, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16_tl", - "sha256": "caf565da35f0a6574e0d17853f07ab7b8474232254fe61160fff6a5b89fde405", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16_tl", + "sha256": "0032025571b4726db2a16f0baedb4b4f8676a11fcad8902d7bd7ee3874022b9a", "status": "ok" }, "attention/gfx950/3d_fp16_d128_b64": { "arch": "gfx950", - "bytes": 159552, + "bytes": 161486, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_fp16", - "sha256": "a886f8bfde1bc76812a5a27a2ad13a7b9353efdf2950b182025aa7dd056b133f", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_fp16", + "sha256": "fb3929f3c7de873b1860b4d64ad068dfea612b2f6bcc001faf5baa30f2d44fb7", "status": "ok" }, "attention/gfx950/reduce_bf16_d128": { "arch": "gfx950", - "bytes": 9589, + "bytes": 9745, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_d128_h4_seg4_bf16", - "sha256": "98e0ba897d0a0ed2d7c2d809bdfb39f6c49c5f1c33b854e73ec5583124f6283d", + "kernel_name": "rocke_uattn_reduce_tiled_d128_seg4_bf16", + "sha256": "098580ab1c7803d72d1b2bd362d07aef3376e502038da6fea1f9802852ae109b", "status": "ok" }, "attention/gfx950/reduce_fp16_d128": { "arch": "gfx950", - "bytes": 9577, + "bytes": 9733, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_d128_h4_seg4_fp16", - "sha256": "e02b0ee7cfad94216ed2f945b38f047c5ed4e7d1e53d9a10a5ad090ccc41f3be", + "kernel_name": "rocke_uattn_reduce_tiled_d128_seg4_fp16", + "sha256": "d5203691a7de9c415f0b9a13d7449e59001d688dc12ee0e3fae1398990911638", "status": "ok" }, "attention/gfx950/reduce_lse_bf16_d128": { "arch": "gfx950", - "bytes": 10555, + "bytes": 10726, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_d128_h4_seg4_bf16_lse", - "sha256": "93085cd44fd95418414f6dc4cfbdce7d93c8ab6fb4961baba927c9bac5306d36", + "kernel_name": "rocke_uattn_reduce_tiled_d128_seg4_bf16_lse", + "sha256": "d820d74b4f730648ace4e8c0350eb0100fa9406c1f6b4f42fe410b5b664942a0", "status": "ok" }, "attention/gfx950/scalar2d_fp16_d128_no_mask": { @@ -767,306 +767,306 @@ "cases": { "attention/gfx942/2d_bf16_d128_b16_decode_i64": { "arch": "gfx942", - "bytes": 465522, + "bytes": 467089, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b16_t64_h32kv8_bf16_wpe2_regpv", - "sha256": "4776e8d28fe58a4befb92306bd75f4ffdb2c452c74be4f48c40ec791260d5573", + "kernel_name": "rocke_uattn2d_tiled_d128_b16_t64_g4_bf16_i64_wpe2_regpv", + "sha256": "416fd50cde6c20af03f65e9b9a030b21dfb1b130561f0528fdf8ad5873607a57", "status": "ok" }, "attention/gfx942/2d_bf16_d128_b64": { "arch": "gfx942", - "bytes": 394906, + "bytes": 396413, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16", - "sha256": "9178be7f03ce761f4c8e68e93c5542ec04c08ae30a5e5bab33b891c55983142e", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16", + "sha256": "b1477e7e085bb8e6047715a7d9a8d70f4907f4e2625a966a8ce3eb5ba78093c4", "status": "ok" }, "attention/gfx942/2d_fp16_d128_b64": { "arch": "gfx942", - "bytes": 383236, + "bytes": 384743, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16", - "sha256": "32a9352ee9bfd836e5e361a7a986d1bfbe327f3907892bb2b58c10ad5f67a332", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16", + "sha256": "9c1723e34ee49b81231fb9dfe7e3104aadd66923380c375c4301bc3d6dafe491", "status": "ok" }, "attention/gfx942/2d_lse_mfma32_bf16_d128": { "arch": "gfx942", - "bytes": 529535, + "bytes": 531085, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32x8_lse", - "sha256": "5e9d09a0d5305df2237666634f17772352dd6df719ed0a5fa24fb1a3ffe9c1ce", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32x8_lse", + "sha256": "b90625c150ce0a1a148493aa96607bf8056bd4b1e48f3c6b1d933a5d9c7cdffb", "status": "ok" }, "attention/gfx942/2d_lse_striped16_bf16_d128": { "arch": "gfx942", - "bytes": 400640, + "bytes": 402267, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_lse", - "sha256": "ec9feed2587a67c4ec662655011ed404310dcd026f67dddbaa00eff39591f97a", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_lse", + "sha256": "ae4087509f74b1fc111d385b2718684fc78b1be40c94c547b58e98ffb763f7d1", "status": "ok" }, "attention/gfx942/2d_lse_transposed32_bf16_d128": { "arch": "gfx942", - "bytes": 488412, + "bytes": 490063, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32x8_stqk_lse", - "sha256": "eb9dd560dde0f9299fb58110dcf4c8dda71d376c03869f7f10c8045c7db75de9", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32x8_stqk_lse", + "sha256": "0bcf6c2957fd7053ab70ae5522e207a76a4c0db0596a84ec7fcfee29e478685d", "status": "ok" }, "attention/gfx942/3d_bf16_d128_b64": { "arch": "gfx942", - "bytes": 356140, + "bytes": 358845, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_h4kv2_seg4_bf16", - "sha256": "4b4572ad4abf3b71171d6bab1411d5f635c51a78806f0f4e9dd20c391be98336", + "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_g2_seg4_bf16", + "sha256": "e6cc4efe309c5291bf751ebc86e8beb998c17a980fbdb8a115aa871ef624f176", "status": "ok" }, "attention/gfx942/3d_fp16_d128_b64": { "arch": "gfx942", - "bytes": 344092, + "bytes": 346797, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_h4kv2_seg4_fp16", - "sha256": "c8668c7491fad1da8a7437d92c0236b0991bc9583db525ac0a7d97c1718f8da0", + "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_g2_seg4_fp16", + "sha256": "1bc69ccd17c9d37f88503cfc4cad479c7ad5ee76e8e861600cc04e79b00c8668", "status": "ok" }, "attention/gfx942/reduce_bf16_d128": { "arch": "gfx942", - "bytes": 9688, + "bytes": 9844, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_bf16", - "sha256": "71c374c060cef19fa32166265a6699405767e9ab966d032ce0a1645f4ecf441f", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_bf16", + "sha256": "ef2e7b04539ca47744a03071852d51c58bc0042bb41962b7c11507fd7afc84b5", "status": "ok" }, "attention/gfx942/reduce_bf16_d128_own_module": { "arch": "gfx942", - "bytes": 9688, + "bytes": 9844, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_bf16", - "sha256": "71c374c060cef19fa32166265a6699405767e9ab966d032ce0a1645f4ecf441f", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_bf16", + "sha256": "ef2e7b04539ca47744a03071852d51c58bc0042bb41962b7c11507fd7afc84b5", "status": "ok" }, "attention/gfx942/reduce_fp16_d128": { "arch": "gfx942", - "bytes": 9676, + "bytes": 9832, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_fp16", - "sha256": "c2dd8745dd4a4b54012c6ec13283b1813074e4eb527dc0c2bd5aea07ba836064", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_fp16", + "sha256": "ad96a8fb6a051e71143a4d5f1309fc437ee7c730d3d4e09fb76a43cdab5c8735", "status": "ok" }, "attention/gfx942/reduce_lse_bf16_d128": { "arch": "gfx942", - "bytes": 10654, + "bytes": 10825, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_bf16_lse", - "sha256": "b3bb8432873a068d142ab5d3eacca0dff88e0f97c9d4c88070aaa634ed6c8bf5", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_bf16_lse", + "sha256": "75c129c12e5e7bf55a9f6e7ce8c1c5974b0dd475540c9faed80550a6f8e24553", "status": "ok" }, "attention/gfx950/2d_bf16_d128_b64": { "arch": "gfx950", - "bytes": 168443, + "bytes": 140931, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16", - "sha256": "ddd16d5480b5c1d4b945b612b109d63eb67792da667ba27fcad61b62fc608f4f", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16", + "sha256": "3a49d4b73156dbb6da335bb7aa059f2cd43c6c1cb101c0a4167d4679daacd2b9", "status": "ok" }, "attention/gfx950/2d_bf16_d64_b32_gqa8_sinks_combo": { "arch": "gfx950", - "bytes": 117156, + "bytes": 117951, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_bf16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim_fastkvdesc", - "sha256": "2a13a454cc9be2c505b156253f1104668eebdb724dc1330155131b46a3e48a79", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_h64kv8_bf16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim_fastkvdesc", + "sha256": "3e6d6bf7936e31fde226ac9122963d6aaa6aeb9028bee4d2ed23adf48e3536b6", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64": { "arch": "gfx950", - "bytes": 166866, + "bytes": 139354, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16", - "sha256": "f4d65d4f6a7dfeab778bc157f0f9d36d67c69c00a22bb8b56e341a32302e337b", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16", + "sha256": "41cb61e3504654c2c1fcb81ce601c78ce0a03c72e37eb05269ad19eb14347742", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_alibi": { "arch": "gfx950", - "bytes": 172041, + "bytes": 144584, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_alibi", - "sha256": "ff90516fc26494f1962d4498bea6bd5c392741d370f5ad52a5474d1e4fdc755e", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_alibi", + "sha256": "38b15a981ea85b74b45f6bbeb8783a2f2ffb088ebe6ef1717ff5ddacff749151", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_left_only_sw64": { "arch": "gfx950", - "bytes": 168335, + "bytes": 140911, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_sw64_tl_rbu", - "sha256": "df6e85f400b1c751eca2ba564d52722582b804c98fbf85d3bc30ece1349626ea", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_sw_tl_rbu", + "sha256": "5f164ca3d0957df740f5f69d9274fea60fc7dcd4c6cd87e44906db11344dc1ad", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_no_mask": { "arch": "gfx950", - "bytes": 165498, + "bytes": 137982, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_rbu", - "sha256": "bb74d0175a4c03b1a922a484c4f29c183a74884c6d49eb753a0a155e7a694ef8", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_rbu", + "sha256": "619c3fbdc69d52920d46410b5a73af60bcfed8f6ea34710067e3f6c200f7ee87", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_rb16": { "arch": "gfx950", - "bytes": 167645, + "bytes": 140164, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_rb16", - "sha256": "b22f1c48d558c124dc347b4b7525fa39daea12047e8f0429ba501ac79f6762ae", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_rb16", + "sha256": "b444a13050077ed6aeb35599b572b445bfaf00b4243ccea73593d978b3736f7a", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_softcap": { "arch": "gfx950", - "bytes": 174898, + "bytes": 147350, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_softcap", - "sha256": "c3d9c4405a1d4fa9a2efd84deaeb0058fe54cfdd9e3e170fbfc352db44425c82", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_softcap", + "sha256": "8b06fd99bccf1aec94680ef81d4d1c3a744f1b446f16dc46b226f963794e0527", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_top_left": { "arch": "gfx950", - "bytes": 166906, + "bytes": 139394, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_tl", - "sha256": "385fd5ab3fada1f7a76f42015833c6b71a8d59feaa22da163a584db8495c5936", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_tl", + "sha256": "e8deb9acc834709eb03c098b90ac58b37d1a9e0714da04c54eb5358f88662099", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_top_left_sw64": { "arch": "gfx950", - "bytes": 169696, + "bytes": 142258, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_sw64_tl", - "sha256": "17cf6203871d2843a0a563c1e93467c642f617bc640a8757c7be3829f59b2106", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_sw_tl", + "sha256": "fdbb67c9fb989c19847623a093d5519d60963e0dd2a60cb4a26395b17e9022ca", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_two_sided_sw64_rb16": { "arch": "gfx950", - "bytes": 170513, + "bytes": 143097, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_sw64_tl_rb16", - "sha256": "8fe4ad07bdb5de7998bec2ab5822d0bb7af0fc94315e6f5e696c9f224a60987e", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_sw_tl_rb16", + "sha256": "f059e1ec56ca6a60c717c156a4011fbcfe79765b7c5479b24d297344122efd3c", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_combo_no_mask": { "arch": "gfx950", - "bytes": 115907, + "bytes": 115303, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_rbu_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "697dcb1f61a740e28bde90711b2e0e211d2e9f805d0cd4581acccc8b6adf8f50", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_rbu_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "274cc97839dfa73e7aca6a971824613afaa0bb536e3949c0ccdc8d7a5927afe9", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_combo_rb16": { "arch": "gfx950", - "bytes": 116243, + "bytes": 115656, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_rb16_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "97be2074b90404ec9d07830bad6026c8e5af2e1bb2b332eebe7e2e9d87c8194b", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_rb16_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "08d2a4e257611faa4083dc7ddfe1447ff545b9ab91ded3d40a0288cd9941a7ca", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_combo_top_left": { "arch": "gfx950", - "bytes": 116035, + "bytes": 115434, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_tl_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "a41b59f1dd014780deb2f8f6481e548c85240b7d96bcc0c5975fe681833bf40d", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_tl_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "669cb49916ba5f302dbca087ef4c771965697667408d65078dd5d976dc621d4e", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_sinks_combo": { "arch": "gfx950", - "bytes": 116714, + "bytes": 116155, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "2f9446585481aa76c6a8074e5212a1408958d62ff44bf3344cdb5f165d7584e8", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "0e1a84c5614990fa68907c703ccbd6e969a694791ab0d420114f47ad649a6373", "status": "ok" }, "attention/gfx950/2d_lse_mfma32_bf16_d128": { "arch": "gfx950", - "bytes": 314844, + "bytes": 287805, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32_lse", - "sha256": "d8d2009e3e81717e739b4ba32997f2b94d28983e5e0fb0d25343c75f576874c0", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32_lse", + "sha256": "50ea36605aa0238f54910ec6b72988223eec414c2bbce0f0351eaa68ed0ce43e", "status": "ok" }, "attention/gfx950/2d_lse_striped16_bf16_d128": { "arch": "gfx950", - "bytes": 172413, + "bytes": 144961, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_lse", - "sha256": "ecfbd1d3263631b2aa039d5fb2b6e8861777b66e36cf76b9a70603014b1eaf5b", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_lse", + "sha256": "ec2c79028133d22fe82a5f16295fd3f6e97ac6d178d440f0ac7a26a987b498d3", "status": "ok" }, "attention/gfx950/2d_lse_transposed32_bf16_d128": { "arch": "gfx950", - "bytes": 353014, + "bytes": 326352, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32_stqk_s1_lse", - "sha256": "d68a9bb5b0fc36915d914b18041e66c8fbab807a4c4734dc973a886007a83f69", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32_stqk_s1_lse", + "sha256": "364a9ed632be83a20cff8147f4cc1e66ead3ad5f5f0da3ab171dd8adbcacce8a", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64": { "arch": "gfx950", - "bytes": 161577, + "bytes": 163511, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16", - "sha256": "8bb93d90bb6b918c9e5ca741713b8f527330c08f0ec64fb8399c5607f95f93e4", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16", + "sha256": "ede4f268f6987f25f6502f1567e583439e01b7406e41f3102022458b7ce874c5", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64_no_mask": { "arch": "gfx950", - "bytes": 160096, + "bytes": 162030, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16_rbu", - "sha256": "94a385dc5f7a749e5b3286996fc9c1891c353bcfc4caac8ebf07bbd40475ba36", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16_rbu", + "sha256": "291ebec1509a9aa66b0e4b08e1ef847eab247af77ba43d05f8979fc5b37d77c3", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64_rb16": { "arch": "gfx950", - "bytes": 162256, + "bytes": 164190, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16_rb16", - "sha256": "9ba252e2a18e12bb50e49d9a53de0d44519fdc53445e893b966ed6510c3272de", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16_rb16", + "sha256": "90038c0ae2fdf637d30f59b02bf1545c4e3414b1fa7ab0a1e2cee3a12ad5c33a", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64_top_left": { "arch": "gfx950", - "bytes": 161572, + "bytes": 163506, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16_tl", - "sha256": "044a112eaa821c7cdb13e0e501b0019d12f13189eb6eedee6118696b9063ab86", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16_tl", + "sha256": "489d7be5499ac03f654285e6532d867f87a64d85d3d7fb2f37c60b17bca591d0", "status": "ok" }, "attention/gfx950/3d_fp16_d128_b64": { "arch": "gfx950", - "bytes": 159616, + "bytes": 161550, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_fp16", - "sha256": "6ec6b7296f4019b340866cec1f71786d02f696256b3195ffae0d1387aa52f46c", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_fp16", + "sha256": "43f2721a7212e095b5805d2bdd5b875a09b2050be6cb68b5405feb6292e18d6b", "status": "ok" }, "attention/gfx950/reduce_bf16_d128": { "arch": "gfx950", - "bytes": 9653, + "bytes": 9809, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_d128_h4_seg4_bf16", - "sha256": "19d7d61c6f782d75843f9f8feb9b266a793b186da1d817b477612cc47e4a32ca", + "kernel_name": "rocke_uattn_reduce_tiled_d128_seg4_bf16", + "sha256": "622b5c24eb92d4b7fcf99a4506b63d6793cf92bc683efe3302418559aa8c3cf7", "status": "ok" }, "attention/gfx950/reduce_fp16_d128": { "arch": "gfx950", - "bytes": 9641, + "bytes": 9797, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_d128_h4_seg4_fp16", - "sha256": "86a97d40e5b330935ede77f65469c2b6be4842406951f2d75192d713a07a0010", + "kernel_name": "rocke_uattn_reduce_tiled_d128_seg4_fp16", + "sha256": "86bfc8e5818512ff10660abe064f7614d71cab37f33447828062c802f0cf36b7", "status": "ok" }, "attention/gfx950/reduce_lse_bf16_d128": { "arch": "gfx950", - "bytes": 10619, + "bytes": 10790, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_d128_h4_seg4_bf16_lse", - "sha256": "d7f993576194612f2c4beef0b5c0855e1a9502c8e2cc54a724386e69ee498963", + "kernel_name": "rocke_uattn_reduce_tiled_d128_seg4_bf16_lse", + "sha256": "431a3bc4f91f1ab1168b2cb00110602f83a8c9328d93b12d080701c07d3b899a", "status": "ok" }, "attention/gfx950/scalar2d_fp16_d128_no_mask": { @@ -1148,306 +1148,306 @@ "cases": { "attention/gfx942/2d_bf16_d128_b16_decode_i64": { "arch": "gfx942", - "bytes": 465522, + "bytes": 467089, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b16_t64_h32kv8_bf16_wpe2_regpv", - "sha256": "4776e8d28fe58a4befb92306bd75f4ffdb2c452c74be4f48c40ec791260d5573", + "kernel_name": "rocke_uattn2d_tiled_d128_b16_t64_g4_bf16_i64_wpe2_regpv", + "sha256": "416fd50cde6c20af03f65e9b9a030b21dfb1b130561f0528fdf8ad5873607a57", "status": "ok" }, "attention/gfx942/2d_bf16_d128_b64": { "arch": "gfx942", - "bytes": 394906, + "bytes": 396413, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16", - "sha256": "9178be7f03ce761f4c8e68e93c5542ec04c08ae30a5e5bab33b891c55983142e", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16", + "sha256": "b1477e7e085bb8e6047715a7d9a8d70f4907f4e2625a966a8ce3eb5ba78093c4", "status": "ok" }, "attention/gfx942/2d_fp16_d128_b64": { "arch": "gfx942", - "bytes": 383236, + "bytes": 384743, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16", - "sha256": "32a9352ee9bfd836e5e361a7a986d1bfbe327f3907892bb2b58c10ad5f67a332", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16", + "sha256": "9c1723e34ee49b81231fb9dfe7e3104aadd66923380c375c4301bc3d6dafe491", "status": "ok" }, "attention/gfx942/2d_lse_mfma32_bf16_d128": { "arch": "gfx942", - "bytes": 529535, + "bytes": 531085, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32x8_lse", - "sha256": "5e9d09a0d5305df2237666634f17772352dd6df719ed0a5fa24fb1a3ffe9c1ce", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32x8_lse", + "sha256": "b90625c150ce0a1a148493aa96607bf8056bd4b1e48f3c6b1d933a5d9c7cdffb", "status": "ok" }, "attention/gfx942/2d_lse_striped16_bf16_d128": { "arch": "gfx942", - "bytes": 400640, + "bytes": 402267, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_lse", - "sha256": "ec9feed2587a67c4ec662655011ed404310dcd026f67dddbaa00eff39591f97a", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_lse", + "sha256": "ae4087509f74b1fc111d385b2718684fc78b1be40c94c547b58e98ffb763f7d1", "status": "ok" }, "attention/gfx942/2d_lse_transposed32_bf16_d128": { "arch": "gfx942", - "bytes": 488412, + "bytes": 490063, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32x8_stqk_lse", - "sha256": "eb9dd560dde0f9299fb58110dcf4c8dda71d376c03869f7f10c8045c7db75de9", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32x8_stqk_lse", + "sha256": "0bcf6c2957fd7053ab70ae5522e207a76a4c0db0596a84ec7fcfee29e478685d", "status": "ok" }, "attention/gfx942/3d_bf16_d128_b64": { "arch": "gfx942", - "bytes": 356140, + "bytes": 358845, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_h4kv2_seg4_bf16", - "sha256": "4b4572ad4abf3b71171d6bab1411d5f635c51a78806f0f4e9dd20c391be98336", + "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_g2_seg4_bf16", + "sha256": "e6cc4efe309c5291bf751ebc86e8beb998c17a980fbdb8a115aa871ef624f176", "status": "ok" }, "attention/gfx942/3d_fp16_d128_b64": { "arch": "gfx942", - "bytes": 344092, + "bytes": 346797, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_h4kv2_seg4_fp16", - "sha256": "c8668c7491fad1da8a7437d92c0236b0991bc9583db525ac0a7d97c1718f8da0", + "kernel_name": "rocke_uattn3d_tiled_gfx942_d128_b64_g2_seg4_fp16", + "sha256": "1bc69ccd17c9d37f88503cfc4cad479c7ad5ee76e8e861600cc04e79b00c8668", "status": "ok" }, "attention/gfx942/reduce_bf16_d128": { "arch": "gfx942", - "bytes": 9688, + "bytes": 9844, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_bf16", - "sha256": "71c374c060cef19fa32166265a6699405767e9ab966d032ce0a1645f4ecf441f", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_bf16", + "sha256": "ef2e7b04539ca47744a03071852d51c58bc0042bb41962b7c11507fd7afc84b5", "status": "ok" }, "attention/gfx942/reduce_bf16_d128_own_module": { "arch": "gfx942", - "bytes": 9688, + "bytes": 9844, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_bf16", - "sha256": "71c374c060cef19fa32166265a6699405767e9ab966d032ce0a1645f4ecf441f", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_bf16", + "sha256": "ef2e7b04539ca47744a03071852d51c58bc0042bb41962b7c11507fd7afc84b5", "status": "ok" }, "attention/gfx942/reduce_fp16_d128": { "arch": "gfx942", - "bytes": 9676, + "bytes": 9832, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_fp16", - "sha256": "c2dd8745dd4a4b54012c6ec13283b1813074e4eb527dc0c2bd5aea07ba836064", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_fp16", + "sha256": "ad96a8fb6a051e71143a4d5f1309fc437ee7c730d3d4e09fb76a43cdab5c8735", "status": "ok" }, "attention/gfx942/reduce_lse_bf16_d128": { "arch": "gfx942", - "bytes": 10654, + "bytes": 10825, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_h4_seg4_bf16_lse", - "sha256": "b3bb8432873a068d142ab5d3eacca0dff88e0f97c9d4c88070aaa634ed6c8bf5", + "kernel_name": "rocke_uattn_reduce_tiled_gfx942_d128_seg4_bf16_lse", + "sha256": "75c129c12e5e7bf55a9f6e7ce8c1c5974b0dd475540c9faed80550a6f8e24553", "status": "ok" }, "attention/gfx950/2d_bf16_d128_b64": { "arch": "gfx950", - "bytes": 168443, + "bytes": 140931, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16", - "sha256": "ddd16d5480b5c1d4b945b612b109d63eb67792da667ba27fcad61b62fc608f4f", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16", + "sha256": "3a49d4b73156dbb6da335bb7aa059f2cd43c6c1cb101c0a4167d4679daacd2b9", "status": "ok" }, "attention/gfx950/2d_bf16_d64_b32_gqa8_sinks_combo": { "arch": "gfx950", - "bytes": 117156, + "bytes": 117951, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_bf16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim_fastkvdesc", - "sha256": "2a13a454cc9be2c505b156253f1104668eebdb724dc1330155131b46a3e48a79", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_h64kv8_bf16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim_fastkvdesc", + "sha256": "3e6d6bf7936e31fde226ac9122963d6aaa6aeb9028bee4d2ed23adf48e3536b6", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64": { "arch": "gfx950", - "bytes": 166866, + "bytes": 139354, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16", - "sha256": "f4d65d4f6a7dfeab778bc157f0f9d36d67c69c00a22bb8b56e341a32302e337b", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16", + "sha256": "41cb61e3504654c2c1fcb81ce601c78ce0a03c72e37eb05269ad19eb14347742", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_alibi": { "arch": "gfx950", - "bytes": 172041, + "bytes": 144584, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_alibi", - "sha256": "ff90516fc26494f1962d4498bea6bd5c392741d370f5ad52a5474d1e4fdc755e", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_alibi", + "sha256": "38b15a981ea85b74b45f6bbeb8783a2f2ffb088ebe6ef1717ff5ddacff749151", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_left_only_sw64": { "arch": "gfx950", - "bytes": 168335, + "bytes": 140911, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_sw64_tl_rbu", - "sha256": "df6e85f400b1c751eca2ba564d52722582b804c98fbf85d3bc30ece1349626ea", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_sw_tl_rbu", + "sha256": "5f164ca3d0957df740f5f69d9274fea60fc7dcd4c6cd87e44906db11344dc1ad", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_no_mask": { "arch": "gfx950", - "bytes": 165498, + "bytes": 137982, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_rbu", - "sha256": "bb74d0175a4c03b1a922a484c4f29c183a74884c6d49eb753a0a155e7a694ef8", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_rbu", + "sha256": "619c3fbdc69d52920d46410b5a73af60bcfed8f6ea34710067e3f6c200f7ee87", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_rb16": { "arch": "gfx950", - "bytes": 167645, + "bytes": 140164, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_rb16", - "sha256": "b22f1c48d558c124dc347b4b7525fa39daea12047e8f0429ba501ac79f6762ae", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_rb16", + "sha256": "b444a13050077ed6aeb35599b572b445bfaf00b4243ccea73593d978b3736f7a", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_softcap": { "arch": "gfx950", - "bytes": 174898, + "bytes": 147350, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_softcap", - "sha256": "c3d9c4405a1d4fa9a2efd84deaeb0058fe54cfdd9e3e170fbfc352db44425c82", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_softcap", + "sha256": "8b06fd99bccf1aec94680ef81d4d1c3a744f1b446f16dc46b226f963794e0527", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_top_left": { "arch": "gfx950", - "bytes": 166906, + "bytes": 139394, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_tl", - "sha256": "385fd5ab3fada1f7a76f42015833c6b71a8d59feaa22da163a584db8495c5936", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_tl", + "sha256": "e8deb9acc834709eb03c098b90ac58b37d1a9e0714da04c54eb5358f88662099", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_top_left_sw64": { "arch": "gfx950", - "bytes": 169696, + "bytes": 142258, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_sw64_tl", - "sha256": "17cf6203871d2843a0a563c1e93467c642f617bc640a8757c7be3829f59b2106", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_sw_tl", + "sha256": "fdbb67c9fb989c19847623a093d5519d60963e0dd2a60cb4a26395b17e9022ca", "status": "ok" }, "attention/gfx950/2d_fp16_d128_b64_two_sided_sw64_rb16": { "arch": "gfx950", - "bytes": 170513, + "bytes": 143097, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_fp16_sw64_tl_rb16", - "sha256": "8fe4ad07bdb5de7998bec2ab5822d0bb7af0fc94315e6f5e696c9f224a60987e", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_fp16_sw_tl_rb16", + "sha256": "f059e1ec56ca6a60c717c156a4011fbcfe79765b7c5479b24d297344122efd3c", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_combo_no_mask": { "arch": "gfx950", - "bytes": 115907, + "bytes": 115303, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_rbu_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "697dcb1f61a740e28bde90711b2e0e211d2e9f805d0cd4581acccc8b6adf8f50", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_rbu_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "274cc97839dfa73e7aca6a971824613afaa0bb536e3949c0ccdc8d7a5927afe9", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_combo_rb16": { "arch": "gfx950", - "bytes": 116243, + "bytes": 115656, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_rb16_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "97be2074b90404ec9d07830bad6026c8e5af2e1bb2b332eebe7e2e9d87c8194b", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_rb16_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "08d2a4e257611faa4083dc7ddfe1447ff545b9ab91ded3d40a0288cd9941a7ca", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_combo_top_left": { "arch": "gfx950", - "bytes": 116035, + "bytes": 115434, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_tl_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "a41b59f1dd014780deb2f8f6481e548c85240b7d96bcc0c5975fe681833bf40d", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_tl_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "669cb49916ba5f302dbca087ef4c771965697667408d65078dd5d976dc621d4e", "status": "ok" }, "attention/gfx950/2d_fp16_d64_b32_gqa8_sinks_combo": { "arch": "gfx950", - "bytes": 116714, + "bytes": 116155, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_h64kv8_fp16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", - "sha256": "2f9446585481aa76c6a8074e5212a1408958d62ff44bf3344cdb5f165d7584e8", + "kernel_name": "rocke_uattn2d_tiled_d64_b32_t64_g8_fp16_sinks_w4_mw32_mfma32_stqk_s1_mask1_hlpv_skipqreg_mlim", + "sha256": "0e1a84c5614990fa68907c703ccbd6e969a694791ab0d420114f47ad649a6373", "status": "ok" }, "attention/gfx950/2d_lse_mfma32_bf16_d128": { "arch": "gfx950", - "bytes": 314844, + "bytes": 287805, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32_lse", - "sha256": "d8d2009e3e81717e739b4ba32997f2b94d28983e5e0fb0d25343c75f576874c0", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32_lse", + "sha256": "50ea36605aa0238f54910ec6b72988223eec414c2bbce0f0351eaa68ed0ce43e", "status": "ok" }, "attention/gfx950/2d_lse_striped16_bf16_d128": { "arch": "gfx950", - "bytes": 172413, + "bytes": 144961, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_lse", - "sha256": "ecfbd1d3263631b2aa039d5fb2b6e8861777b66e36cf76b9a70603014b1eaf5b", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_lse", + "sha256": "ec2c79028133d22fe82a5f16295fd3f6e97ac6d178d440f0ac7a26a987b498d3", "status": "ok" }, "attention/gfx950/2d_lse_transposed32_bf16_d128": { "arch": "gfx950", - "bytes": 353014, + "bytes": 326352, "family": "attention", - "kernel_name": "rocke_uattn2d_tiled_d128_b64_h4kv2_bf16_mw32_mfma32_stqk_s1_lse", - "sha256": "d68a9bb5b0fc36915d914b18041e66c8fbab807a4c4734dc973a886007a83f69", + "kernel_name": "rocke_uattn2d_tiled_d128_b64_g2_bf16_mw32_mfma32_stqk_s1_lse", + "sha256": "364a9ed632be83a20cff8147f4cc1e66ead3ad5f5f0da3ab171dd8adbcacce8a", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64": { "arch": "gfx950", - "bytes": 161577, + "bytes": 163511, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16", - "sha256": "8bb93d90bb6b918c9e5ca741713b8f527330c08f0ec64fb8399c5607f95f93e4", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16", + "sha256": "ede4f268f6987f25f6502f1567e583439e01b7406e41f3102022458b7ce874c5", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64_no_mask": { "arch": "gfx950", - "bytes": 160096, + "bytes": 162030, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16_rbu", - "sha256": "94a385dc5f7a749e5b3286996fc9c1891c353bcfc4caac8ebf07bbd40475ba36", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16_rbu", + "sha256": "291ebec1509a9aa66b0e4b08e1ef847eab247af77ba43d05f8979fc5b37d77c3", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64_rb16": { "arch": "gfx950", - "bytes": 162256, + "bytes": 164190, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16_rb16", - "sha256": "9ba252e2a18e12bb50e49d9a53de0d44519fdc53445e893b966ed6510c3272de", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16_rb16", + "sha256": "90038c0ae2fdf637d30f59b02bf1545c4e3414b1fa7ab0a1e2cee3a12ad5c33a", "status": "ok" }, "attention/gfx950/3d_bf16_d128_b64_top_left": { "arch": "gfx950", - "bytes": 161572, + "bytes": 163506, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_bf16_tl", - "sha256": "044a112eaa821c7cdb13e0e501b0019d12f13189eb6eedee6118696b9063ab86", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_bf16_tl", + "sha256": "489d7be5499ac03f654285e6532d867f87a64d85d3d7fb2f37c60b17bca591d0", "status": "ok" }, "attention/gfx950/3d_fp16_d128_b64": { "arch": "gfx950", - "bytes": 159616, + "bytes": 161550, "family": "attention", - "kernel_name": "rocke_uattn3d_tiled_d128_b64_h4kv2_seg4_fp16", - "sha256": "6ec6b7296f4019b340866cec1f71786d02f696256b3195ffae0d1387aa52f46c", + "kernel_name": "rocke_uattn3d_tiled_d128_b64_g2_seg4_fp16", + "sha256": "43f2721a7212e095b5805d2bdd5b875a09b2050be6cb68b5405feb6292e18d6b", "status": "ok" }, "attention/gfx950/reduce_bf16_d128": { "arch": "gfx950", - "bytes": 9653, + "bytes": 9809, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_d128_h4_seg4_bf16", - "sha256": "19d7d61c6f782d75843f9f8feb9b266a793b186da1d817b477612cc47e4a32ca", + "kernel_name": "rocke_uattn_reduce_tiled_d128_seg4_bf16", + "sha256": "622b5c24eb92d4b7fcf99a4506b63d6793cf92bc683efe3302418559aa8c3cf7", "status": "ok" }, "attention/gfx950/reduce_fp16_d128": { "arch": "gfx950", - "bytes": 9641, + "bytes": 9797, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_d128_h4_seg4_fp16", - "sha256": "86a97d40e5b330935ede77f65469c2b6be4842406951f2d75192d713a07a0010", + "kernel_name": "rocke_uattn_reduce_tiled_d128_seg4_fp16", + "sha256": "86bfc8e5818512ff10660abe064f7614d71cab37f33447828062c802f0cf36b7", "status": "ok" }, "attention/gfx950/reduce_lse_bf16_d128": { "arch": "gfx950", - "bytes": 10619, + "bytes": 10790, "family": "attention", - "kernel_name": "rocke_uattn_reduce_tiled_d128_h4_seg4_bf16_lse", - "sha256": "d7f993576194612f2c4beef0b5c0855e1a9502c8e2cc54a724386e69ee498963", + "kernel_name": "rocke_uattn_reduce_tiled_d128_seg4_bf16_lse", + "sha256": "431a3bc4f91f1ab1168b2cb00110602f83a8c9328d93b12d080701c07d3b899a", "status": "ok" }, "attention/gfx950/scalar2d_fp16_d128_no_mask": { diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256/attention_d128_swa.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256/attention_d128_swa.json index 9af45e90b056..16d414babbf1 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256/attention_d128_swa.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256/attention_d128_swa.json @@ -5,10 +5,10 @@ "cases": { "attention_d128_swa/gfx942/4warp_gqa_fold": { "arch": "gfx942", - "bytes": 398218, + "bytes": 398438, "family": "attention_d128_swa", - "kernel_name": "rocke_uattn2d_tiled_d128_b16_t32_h32kv8_bf16_sw4096_wpe2_mw32_4wgqa_fold", - "sha256": "d07edfd2a3dc52bd1e53d9aafa9a08a7d02a3a0a3a146ee75b278cd833abba53", + "kernel_name": "rocke_uattn2d_tiled_d128_b16_t32_g4_bf16_sw_wpe2_mw32_4wgqa_fold", + "sha256": "27332106786c2556edbe87534f3c3091f416744e1f4bdc8574d0f9955a5307e7", "status": "ok" } }, @@ -18,10 +18,10 @@ "cases": { "attention_d128_swa/gfx942/4warp_gqa_fold": { "arch": "gfx942", - "bytes": 398225, + "bytes": 398445, "family": "attention_d128_swa", - "kernel_name": "rocke_uattn2d_tiled_d128_b16_t32_h32kv8_bf16_sw4096_wpe2_mw32_4wgqa_fold", - "sha256": "f3c8eebff4d56db447aed23d80287e9db624f0839acb03331552b77f53fa5665", + "kernel_name": "rocke_uattn2d_tiled_d128_b16_t32_g4_bf16_sw_wpe2_mw32_4wgqa_fold", + "sha256": "f21f758dd2fce376bbbc2cfed37875427c90f87f71cd7b556e361cc3ccfbf280", "status": "ok" } }, @@ -31,10 +31,10 @@ "cases": { "attention_d128_swa/gfx942/4warp_gqa_fold": { "arch": "gfx942", - "bytes": 398289, + "bytes": 398509, "family": "attention_d128_swa", - "kernel_name": "rocke_uattn2d_tiled_d128_b16_t32_h32kv8_bf16_sw4096_wpe2_mw32_4wgqa_fold", - "sha256": "055026137da184d8640ca006fbe17d0d7de439ecbbd846af4d63d07adbad3141", + "kernel_name": "rocke_uattn2d_tiled_d128_b16_t32_g4_bf16_sw_wpe2_mw32_4wgqa_fold", + "sha256": "2c844b7c30e23d370aa5057b645020505ccca11954963fd905fc896bc64b67be", "status": "ok" } }, @@ -44,10 +44,10 @@ "cases": { "attention_d128_swa/gfx942/4warp_gqa_fold": { "arch": "gfx942", - "bytes": 398289, + "bytes": 398509, "family": "attention_d128_swa", - "kernel_name": "rocke_uattn2d_tiled_d128_b16_t32_h32kv8_bf16_sw4096_wpe2_mw32_4wgqa_fold", - "sha256": "055026137da184d8640ca006fbe17d0d7de439ecbbd846af4d63d07adbad3141", + "kernel_name": "rocke_uattn2d_tiled_d128_b16_t32_g4_bf16_sw_wpe2_mw32_4wgqa_fold", + "sha256": "2c844b7c30e23d370aa5057b645020505ccca11954963fd905fc896bc64b67be", "status": "ok" } }, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256/attention_d256.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256/attention_d256.json index 9c317946c12b..93da0ef05758 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256/attention_d256.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/golden/rocke_representative_ir_sha256/attention_d256.json @@ -5,26 +5,26 @@ "cases": { "attention_d256/gfx942/4warp_gqa": { "arch": "gfx942", - "bytes": 471570, + "bytes": 473577, "family": "attention_d256", - "kernel_name": "rocke_uattn2d_tiled_d256_b16_t32_h16kv2_bf16_wpe2_mw32_regpv_4wgqa", - "sha256": "39944b0c981aae3fc1430cce23b045bd537cd748e59fd66a28cba9452cd91bf4", + "kernel_name": "rocke_uattn2d_tiled_d256_b16_t32_g8_bf16_wpe2_mw32_regpv_4wgqa", + "sha256": "536739a7c30101a04c2d53028d14cef22840040be9188b9c535ca9682fcbcc9b", "status": "ok" }, "attention_d256/gfx942/4warp_gqa_d192": { "arch": "gfx942", - "bytes": 363560, + "bytes": 365085, "family": "attention_d256", - "kernel_name": "rocke_uattn2d_tiled_d192_b16_t32_h16kv2_bf16_wpe2_mw32_regpv_4wgqa", - "sha256": "105b06b922bdb3c7c0f4a1af89b1771e29345e7cc1bdd515da7235bf419f3c6b", + "kernel_name": "rocke_uattn2d_tiled_d192_b16_t32_g8_bf16_wpe2_mw32_regpv_4wgqa", + "sha256": "4d0c91fc786c27a5d5beb4b1afa8e09cb2f16e33863701c327ae4d504364f161", "status": "ok" }, "attention_d256/gfx950/pad_interleave": { "arch": "gfx950", - "bytes": 598203, + "bytes": 534817, "family": "attention_d256", - "kernel_name": "rocke_uattn2d_tiled_d256_b16_t64_h16kv2_bf16_w2_mw32_mfma32_stqk_s1_mask1_hlpv_mlim_ksb_mphsplit_smxil2g4_qdreg_qrev", - "sha256": "cfe4cb11fb891bcdf253ee89ff369ed6eca89389fa284897b9d1d6c613fab184", + "kernel_name": "rocke_uattn2d_tiled_d256_b16_t64_g8_bf16_w2_mw32_mfma32_stqk_s1_mask1_hlpv_mlim_ksb_mphsplit_smxil2g4_qdreg_qrev", + "sha256": "37651d0984c18e6e54de8271c6e9bf515a4d28ddb9ce155f27192d14e6dde076", "status": "ok" } }, @@ -34,26 +34,26 @@ "cases": { "attention_d256/gfx942/4warp_gqa": { "arch": "gfx942", - "bytes": 471577, + "bytes": 473584, "family": "attention_d256", - "kernel_name": "rocke_uattn2d_tiled_d256_b16_t32_h16kv2_bf16_wpe2_mw32_regpv_4wgqa", - "sha256": "81cb5af7ed6b63be55783adbc8c37e270d9e232be4cee0f1a09b1644d5e10e34", + "kernel_name": "rocke_uattn2d_tiled_d256_b16_t32_g8_bf16_wpe2_mw32_regpv_4wgqa", + "sha256": "5f896758edba87ae2e75707d02cbd058db8aa5347554b7d8866b91a9f0ce0617", "status": "ok" }, "attention_d256/gfx942/4warp_gqa_d192": { "arch": "gfx942", - "bytes": 363567, + "bytes": 365092, "family": "attention_d256", - "kernel_name": "rocke_uattn2d_tiled_d192_b16_t32_h16kv2_bf16_wpe2_mw32_regpv_4wgqa", - "sha256": "ff9bfdd9db9c2bb5d331720495434f0a4f9d17d18cd7e20814b8d4b88dff7645", + "kernel_name": "rocke_uattn2d_tiled_d192_b16_t32_g8_bf16_wpe2_mw32_regpv_4wgqa", + "sha256": "6e36e791ee8fe02e3e40cec668325e0eec4aec4d75eb47e26684f28dbcef7cc1", "status": "ok" }, "attention_d256/gfx950/pad_interleave": { "arch": "gfx950", - "bytes": 598303, + "bytes": 534917, "family": "attention_d256", - "kernel_name": "rocke_uattn2d_tiled_d256_b16_t64_h16kv2_bf16_w2_mw32_mfma32_stqk_s1_mask1_hlpv_mlim_ksb_mphsplit_smxil2g4_qdreg_qrev", - "sha256": "b0cab82a07b3a1aae297fafa7d308a485faff18381b53d360d21ff5da4d3ac62", + "kernel_name": "rocke_uattn2d_tiled_d256_b16_t64_g8_bf16_w2_mw32_mfma32_stqk_s1_mask1_hlpv_mlim_ksb_mphsplit_smxil2g4_qdreg_qrev", + "sha256": "9360a69ce33fdf232199b66c4487ae830bb903e144a5b1ec6cebba935243a5a2", "status": "ok" } }, @@ -63,26 +63,26 @@ "cases": { "attention_d256/gfx942/4warp_gqa": { "arch": "gfx942", - "bytes": 471641, + "bytes": 473648, "family": "attention_d256", - "kernel_name": "rocke_uattn2d_tiled_d256_b16_t32_h16kv2_bf16_wpe2_mw32_regpv_4wgqa", - "sha256": "365c38d6e523b041970c06a2aa1fe6aaa4879dbd4b9126afecd6d9ab6df13f98", + "kernel_name": "rocke_uattn2d_tiled_d256_b16_t32_g8_bf16_wpe2_mw32_regpv_4wgqa", + "sha256": "ba474f7020fc90f23170e9016dd51fdde3a22a0d229ad031ee7f37721a2c02c9", "status": "ok" }, "attention_d256/gfx942/4warp_gqa_d192": { "arch": "gfx942", - "bytes": 363631, + "bytes": 365156, "family": "attention_d256", - "kernel_name": "rocke_uattn2d_tiled_d192_b16_t32_h16kv2_bf16_wpe2_mw32_regpv_4wgqa", - "sha256": "01b216f302206495d970142af4a2ae7296765dd60faf44a565b1b9b899954d8a", + "kernel_name": "rocke_uattn2d_tiled_d192_b16_t32_g8_bf16_wpe2_mw32_regpv_4wgqa", + "sha256": "17e850fa54c0cbcd47118d2cc1b6f3129fd3f825c60a0515665f5c51f6b6636e", "status": "ok" }, "attention_d256/gfx950/pad_interleave": { "arch": "gfx950", - "bytes": 598367, + "bytes": 534981, "family": "attention_d256", - "kernel_name": "rocke_uattn2d_tiled_d256_b16_t64_h16kv2_bf16_w2_mw32_mfma32_stqk_s1_mask1_hlpv_mlim_ksb_mphsplit_smxil2g4_qdreg_qrev", - "sha256": "074ec36d6193ae0db410de0f2b157ad4323a592a6bc9f2f28d70384808ca5abe", + "kernel_name": "rocke_uattn2d_tiled_d256_b16_t64_g8_bf16_w2_mw32_mfma32_stqk_s1_mask1_hlpv_mlim_ksb_mphsplit_smxil2g4_qdreg_qrev", + "sha256": "cac9740f074aaed79d86db8184a52ab16b9950600dc43dd631cc3e0f5667f603", "status": "ok" } }, @@ -92,26 +92,26 @@ "cases": { "attention_d256/gfx942/4warp_gqa": { "arch": "gfx942", - "bytes": 471641, + "bytes": 473648, "family": "attention_d256", - "kernel_name": "rocke_uattn2d_tiled_d256_b16_t32_h16kv2_bf16_wpe2_mw32_regpv_4wgqa", - "sha256": "365c38d6e523b041970c06a2aa1fe6aaa4879dbd4b9126afecd6d9ab6df13f98", + "kernel_name": "rocke_uattn2d_tiled_d256_b16_t32_g8_bf16_wpe2_mw32_regpv_4wgqa", + "sha256": "ba474f7020fc90f23170e9016dd51fdde3a22a0d229ad031ee7f37721a2c02c9", "status": "ok" }, "attention_d256/gfx942/4warp_gqa_d192": { "arch": "gfx942", - "bytes": 363631, + "bytes": 365156, "family": "attention_d256", - "kernel_name": "rocke_uattn2d_tiled_d192_b16_t32_h16kv2_bf16_wpe2_mw32_regpv_4wgqa", - "sha256": "01b216f302206495d970142af4a2ae7296765dd60faf44a565b1b9b899954d8a", + "kernel_name": "rocke_uattn2d_tiled_d192_b16_t32_g8_bf16_wpe2_mw32_regpv_4wgqa", + "sha256": "17e850fa54c0cbcd47118d2cc1b6f3129fd3f825c60a0515665f5c51f6b6636e", "status": "ok" }, "attention_d256/gfx950/pad_interleave": { "arch": "gfx950", - "bytes": 598367, + "bytes": 534981, "family": "attention_d256", - "kernel_name": "rocke_uattn2d_tiled_d256_b16_t64_h16kv2_bf16_w2_mw32_mfma32_stqk_s1_mask1_hlpv_mlim_ksb_mphsplit_smxil2g4_qdreg_qrev", - "sha256": "074ec36d6193ae0db410de0f2b157ad4323a592a6bc9f2f28d70384808ca5abe", + "kernel_name": "rocke_uattn2d_tiled_d256_b16_t64_g8_bf16_w2_mw32_mfma32_stqk_s1_mask1_hlpv_mlim_ksb_mphsplit_smxil2g4_qdreg_qrev", + "sha256": "cac9740f074aaed79d86db8184a52ab16b9950600dc43dd631cc3e0f5667f603", "status": "ok" } }, diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/golden/llvm_gfx_all.json b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/golden/llvm_gfx_all.json index 09c88c3f4971..afbb12d979c8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/golden/llvm_gfx_all.json +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/differential/golden/llvm_gfx_all.json @@ -18,20 +18,20 @@ "5": "643cb437a319d7be09ef59750c9379e6de4f7458f3f4a6d580bb237009245d78" }, "attention_strided_kv": { - "0": "3db5edf30be9ce97cb3a092984fe922a77809c5e762026249928f658dcfb8f7e", - "1": "6d9013a4914a2815d92778aaca640fe6bc747db8209117e781fb67951abc5f07", - "10": "8ec3555ed2e64c59f0a32847175a1dfe0416d83036b645c29da331e32f640f57", - "11": "4bd0dd8a84612f0e25fe82e1188eea5a23e7951179a5ed266d969a391e994382", - "12": "86accec45af1c586b7594f090cb4750c55a65039a014c4f508e808d9c4226cbb", - "13": "6a0c63a1574445581eb6767c40600b6bffedde472c9704bbc5511855db030775", - "2": "76d7feb82487213ba920e107950d09b999982f3695467c56df38d7e9737dfa28", - "3": "1caedf0196d6b6cb889496cefa61d844373d4db06dc9b98327f6aa44de2f8b48", - "4": "d93e5587c7f3b2432d8248a8639f19ab75416920cb6f2139198bbcddc0603ba8", - "5": "f7576a337c2599cb03991ef9a8e82541a50208c9ef1704b4b3cb63d3933ac7c6", - "6": "c53c3dfe719ab0c2c978eed42b9846f35a947d71f91b2d91cea008f6cf65c90e", - "7": "4d157a7f700be997c772d8eace3d48a1bd98e5a6575297d5ce7547fb672699f6", - "8": "668c6985a3d815d88eeeac3ea798755f50efb34859ced595e81fe6c273fe479a", - "9": "b932c7ec5c473d26f9c5d3364d503eba33b0d916d517114842dff4352274241e" + "0": "c99fe0ffaf340903e7c913700011f4e4e6aea5630ddeca60c24a66916228d9cb", + "1": "7a4040765b46c08f298f74dd3952e8a830aa02d4e397d6d3d37fa38f7cc76e02", + "10": "e15390610f1eda44ef679b865738e82fd9ecd1646ff82f7da5dcd8f158e6cfff", + "11": "88f4619bb13a46d71165b57e6af170f437760aa740d07fa8730e6d097f034d0d", + "12": "8b9fdf9d81f108bb733518efe3f16ff3d3a512b5a735c2038666b5bc98c0a51f", + "13": "e60f0ad39c4f7a93deb98820eb0ff822d9bd687c7c7b6489e22c1a956f863700", + "2": "5b2da99305995476f23c4aa3e274c249fb3dbab830d984fc641a57644d0de8b0", + "3": "2e17e193a646496fb2307f2bb4fe061c18310622743072ead233902b2f2591af", + "4": "9a78878c3b5e7b90231fd8ad907fc629e164c1bfa7265a8af16019ffaa957297", + "5": "b4b7ee5c63360b9231f6f6301326ad35b3877263050c43631a60704339bc77ca", + "6": "382bd4a5175627bbd9090a22346b587ddc3d86cc2b461f6d98b20ea7b9953260", + "7": "136adbde306c73fbbde41916fb93ac4b596cb116b2730727ea0802826580941d", + "8": "e5f2a61ca1e3b775cbac546a6cc531c22a5c978edeae0b0c082b9b05d083b87c", + "9": "ec8c7102236fbae47258f43dcbda0cbc0f77c2c9fe516184ae302ab27a6e59e5" }, "attention_unified": { "0": "659f1b249bf873d6735e1b0f0096f58fe767fe5c30f33581277151aef0c351d2", @@ -1322,203 +1322,209 @@ "99": "ed953c5c5fe0bcae664c039ec8edf2016474863875f151b5e946b9e37fab8044" }, "gfx942_attention_tiled_2d": { - "0": "1b7d7d59a5fe839b98f3a37b3a22d53faa6d4978093ad132c3aa28dba32ea40a", - "1": "f0d31b0def53bd0f9500cf4bdb93604d7d03c2fe3362b295cdbd3cf462fd0f60", - "10": "44821a97c4fea300dc24d1998721e9cf5798aa5612e9b38bcebd96c774a1f75a", - "11": "6efa9e24b9069ce0dcc6092b2a1c118d5ec2b4f65191b77dee24a938b36393fa", - "12": "6227b7f68633bb7fdec851a0d95337283504737a41b880ef1a8ee6d24527c6ae", - "13": "86b8b87bebefdb7322ebf7d0ff097a60a1ec1412d180a4a5cc0c5f8c2a32c07d", - "14": "93fb6779d998c4ea3383902186d2d852da356cceb62400ac7b09702af08554a1", - "15": "17eddc84585316dbf4c0f2dc8898e0d8eb7670e2c413ce06b10b36eab20e9873", - "16": "bdbce6928675f1cf427b512d70f62fd155b5352e3577501bf231113b3e42b853", - "17": "50cf1139e9102f9a34f2364087a1b6a5ce94c21229904095c196ac81c2373abc", - "2": "62b745689ce02651a5de2772c5c32f72084fa8425d15ba87461e1542bd7dcca9", - "3": "f2e2d859a3f4110a51501ee17fd0b908faee5c3d54b47ee5e2263db9157b1f3f", - "4": "944843338385871cf63cb7da57c924989738376783252afc1abddd9b042ea3ec", - "5": "92248d34f8283b85ed2421fe66a3e662425c1e742b325fc3ea3d898ee0057258", - "6": "22a81ba1566cdad5414ef378e6b7fb1f7cdaa01539ab783f1434322b64863e47", - "7": "72082878310a6d733f1b6dc5532e5d2c90e964c3cc361e33ab20adfec2f63887", - "8": "892ee40c3601e34f5c3a7a89ed118547b52240a3644beb0ccfa3d7fe8f759585", - "9": "cf26f783e4c33e58d878e397d914b55e9aa5aaf9e8f94eb7cf51ba693fbed556" + "0": "7f76dbe0d5e41e601888811064883de1a13d712d9ce1f5feae1f1e4c3eb18de0", + "1": "9f090d62d8b0e71bda9da64bcf3484e7d873d7bd19ed7eca3bbdc7affee4bd9f", + "10": "59c756da4ce9911a22ceddd558119d6bb86d4abb506784357ab3dd545b463fab", + "11": "3d1d15049c4da732415f73ce366e31571dfac0ee4e955300fe6f05179a8a5035", + "12": "3279171ba0a9798b9fb9bf5ea910879efe564368af75ec52224ef9c9f44bc5c1", + "13": "e0ee0eda2848076d6baf57e18ff158cdf5f6fca9ba7b5cbd3a7a1358a8571cfe", + "14": "080ecf31fe1f162872e82346ce1356cbc98bf06d5ead424193c5685380e6ee79", + "15": "735c2e3ba2623bce349474890678251a7dcd5e7e784b5b122456ed2f337d0050", + "16": "c508f27798eedc9d66c32dc4a540230a6502e5d0df90a025b57037620eab017c", + "17": "d08079670cea3f281cac6d6cdb26e35be94068953e00fff9008159fe6970fd6c", + "2": "ee4683453afbbc49e5ae11714bccf200fada5e75884a59405cc6676af6e57b81", + "3": "b364ec606939a153220377511d8bb21345bf28f998a41c1f8e3b0d96cac6c4a8", + "4": "66849d43bf1485879bcf1c5f173d72862cca98a1985d0d2d0ef824fba8c5cbe6", + "5": "82ddf00b3ed1d5d461102d16bb0f48e64ba746a67c9112b3125c91e9b0de2234", + "6": "db6606aa71c7cb3dc3749a624b6d8d7c3152feca82ea2750b14c6b8019f53e7c", + "7": "1563f4f2baf3a37d2b779851f82704b28653b8e5eae1157585a642b509f3071d", + "8": "a72a1e70fdedb6afe9bf216901cfb0a6a047fe057c68c2b842e327cfc58f489c", + "9": "40f51920afbc43036c2e44d5649530c22b4a10b86eac40139a2413607226aeee" }, "gfx942_attention_tiled_2d_stress": { - "0": "1d829415b80c68395396da5cefbe85143cc96a841c2cd53a7dfdfddae8382dbe", - "1": "ea4d34355709d176e8b3bd3d3e1df5f26a4533719a60326ad1eb6d98dc6c5239", - "10": "6955af0c8ea229368581101ede8787f024a472749cd9a0326499aacf2bb7fe13", - "11": "0e70f11666d3ca03adc072e9850b9265ffc67fc9f9117043eacd1eb8a06e3826", - "12": "7ecd687934e05a1d1bd3bcec510e445296ef8e12fee7ba3e2d88bfe0d93111b2", - "13": "74a1293e2422f95a2b0fc4b9470882fbd7839277def2eac39bb0e47b2c4d6814", - "14": "b4b45ee606bbd37acd60c633c8fa73ba2016af712fc861cb10b8c6d9bafaa9e0", - "15": "32add948d233b5b437516be6cbacd486c510256514a151cf22245db1add40c60", - "16": "c53b2022d56ef86d062c8a2fa4ae8171cf98d979099fd38954056d76e9ef2f51", - "17": "bbb4456bd1a56e33619c008df7f9dd798e4eec235abc94cddf9c8ca644d1ab24", - "18": "f8ab73295e6e28702fd69d8b4cc5894a7cb27eea350b75d984897bfd67a3721e", - "19": "ba56fc5ba604dd92d20a5a9897913736320c9b5a99275e4b6e37a1ed5e596841", - "2": "47d4e1b7b06a2f37f25ff4ad4c488948e049dbf280ea08f484240f3f536ea7da", - "20": "01b119654d2b72b40d81829bbaf4dbd6330cf3bb9d947ff1bea1158a1d4dd3bf", - "21": "218d5e390f9310773808d5bc8f7834a9eaec2c27cacc47e08fc5a5e08e771660", - "22": "d2419dae753cd9645d8986be377efde79f896b9407cc64428f89caea28e44014", - "23": "606d36422099e3516f96cebe2d0d45ce460e7ec1c061ef8090db7cf36bddcf2a", - "24": "955ed76466427bfbecf5f8521f88d612305f91d135b6b4941981cfbbf35e2d93", - "25": "267022620d6d3431ba9d35678823bc0b199e7b78acd19f6d3ce83304dba78003", - "26": "a6a525bd72b7e2504962aaf33ab537bbf7b97dfce8ee0c3630ad72ed87224f92", - "27": "93d0b1779c27f74ce5263f8947f9e9862a8671678c16425f8ce7601028c4e1b1", - "28": "54fa67ef14262613c75e0e61831fd1366c2ddf945589c7b07a152abb0a01cbb9", - "29": "88de6bec14a95faf94d7eab771ab16ef38fb285bc6d99c08ead85e75e701efca", - "3": "8941fba1e3dae3935b9d6774bfd3c123dc2665204fd26417b86f756923fbff0e", - "30": "6952a0f4d95ae9a181ab32875ae101d997617d56faff1cc1bda99747724b3d88", - "31": "b59f6ea9c2370bce209faee471d34058b31167d9cf932d6a5c1fdf4dfb105de5", - "32": "657d32611974806036f0a3170ea5b77070fbd0fa735349de0a6cf3246cfb5e9e", - "33": "681a225bdccccc6f94a2c18a960f64d80bf612128314afd0d7df34cd85a25b40", - "34": "f14a4c2a26c5cec4300cde64fa98b3f9432e8c6314e8f362e7a7fc1b21fc4427", - "35": "90878510d63c5b9c7e00657c0678338e66881dd32723f4f869b6ea0383115a8e", - "36": "4eaf337464f1722268c79b577c6709ea9331ab2ba0926a63fd8c4b067ec9f59b", - "37": "78836c490dc82c5c3369aa86280a52bbc84998e71f4ca729c0334d4d49b97f52", - "38": "3040e109035929161672163b0123befbc826af6575b7e1285f57e5b0bcef66f8", - "39": "872b9947a9f9208c1957f36b78d13540a909d77a7100376e03c13b07de721bb9", - "4": "a52c2cd20f27f242ba0008f2125fd28e7fe9874ebf2e74828f3e30bb95dde98c", - "40": "5268f32651c2de1d27946ded9fd29be9dbe20e94ca9c363f8b3e7ecc60fa9cb1", - "41": "5e823b9c9e47cf12665d89d4dc93a8dd3fd16731ea12be29136555160a2767fb", - "42": "746248abf8c0f4bb52c4ab026e1d0c25bd0e912c99a847e08dd3518ab1f88921", - "43": "5a7deaf7fcfc61c6fd951e4fa6ccf91cba85a1054bd7f1c8f4f7eee0721c6210", - "44": "56c45e1fb9d5442715e6ddf4272c9e48c75f3efef14498f3851c32828e8371f5", - "45": "0c79c3d803b1eee8a9352406cafc9834e854d07acb0fbca44ec21c14bfc647b3", - "46": "285a14868703fe016a6638aa71d4fc1b01b39e6033b1d23604c12c1788c9bbc4", - "47": "b7c4013b9e291d0d5e4c40bbd7dc020c08b7c7c6d38a249648952a6a6590350a", - "48": "ea0f41f63c3b0be9d0feb84d2aaec46942dcd7f6fddb83593b89f1e3f4302dad", - "49": "f3f2a72324825bae0035493475025c2fa635a8b58f364570751a68c268baed48", - "5": "c78cfd4dd80ef4a81ff90b7e1a0286fe0ef3bcca792f660f4a5dac336920b8a9", - "50": "b9131261936deb6b06c01d9a2bf4d7d41b18efbc2182d18afe56b75ab5b85034", - "6": "e1914a84d1e288efe57f9feaff915b634aa4295b7cf11f7150c71aa7252fbd0d", - "7": "f0d31b0def53bd0f9500cf4bdb93604d7d03c2fe3362b295cdbd3cf462fd0f60", - "8": "6b632cb29ff337013f169f1b522f070b39dcba12f4096fd4bd4c58694817751c", - "9": "944843338385871cf63cb7da57c924989738376783252afc1abddd9b042ea3ec" + "0": "ccbd873ef0d321aad3c483a89c260889eea3d07722d2af02b00360dbf5abf451", + "1": "3730734fa4bce03da9d9a85f0d1c4dceba8890fc022db852ee53729e22937280", + "10": "56bf714d2d9d25e1170221deb03facc1ebee8a90c30ebfd0ed5c5bb3791ea97a", + "11": "abf17a5a76bd9aa62a755b13f53a43d69a692c767c94c69ead33d3c637485c77", + "12": "74cbc7b9cddb9f6bca65ce9e9dd834a59243e9489119d8d928d9ea9fbcb79dca", + "13": "ae4cdde88c12393123bed9ea4571175daadd8209327ea2736d80281f6e3ba5c0", + "14": "1e4f681e6c5b5db81838cc64eea543226ad4a1013f84efaa06aea2701e0b08d7", + "15": "3e44dfe85492353dc725cdc67228f0fddf8f601aea8845820646ee162d4d04e1", + "16": "ccbd873ef0d321aad3c483a89c260889eea3d07722d2af02b00360dbf5abf451", + "17": "ccbd873ef0d321aad3c483a89c260889eea3d07722d2af02b00360dbf5abf451", + "18": "ccbd873ef0d321aad3c483a89c260889eea3d07722d2af02b00360dbf5abf451", + "19": "8dff4368d397f6a9e209fb74ab18c6ecb183d1b83d628099ba313247509ee385", + "2": "7f76dbe0d5e41e601888811064883de1a13d712d9ce1f5feae1f1e4c3eb18de0", + "20": "74d73e46de452e86c8feaaa1be020dd556fce4b54e4b57419036a440c960e936", + "22": "4f009d277496d367a65b276f7a4756a514bc555ac1380275c2edc1aefcd072ad", + "23": "6545559600bd6b1f66e0f7847b8a57ec453407892c650f10b5cbc5ce61d4aac3", + "24": "e786f0a81576ff0dd372ed0ff4f44532441fd4fb1fdb3bc174e48e313b4464ae", + "25": "3361dd0af9e957ba113335783db8650a8f289f14d027d2220c542cdd1da33c75", + "26": "3daedfdca1012caaca91f526a3f0d007d0b5f11e6a37b4215848ca019ed2be00", + "27": "07c645ece20c96c166e0c21fc2bced731f1e7ad792edd7bc71a673740d4fd488", + "28": "2e4d445a1eaef9914a04677d692c2d06a780560f4f10a06cc828904fc6baa1a3", + "29": "cbd5aacfc98eb0856429335ace0ae99f7308e5728ae814e16eb3e4990cabb578", + "3": "40fcebb2abdd7315532e051f5f06833ffa3f50f926f0cfa1325dcb5a0a7a6c8d", + "30": "be18ff5ada604f22454249e0bb49afe55a47da478ea84ca0585037dad3d7012f", + "31": "fe8bcb9143145a4203f483ca2b9486e859f3dfd6188d71a80fa446ced4289134", + "32": "ccbd873ef0d321aad3c483a89c260889eea3d07722d2af02b00360dbf5abf451", + "33": "ccbd873ef0d321aad3c483a89c260889eea3d07722d2af02b00360dbf5abf451", + "34": "ccbd873ef0d321aad3c483a89c260889eea3d07722d2af02b00360dbf5abf451", + "35": "2deedb5cf80f13fc21f714f54bde905a52085ccf052359732f14d28d804c01d1", + "36": "a8daf108a0e504983f1b9ab0bc7e66eaebc72738a5cb633c57cf47820fbe7ea9", + "37": "e3a3ad98dc970e686f02e1885221541ffc1328bded6e3ec1aa51291c7ba62177", + "38": "1c8af3dac76a217680c454ec5ece6d7828d1445626639c9b39de2447ae09f1d0", + "39": "0f579d6428e8fdaecd3b6fc26635de323c73aabbc33a5ed6a9f34decbebd6ab9", + "4": "bd9fe2b7060ce31e0aacc7d7f05aeff289aeffbadca5ee96ee7ec0df0b4f4c72", + "40": "87f1ae8d3e4faa6b4e21b289411942e5db573eaa0c81f4677a8fabe440e56113", + "41": "81742f6f635857943bd0109dddd70168ca446a3814f305c0ca75434a03bb7911", + "42": "1acad46aa9c2c2df202df184ee67b9fec871815e93572155ef10b5e5869807c1", + "43": "f7eb3023a22be4fd93da0eb8e582aaa1e54b857d38d2b6340b8e8e397e669b17", + "44": "7f4bcacbfa795016b81589424496dc7ad67c5e22e010932082e0527bbb7dcc70", + "45": "0b3cb650e60faeff342fdfede8ef950301d1de74911e08615f445712b762c076", + "46": "b728ab02e2530b22bfaacfb03e429a9fae48656b463e0afd0c0368972c16aca4", + "47": "91fa1ddce36b536d0ba6618d51e1f03899821267cbbfdcf4d57cd479a1720438", + "48": "7a38bf04acc5d869a20d83a2697d3d1fd8f560e685c9b84c88900154c9eed5c0", + "49": "2cbe9010e264083c269e28f39630f89fdb42cf40f4dd0335897d1388437ef8a2", + "5": "2a8de59571976eb94aa1183a0a234799930bca72d5f10f22e56fc9299761b238", + "50": "59590539bf7577b133aece3eeb269b5bdecc6473f8aae5a05222a319a98d8c9c", + "6": "8125d8f90ff4a8967778619dddd21c61a034058dc5d320308b26167f2ae5e4c9", + "7": "9f090d62d8b0e71bda9da64bcf3484e7d873d7bd19ed7eca3bbdc7affee4bd9f", + "8": "bf1f3d1a410c6c5ac67b5cebb6afe096b8f2e3e0c7ac755c256ff44e9531ff06", + "9": "66849d43bf1485879bcf1c5f173d72862cca98a1985d0d2d0ef824fba8c5cbe6" }, "gfx942_attention_tiled_3d": { - "0": "82c3c72998cd823e2637f88b0a1c72f348eb967f0614731fb61c69957d00c4dd", - "1": "7e58c5e2e6bc2abf17c4b8c7beeb2493f06f659921daca2b03ad7ec7cc789b70", - "10": "e68e591dddb5a3597a3a94a8dd5198c7a26656ae7ec0f7b85930a5d290932709", - "11": "db36573107c98e859fb58951a102dfe2637e9cb91780768ef987e313d13f6fca", - "12": "16e0eac8fe2dc90647b16810453eaec2448c15d5d35d87d56eea7958b4300a95", - "13": "497672e27a6f1151ea11f56b69d770b8f7fad4d3ad497c30c2a9d4bb36efa5c4", - "14": "7017aad68c2f9bfbb7bc8b4155486d3beb26e708a69008ce85529b1d302b7c85", - "15": "74af62c3d1e130aa3be256ccd052038cf4979b0bfaebacb9a68203f92ba1d3aa", - "16": "13a61b43fa518268e0a66960b23eab2419f85915dcf852cdb4bace1bccf16e81", - "17": "77d0e1ab4cabb014c931b54c7bdc0aedadf53cb99174979686b4018e056a5a9a", - "18": "48d51c38964686d05421a90ba7ef9778fc1f3cb53ba200337ee66f761053bd04", - "19": "520a827d9968e9fe8eea5be9898b4730e6bcfb60573818b86fbaa69cae7706c4", - "2": "82cdca8f08ee9612cf5ea4f3ac729e826d283615ce0038eda2ce983c87bafd93", - "20": "faab3a2be33001ad480303f9ba04414e564ec4e98ebfc9f9a32bbc2b1d7733e1", - "21": "82c3c72998cd823e2637f88b0a1c72f348eb967f0614731fb61c69957d00c4dd", - "22": "619c8831f09e24ec84c6e11a4be906ba4b9a1ca485426056b0d5bebbd8b861d5", - "3": "0bc5cf7cbe950277ca54549cb078916678e75b4db7a3880e22328478d89ffb3d", - "4": "168b4be8598882c4beb38e84029c71293ab2f86538c4f8290c8c25b76a9634ef", - "5": "a7494ff8169f6e0c7788dfd9321e6878b87226448c14d838d1fd7a7135d05b6c", - "6": "333e7c93b7dbbb711e73bd6fd503e0c7d6bda0615221aaf2bf863ffc41b6765d", - "7": "579555ecebbee98e80d4b346cb21be3107eef939c2bd9edac5d934094f8605c7", - "8": "edcdbfb38f33f419b6cb5e35f30de6b6ba5cb0c15a163330966781f66784c342", - "9": "8101c7633b59e2202adbb913a3f9544eed1fe0003f67aa32de54f42642137c5d" + "0": "aa350c023bc7703af40f6f8b7ad7c5357b52be804fd2da3cb799809a5f528f71", + "1": "c2dd79a1b240714016fc2c91f953d7084e5ae3d6669780d0adbc52cb95578b30", + "10": "f90b66ee3e51ea2b98c85f439efe8a8ee907480d3f7a6743333a9bf8bc2d4f49", + "11": "c4c2bb8b050f8f793e5528d219fdc38d4d12f6da150499a84b304b472cf90ba3", + "12": "a1de2dcfce3a7b5b1e7ee71e17a71374cc6d83eb00c558af4b039e2fd193aeee", + "13": "72e27731f1fe5c334069b1a9e78784f7836c4e2298ad3b38b785de6033c01317", + "14": "d84c36ee99f9137188264361427cfdf85f5b602a2ad7914399ce4e19aff04a2b", + "15": "9d24d520c674c8e127276e0ef1dd37343beb6ba5534d3759d47b3a1b0a3151f9", + "16": "ef3452234835a8386dae0a4ca8d60ab160f0489d6d2748ba950990ebaf6d02b6", + "17": "236863bc3a96bb609c1c441894e8f208e1318ff5ee231d30929e9b611008c89d", + "18": "8fb22577fd266abe0dae13331f99846709e188d4a51816bbfc6f2909046acc11", + "19": "5bf14d4f62ce98e13465cdd1be0ace6c393121f5a045a84e87698e26338e24bb", + "2": "0aafc6fb66c1c4b9349a4ecc265ae0d70cef0697176488f9e365f3ad9ad72a1d", + "20": "0460dde695f99f1642702be6281ba0596c838c969eea4ff7f244e5c0c16b008c", + "21": "aa350c023bc7703af40f6f8b7ad7c5357b52be804fd2da3cb799809a5f528f71", + "22": "82311e7778a2f7e7c9b3beff0dd64a6bd2352ef3aa6ef57ebf57263950f89c5a", + "3": "25b66dc34eaf1122e985b4cdbe27e6ca17640a7714e7a75f1cd85264a31fa56c", + "4": "0b84bb6a3a6de5450589c36cc549ff01d325b5cee13e65b96498418a7cafc057", + "5": "0efd5e6e4f1590492f54ccdca10017498c64d0a7afccd573fda12cff8e1b038f", + "6": "3712baf35493e3d35d23b0510604b79599841682213c6b0f820e77ad0132032c", + "7": "5a489ec10027f3e4f4895b71c603be723e09cf753c70013370297cab462a2217", + "8": "1bddc9a616459227476f58036f7c33b38d2cda247776a193e37dc1ddcc20b62b", + "9": "d1dc34cb8dfca4f7fadafcd65d226b540ab47e634f24931abd0a802f02042dd5" }, "gfx950_attention_tiled_2d": { - "0": "0f9f08f28c3776a62f0ec66ce3e4b6978b2fe76be3e5562b3a2efb4e81c9d9d3", - "1": "4cb2892dc12a08f693ee470d94807d5110db985b5624e35cb9c82cc400f70135", - "10": "c78fe69dd296a690ffaccb0a503825ca578d28625e555769acd4bd349f552ea6", - "11": "c95a740de5a3cf97d064a79837ddccc74afdda0b540f99c5dc8be5350f856ed0", - "12": "8df00cfb0c849a3d2ed29820d6415fce4201f4125e196892cf34f6568c2221b8", - "13": "e474a8924a0ddb4c2ea580f03f5bee4a70ac43b51d262acb3b9399ecdb27e7e3", - "14": "4509e4f89bc136af028606fdac5ccf7601dcc90714fa1c7451e7ff8450fac8a5", - "15": "97eb9ac0f6bf04514196c8a97f02a9cebca6b83713586233ed4a1391cf27063d", - "16": "84043fc395af029a3d90334cf74d1a947a949c194fbecdea63eae3f59a917de7", - "17": "d517be0efb1c2dee8b74806f41dc611e087d514bf938511b158630afd122d3e2", - "18": "2a1db744f2dfb7cb87c0504c5e4cb50cb81a6ca65a21041ca6ac58ceee0951f0", - "19": "8d49d8ca679a5c2a2aca7179b41e8bcfcef578d4af842b0f6df3e60fdaa2e0a9", - "2": "f4765c826b72b85a63a1ed9395648761803ce4b19b4fc7d0dd577c99a541e82e", - "20": "c5ab21cce72b78a9e500d6240c3b115c268374c9a94031d06be124fba7659509", - "21": "cd4ca6d882ecdb4d821a18c209ffd42016b021e4d0e164075a1850da021dcfc0", - "22": "1a12aeb231270e7475f5dbebfa81800439201e2584f23036d2331ef70d1694e4", - "23": "28d1ef438b5521cbd1befc4e981376343531acb039d70a04433902ad1a0ca90a", - "24": "68c34db231ce4f97f54803b5b7fa16a39627f781c2e989b3ad51643a3d011810", - "25": "7cc1d6d21cb99616dba150d9969111d349ce9dd2a5c0e94726706572039075a8", - "26": "c19a2c3a0e63dcfcc672988bdf38303240db84f88e3b857984e99d28434f048d", - "27": "a9111e09f621f0464213e92aca41f9506b7d2a70b0def21842bbd0e50de041f2", - "28": "19210948da37dafe2714878f12d9d57bcd6322bff2cec0ba75030723e3a85b3d", - "29": "355fed43577bf75b9b39b83434d7e3229b083d4ef1ae7f8f4b34df51149a5da7", - "3": "b30b8e4d24ff3eb14957bd80fdb203064ce4effd46e511b4f94f19612c36c2da", - "30": "21ff0eb92867d34cbe8a183802fe44d78e834157219c1629fd37fde981279d3a", - "31": "25c1094ca28f8e606f6a504fc9617aa44e66230b35b5cd27edc0dc2f9ab36b7a", - "32": "ebbda7be4b914535c7a94bcf72b0159af3728ae2bac4a2280265723fa2825065", - "33": "4b35def3f75cc99f478c4c7b812dd84bcff207c4786bfb6444e6947a57b1c403", - "34": "3e3dd68fd289e0805c26d5d029447b4235303f57f6f35c9c15d730c773e0f1b6", - "35": "6c8c45bf267b0cc7c1d91e6c6bd17966971bb7e99a993f07bfc8bd3aa252f3e4", - "36": "18d4a651f09f78f5103e5974a962001921071a6213af2858a0773a84fa653369", - "37": "4c52c8b9d39e60210c6608da8b05f0899950829af13d1ea7b3004b010ed29582", - "38": "8248216f47791abec33969387a0458935e2f9dd45c07784213a1a61a10066d5d", - "39": "587edbf271ad2c976a94825f5ad9bb5a27957b00772cee7a32a363fe7ba2f9e5", - "4": "f06d81b89f5d62e17bbe848ae3d13ea8e90901bc4d10d682e32c0ab8cd5e8520", - "40": "bbabef0bb9ed4e76ec08688a77fe40845806c11556d750584f7848fb46a6903b", - "41": "60baad4cc6105f5a2ea2348cfd1aa813b58a9679ec9fa5f8dcf80ec1d05f3a3b", - "42": "5fcf573b456a2b6970b05e15bfb4a8a809e4561cfb84f2748480312d69b6abb6", - "43": "38bb6009175a49916481548bb8805c61fe8bdde19b6661d10e2e41409f08d115", - "44": "bd63725205897e6084f1fff1d719c0583cb5a43649f136f309afd957d7c7ef2b", - "45": "e9b5d70aab7daf94429801a7fa39247770fbafa98168d19e28cb2daddd34ef7a", - "46": "8bd0982f125e6f6f86e2a9450d4364cf8340b77f24ff7f27d456a5ddffb66509", - "47": "512f8ed1f6a098d59230080feeaf8e1844b388da69fe16ee43ed4095e439f1dc", - "48": "be5461aa1acee6cab3f0ca65e303cf43b807afe1a639719e4919ea8b1f20de35", - "49": "419911e066bc9c1277e59067c07807c49f111ef976d7107d2185f4995ea381be", - "5": "5fb0397f7d0667a6c5b7ddd1801883065b30c413b9af61c022c57589f66ef723", - "50": "d98f1387169a925a4858f6107cc8ad1c438cd9d04b688a2d9338e1922845f4b6", - "51": "f1bc2228bcce53d64e378762e7eb62bc9323bbaaa2e9822821d24288fa0f52b0", - "52": "ad584366d355ebc019f485cd3839e563217d35d0dc0e428c4e8fd9ab7a0d3dbb", - "53": "4b93701f5796dd08db07e5a5fbd07743e8479aba4daa7e28b2202c5240168f5c", - "54": "7f68b1e7bf8a25b1e104a1b8630a7f01c258a2a820dc81699c048e855bb9a561", - "55": "4f98487dabc2cdd8ceaed90033061954b8a00ac7efcd36aaabeeb1ece3e2b4e9", - "56": "ea042ec27670e324a6f7288384d476d7bf7365f66395e56cba36bc65644a6e6a", - "57": "350944dae19412e96e1653837079595e5030ae6a36936f495775408b35268e15", - "58": "91fca5b2d00ffd6822630e50a6a674678d87ea1b35095fa5a277593956ebeb4f", - "59": "ea4556d340ce959051d342eaa981ba15099dc1bfaad6e28666a36450e0dda5e0", - "6": "5bead5813cbbe10a384d6e04fd50abc397ef91d6b6c14aa4c7dc3f7831c0df34", - "60": "c1bbed73c58ea00ee021a2f9668abee7a7240aa8aa4a0a16c0b4cb1c540abe19", - "61": "6b9a1c2d6fd249dce2a15765149966ca576bebc58bffae91b315a8bd8bfc4e48", - "62": "4545de951e607ad98a43162b8c4eeeab1a1a250a56164956641c27b13c7149f7", - "63": "debd10ac4a05b4f11b53920ef661edf35733734c41cbcb5a812f6f97ca208b6f", - "64": "dc5d14cafdf03825f97769fb812e0e405b978c77b99acc1d1b65c9e576ff42ee", - "65": "718e81bcd09bf25b0693a4afcc6dadb399b954bf0ce06e28aaf58648e62db238", - "66": "2a9e788d50b5deadc1d38c7c0580b2b0821f0e5a591c5ff37679574ebb48f078", - "67": "d686f0027e63fc1d20d78c7e420a6d8312c53580199689f372cb8d0a7c40111f", - "7": "d1056122b85e84d5b861be3e631221c852f3c7cfbafb5382dca9fe839eafce85", - "8": "555acbdb624c8128e10d5288bc12ea404924ea5f4a869a058309627ebdcbb7b8", - "9": "acde5e329c2e56550f55a2dc629ae32e5cf0de51d521204a3605a23840854b8b" + "0": "80d97425430e6fade9622a7b10478a75cc95a68888dcf71904c3ce5018ff7699", + "1": "dc51d43d731188e80596f64c220caaf628cc91ea146012a5b491653746a446c5", + "10": "b4097119ce01cd07abd3ac16a57b582c69900a9c721d063cf19bbe383acfc5dd", + "11": "8215db3e27d5561ecf8e70aa9a519547d1364fa694976b2b84975d9dba7883a4", + "12": "efa9d48a9aad07d8c2f681cb01fee5d1571491764ca9dfe86753773ab75e8722", + "13": "e0b3543820b7cad9181d4bace94c6afc2289f4c9f1a9a6f73928b6364e0759c8", + "14": "6230f1038c63d43f5dafd9fe229c26356738ed78131379a9eaff6937a702560a", + "15": "afb6fb5d72771de720ba0ece82128b1875b0e9983287992f3b2cf8b4d0c9f3e6", + "16": "a75a285a5678240dd86952373a7854e17c803c5020e7e677864f35c895faedf5", + "17": "91eee67912b9f66cb114b3fb43b3274252b5a8861ef9b09dbb816d60a3e33c27", + "18": "1edb90cc4f0ebe0ceb565c750c1d95eaea41b1345f57e3ee1f255b6677c5d8db", + "19": "882a113117994862522f96c8b41eb33867e560e1e7ec03697aaf05d676619429", + "2": "e4d394fd1e8d1ee708e831690fbe7b0cabbf252de155e0216668373498e8258b", + "20": "324e5d29a1d5ce7d14cfdefc5629e04f262a5d80cfa483927fbda2439730d19d", + "21": "d2588b9ea7a1daa844f457a7447dff95cea53fb07638f37815da94ff7dadcdf1", + "22": "58350ac251cecc1b83f470e4220b5845a985fac632e7c52648df9c9abf6e3bf7", + "23": "1f3df7ecf110ba262f2b05f98c3b8d8498c141b599fbfa89080d47ec552729cb", + "24": "c3d80777a4b20df05f58a0676781c1698f574b655de90cc9b03e0b3675a601cb", + "25": "8215db3e27d5561ecf8e70aa9a519547d1364fa694976b2b84975d9dba7883a4", + "26": "8215db3e27d5561ecf8e70aa9a519547d1364fa694976b2b84975d9dba7883a4", + "27": "b32428f69eda9880b3ef2f9b60bff8395a8dbcdd55b0ba673042498f119ea5ec", + "28": "8b0d9eca39cb974546e2dc540816036aa0fdb0737e2fa7f3d661d70039b17155", + "29": "8ed38761de16d378959de57b27bc0a2de70af51abff283af315d4e1d143faa98", + "3": "6efa98bbaf06b0f553a2c43eda1a4533c0a4ebe7dfcdd1d66ced634eef6cc093", + "30": "b0f4153e89ce390e50175889e15a53d0e9fe61ea977b07ce2531a96f5720c5ca", + "31": "b461c77c436d0646dddd07e29a6b84f0b93eab72ac894fa60f613fbdbe75bb53", + "32": "6464e2d4024fe25dd9fdb6b599bed2a5b16931810360fcf6a17cd73f62ea6efc", + "33": "611bac889bfeaebb7b55c31f72be4b2558e2f7f40b84b4b5e48931936568bb51", + "34": "2bff7da93e3ec961621dda1b5e24e4fbf6a4d4cce899dd853f6e86a1a5eef002", + "35": "88bffdfa2a668ed2e1f24feac3d3e8c0698f4981c2c426c2693007f76cb87242", + "36": "0afe5d5bc26a16105c8928f2f159dee554cb7ea3f2adc9abf8db7a42875822d2", + "37": "d9fddb5ff4ba53a0d862eef023cdad24898ed468bd627aab85932df131441280", + "38": "3e385fe9245c08fc7178bf60fe4bbb96010185ae7120db64c7c5ad92ba0e5045", + "39": "a14cbeba1a7da0bb1b4a228de3776d7a3da1ecf3ea916700198d6d4a3fa2a899", + "4": "5c6525e14f18546e2467d6aa2772d99a47bbd0c5226500818e5aa7971e9b98fc", + "40": "69ffe9036e5e5395b07cdab9263f5dac73472b4eb1c861c840fc5c5ef70c400b", + "41": "a44df0a9b859a8f0bda2f9a3a501e508ddb31ab1108a9321d23dad213d5e18ad", + "42": "57b448d2eafd075ae692fa10c315ec9bbd13907354d78652a5aaaa2048e04709", + "43": "482b7c36a6ca558e358978874565eb2542128411d70a2a1cd39694d82f0ff183", + "44": "8c9565c83d5e70b61219daa4836aa326d9d5afec3954258f23651afb5a499d38", + "45": "b87a901031467788019cf8e3ca4aa53dd8bd4c73d3647b62355821df1092ce32", + "46": "28eced4b4c9c54cda51ebfbd15ba12ba9123b68d40a269f9fb9f5dbd31937cd9", + "47": "5cac642b65239b7d14b995f47228f5ca53feca1e48c6e6cf23ab397345bc20ea", + "48": "5896c6d5a55b57041e4d03caa67c2d270eccc1f3cce05a863335e66afce9488e", + "49": "be2b04f10016c62e04d3b0eed3fe966c7658eda91b57b63e857ce8cbac9ac905", + "5": "8215db3e27d5561ecf8e70aa9a519547d1364fa694976b2b84975d9dba7883a4", + "50": "fe7958d3b263cd526f84f30acfb1d884aca5bd1fefd03eacd32f3ea49f2ac8f2", + "51": "f34ddbdd3a44f177fd51c7b5cea033e317d7f3a14d4834b9d97c10a77ec344a6", + "52": "3d918a62f106986362029ad0a25cb7de840c5980e048df8446462e8df0d5477e", + "53": "a9c61c475febf2e5c7edee61724125617b62843fd2da86a0730d74a232dab8eb", + "54": "1261c2e45aece3b296b39f8fe9212e76d26c9e774479791e718b724b2e4483a8", + "55": "4f6f32ebf3d1cf17ed1c34ca1b4c30abf611bf03e5072a1446adb1d8c4903f2c", + "56": "0217021c22e3ce8ed2f45c2808c8a98c6fc34700e3c270a4de44c19c07c0020c", + "57": "87fea953b7f5652e0e157054f200a70f435742c3166269ade25f210632b3c632", + "58": "7999a90075d8919d7a33fe287d5352ec2c0dbc78ba725ac3bfa9a100be5821fb", + "59": "21d3e28c8026d3a79be3196bb912787a21706237945cab9c264e1003094de966", + "6": "0264972169433b1c420b0e04c6e9ad7271bfe1d467adade6dff301b713774c16", + "60": "9f03f01c81b8e266c8d66293e06de25089f65a4f330f7cffa5866fd56ef771b8", + "61": "b7246d2381559cfc6bccc594b9e0fa45ea660a9f37e3cda70082c07faafc42e0", + "62": "183042c39ec254484f870569f3783712e50f9afb4840076f9801a57b785f80bf", + "63": "1ca17159496ade8a1cf30c9e532610d61409353ac64d088f72df64de2640e0e3", + "64": "756ea1d199f01c23b2c1cdd8fb198b09d1476cf3668623a48b3491de10182b6d", + "65": "2df99520f6c3d7eae0dce2ccfcafb563a63951ff22fed87930dadf1bf182a45f", + "66": "d003109a7a6aed69c7141e36e720efe1057a8328fac34d1b3d055d1f9d43ad73", + "67": "027b38d8b15b865da9e9608a71724bbb85dd3bd1c3e0f9c61237cac7b40ea4f6", + "68": "5c0d18377ef91002826c50aef5ccb85024bc77044cb50c8d40f07b173b9c3a43", + "69": "e2c2040c8e161845722093b62e9d173e79947e4f4a22e40d212149fc9e126bc7", + "7": "1067ddfadbdb481e9623ec4389070eca0a385d2968de4d5218cdaf62f68481f0", + "70": "6b98a1c3fe47f3794abb2eac2c2be60dddd0030b9247848df12867450260a76b", + "71": "f072b9b61cc857838d5cddd76a6b9796a00c613392c172c6e837a20f76969019", + "8": "f4f0267af6f85a06591533f53a3cc74339723fa2e33c5f68e5fcd1b1ac4a6520", + "9": "3c0bbb4435538d794f348fb385377e07b8274d96162257c6117e15e995afdf25" }, "gfx950_attention_tiled_2d_fastkv_regp": { - "0": "c8729a014eceb8dd3a680705b81373665947047a29cc347a32e7413fc905068c", - "1": "915a81433a3abf54daa7a15dd58efd25b74752918a46b0dc55456f2bc7f4dcc5", - "2": "d99a2e6b39ea9d9b463d9516ec55ec2ef8a0d4f1204c06517d4a8389682ddc20", - "3": "1f4d7605f2cdcc4a94ca60aad0b10a45e66b677bb862554cba0d0dc45334d901", - "4": "4b2f221c41bea34984f2156fb611158ea1c2fb4289e7ac4f44ca7b91ef696d1f", - "5": "1f480e98c92293827d4511c82d9b79baf45ab4df57bda8840a46fc64801ba28f", - "6": "093eea2640fa1b5b359284007f678732cae2da7b83fae90a772977b61616b8c4" + "0": "d21a86dc7cd4b574b7c58faad7dff5e113e6ad5fd5739abed3d049e27de730fa", + "1": "bf6fc9c9b6c748d12ed22f885612c536640caa86cef24d869d2aeb60c3fea860", + "2": "ec3c0c4b32db6bcd1335199a48022b9cb62854ce643b38cef3b186b037f8e5b7", + "3": "80a6629009f1f455792bee8341ff820ec097543003d44ea5f82f49cea9cacfad", + "4": "6c01f7a5e85997b4200e97e5aa8405556a8f460ba28f490e36c0ecc9ce8f2349", + "5": "e5962a468d4963d5787f37b472f12deb313da4aa058d82595362307b1460f8ca", + "6": "65ee1f9774fc3db87771ab498bb6e51add76a74553e098b42b64ba509565d327", + "7": "74dbde555196dbd24287f69f7a22e42cc5a6a41f77b2b03a0795b78837835063", + "8": "20db41241ef0e16d315eba50a809bf0d1c66286b45d8cedeea3a6dd3a12dd1c3", + "9": "54e040e00babf6bb603410fe782a86b7ea3fb803458ab880941aed20cf72e60a" }, "gfx950_attention_tiled_3d": { - "0": "63c8cefe4601631d737cc00b2a88c787f723f2ba33cab0223da78d809c6c428c", - "1": "8a239b390bcd3c552c2e7f7bb9012c5b0ca292f86468d7bfda242bda1eb391fa", - "10": "27609aa7e76bf0f6c4ca2c54ca743d097b1f673e214c6569564021e8b96f12ba", - "11": "76edcfea215c922cb47cd15263b4400b4615ebfc9f8a06137c34c23ae409f675", - "12": "39ad47d2eca33acfecbc757d8a64de44c32bfbb1bcc5f8972bcd77baff135461", - "13": "7fdb55c05de4dccfeb8f1b15ae41c25ebf399b98f7b8b1c3896a50478d8cadff", - "14": "a86b7bf57b351c35c60e17f70a593e3036bc35bae025454bdc1589840a1645ac", - "15": "77cbc7e611c9046832a959c50392299f8f007f65024a66d6fa6fd9d5ce3bef36", - "16": "ac768048c589cb26f18c8ac4a0ac29ad00f109c591239fd601ca1803eaa458c6", - "17": "6b5481c778882a5ee6110d0cac7e764c91d9f0028f95cf91864502ef91625d86", - "18": "2cee2330772fb5d3b34aee193719650d5419f1e47c46bb4a504eeec2587df6bb", - "19": "63c8cefe4601631d737cc00b2a88c787f723f2ba33cab0223da78d809c6c428c", - "2": "7ca8ac489d8db2ad713d0c94d5e2b5814a38e1df54d82105e8ba1232ed6d1115", - "3": "030fa0f8de882d8ecf675814aeed194d744c6642345c2e1f87a736f386a5ff7b", - "4": "206a5d7824acf278b093b00d7b26723a9b2e6815b66e097d64296c4df3518ec9", - "5": "2f141c4cb4050ea9408f0597541b1427dc73c4a000b08a03e8c9013633fc85d9", - "6": "e9059831e394df815a98157611853cce5f53641781c5da8202dd05550956ac52", - "7": "96e48c0a6a6cc1464a40bfb01aa61a205a37e5dc446125addc3737f3e1e71eaf", - "8": "ed364897692af264f57616e281ced0602de18fccc5b0c9c788f590a8f7e3e5e8", - "9": "a2c55c19220b6d907582f436785f1495c1ab5dab4549c3f3ff6e45f34ac7f6c9" + "0": "794ef52dabc1565190db106078194ea67db68870c0f19e229148f6418f4c2c8c", + "1": "b93369f85eab79d0dd554e8d9c3296b51c2cc460a4aca45b17141a6657793a11", + "10": "7f2a38dcc59a132fe20a9770b20e806659f6a63ee2794edc1712f8ecbc6c9581", + "11": "39cfa97d6cfb65524952e5c56bffb65e970d71ed464d71c21082c8bf8b3cd151", + "12": "2d30096a2f0f39c5556acd2a8a9ec014369ea9e9fba758afa2b0e9ae70e2d4c6", + "13": "c1a45fe713393b43e90642c66c4c6ed045ffd9c854b1c006954f6724613f356b", + "14": "1bb245d5a9c95809a84a423b4f2502479c0ac60f7f2f496cfbdcba06d4f18c69", + "15": "f9d045dd07f039f52374cb96c4e67eee5d640dbbf5debce34b2d56d7adc9f060", + "16": "d863ed32dbe8bfa8fdeabe04f41b4ccc396ccb78ad5832a4a14c993171b527ec", + "17": "b9553cb69c34d5b06938440133a2efc247fdc6fe85dd3a1920e4cb8d339b13ac", + "18": "51e2af096a6be4086d522275d72349a1c00d9fef23544fed16ab026d44ed6e31", + "19": "794ef52dabc1565190db106078194ea67db68870c0f19e229148f6418f4c2c8c", + "2": "5e8da8b6d8d6a96d6f5eb1e74b05f449a69d35bca1dabe1ad3f848839e433085", + "3": "e68b749c715ce0e26d17b3dc8b356639764eaffcd64e7a0b93c5878c1134928f", + "4": "8e6255d95fca99ac69a29158860ddff1b38c405be43ae8cc1b7979aca4736ba0", + "5": "8641b5c679a1b04b9aa07bec3ef984c3c80db2702cb87806c6f0ce74dfc5cc98", + "6": "f17b42418b62a82c894431dff094caf6da7e03e57f21c8762b22e474a220d099", + "7": "a6a9a2c2cec53389bbb153f2518c44f230344157b6f5d6d4023995a36900a16e", + "8": "77862d101a9fb155ce10e340ff9de1755e4607c6d471a5fc8b0e87fdabd0abc3", + "9": "40d658bd39ef0ecdb80772d43aacfb64b402173ca07d2b56989e6c9613f4ea3b" }, "gfx950_block_scaled_gemm": { "0": "2a9ffaa380e90d74c3ae5e89b280ee5cbec54917e940b4ebc05541adcdd7ac9c", @@ -1949,20 +1955,20 @@ "5": "9197a3464b35817737bfcf6bbcfd45d3dbbd6b3b441271df691b969126adeff6" }, "attention_strided_kv": { - "0": "1626048eeaface4bf6eb73c5bc0c81a653da20fc18ca502898a55e6111d13fb2", - "1": "a9d06552869ca1485cb3a7b8a362d2a448fb8d55220f222d50d4a54ded0a2488", - "10": "05d6f2c670f9c66f59974f7f2715eed844799ac869e2765d3d1f691bc417d2da", - "11": "68530d1c8c9c9a12300743809054235d88e2a1698f811bbd86a531eb47113d40", - "12": "fa6ae13282f4e5748eae40227d1355529e793d2306f210136a498b6621fcff96", - "13": "d7982e56673824863076878c3461209f8557cfa8507b102071eae22d712b6414", - "2": "8c0cb312ca581170ce8e387a94a1b77ee52b4c7fc233aa5b15f65f0c707eea8b", - "3": "272f3227c873b525dc8e48c4f024f7885de375178e20f4c4d6686b4724d82c0b", - "4": "b25f6bfc84f5db8903883a85671f13b109256697e33c67877686b4d90ac07606", - "5": "bde0dfb3994fb2a89081fa05809323059111a5e7d4df55ed8859ec2f6b0cd930", - "6": "db68f7a59b6a018e310c3d86d065432fbc4a7201f3d914114dec2d58d3200a19", - "7": "e15ef2766af1f7f39f1e2e1e6f1602e17ed832ed960fee0fc87c54447742c45f", - "8": "15c5cc154e40e4e5f099d1bf6e476e283a70c450824db3ebb9df927b2f7aa2e0", - "9": "7866c9eb6d414e0edf3678a477c387e2742057eed6a96616544f29057d609d0a" + "0": "8cfb54512441a845a2ae325065291e39e5367be96f320596e8622d7e308ec610", + "1": "b1cee01d468e92ecb5b8571323ffd897bff01c7d2e42dc8ba50814387f908a63", + "10": "1238b48be61bb4a6c83e3581ed269a5de59b717901772d81d17577988a788e9e", + "11": "c996154b2f38e896037f4e14e8574c7c85d938d297ec93068407f0cca78cbde8", + "12": "6adc12acb6dd75f05927ef1e4f6683b85c780abae25d76588bf57ae25f8dfd88", + "13": "5266e99d4e82df6558795288cbdaef86099a84229f002945d227977834b89ffe", + "2": "a80d03691e079aacfb8653f7054a836d2ecd57256285274fa51c52a8a398c727", + "3": "e9f5684da874f4c1bedfccc3a38750263a8d9364803d660871cba0b1176be7f4", + "4": "67c89f7a4550323dd055e30f2fdf5c12a7708b41f2af7e9cfdf1dc5faaa090e7", + "5": "77d12b3c9210688cfa94e005484498b7c285e7c173f48b46d2b87cbf9a3c3ff2", + "6": "23c2881b225837148b997823910e598ad844d781441a1f122a5d370b4b97900c", + "7": "966f73aef9172bbe60348f970553576d609299131e029e6842e1f4dc49c96da1", + "8": "c5f4baa4b7f0237ce60bf63f6694fcca24bdd440c0fbc34a1dc0a6c8947b791c", + "9": "977a23a9b768a42645ef5dd393278c4d577de5f3cb0c5b7d847260c1bf0b4462" }, "attention_unified": { "0": "2658e84fe45bab8b877964c629e37e3daeed2673cef03f906fa967a5e5a336f2", @@ -3120,203 +3126,209 @@ "5": "1e92940d1bd0df8fcc31d5e3ee7e763c1e6ee96f525d36dc9f3b9d28e4e1f2ac" }, "gfx942_attention_tiled_2d": { - "0": "2e69adf15aa134e8f718a8ac0a173350d9a6b25e9280a97d4b1f32522b5810d6", - "1": "6cabf9aa26280550afa5656851b2ff1ee3326102a8514ec400bd48b36bdb307c", - "10": "ee85eefe00a7928f144f87e0cb981d7546e4eaec1a6a14ae1b834c1e2f9efcdc", - "11": "b7af68450f2fec2a072aee5a1cc7b48260162920a57dd3e8f45130565370a254", - "12": "f45f3e00722910bed025b8357ea96cb9da4cd9ea5d453ad14cd1669f332a8a3e", - "13": "f40014e952c10d7e1a2d0d8716c8633d2035800642361e5d462112016b6cdccb", - "14": "704c01b76f73b57f4a0305a65b2ae4f7f1058b7c8f12c8ce1ae405ec94b7f286", - "15": "0c9c2dc50bdfab35ac52d045e126c87105ba32cac0e848ccf560ee0b2e78b53e", - "16": "9a186f05195671f4e316596c77acbdb4c433cf639173a438cf74c798d1de398b", - "17": "48ffdbb3346df9ac2bc5f93d3f39b498aae6d37feb428f859def9ce54d802073", - "2": "cd7f2527912b19aa6aef370b48356b5ee8036ea8045123e6860b52f323df7c24", - "3": "b4d7b76efc987ffeb88b856dbd470ad5cc4247c368a125252454529b0f1bbb2c", - "4": "2c1635cdbf8c8314b0a5c15e076d6f248d883d85e78cc16d89f5e20146655aa0", - "5": "2a5bd4068064d2da40b5dae73a93e3b43bc9e6c8849a7ff832810ab437a07412", - "6": "c819515baeccb3297938d038c3f97c92a0eac7965d4f14230770f09dd2530566", - "7": "2f88ce53cd77456946d05366db2fcf99abe21659a066a77fa6a9280ca5689b77", - "8": "ab5a8456851a154af1c7d4cf74b70bca585e4034d9ab7a9eaea239e389d33daf", - "9": "36f7eb1f282c1cc80bf0fc37d0f6bcaef0e14eb64a03a6c17e1390163952c91e" + "0": "c48d213c51a2cb49f42b8e2e483bae3d60cabea50358d34cac100a023754b4c3", + "1": "0b1c42c032a0b200aea44defa8cc88d7a472d7dcc6ba052af1f0306b93396b28", + "10": "2a15639c30d85095f594d732ad43787498de6135d0b16785a5463e36025f666f", + "11": "5c39b7f4279fe3eceb18e39b407891ee21c0bfa3f1d7bd0f4eb4c6e4d86ecaec", + "12": "5a6e59d0953582d98175b997817d45cd3cf8b654f3ded9dac9e513b66857162f", + "13": "c6b1a14b64af17f144a567478a329e601e61160923da358e406dffd216bb3998", + "14": "7166215f60e8af6289e8557cb0a801b88df95e36b8fd10fa31c4d6cb41843b5e", + "15": "47caf58dafd47c2db5020cc1dc7a54380109af9098db617c3acf8064ec9baae6", + "16": "4f4cd97b435010201e83d1fc3cd9d1a28c985c78cce36b1d8830f2e5648171c2", + "17": "75d4e0669de5e107eafbab65717fda1d6c7cc5e15466fe35c71823311cc20d02", + "2": "5841c730e964542b171c289e6075eaf6332b3cae253707da9fd74d8bb3c9ba9a", + "3": "bd9164bfbcff6da42e08428ca167eeb648967f4c40c019f79ef435be664b0d6a", + "4": "1b04fa2470af0aa59452389cb41125418154ab3939758ed4a38789ce290b33ab", + "5": "4012b806eee5e339b49e28c17bcc519e75c27a2331f87c1da4bff7308b116cba", + "6": "e3c4357615e1bedd2facde2d3ad899dca9e809959aa0dc1991397aea12547723", + "7": "749e1d93638da6ab37bb118e288b67dd31b58aabdf2dffdaa2b154cc78ebab9e", + "8": "068e98bee71e601eb44a042ea57327fb4c4e48bd8e82d014015d8e4e8b8b6d8e", + "9": "2ccf6696602a8821130322fab4978a5bdee831fd693c3ec4a0b07f27838cb672" }, "gfx942_attention_tiled_2d_stress": { - "0": "8999823297aa8bd7648243de94180d2ca48e382d57e82a2ca0c77563e28de352", - "1": "322d219c5abe1cc120d30d74170f95fa73964fb288817f98ba312a5d8c0025ea", - "10": "df777efc24a58d3d1f1e9c4e34f1cab4d669e379766991db2321ad08c1fcf615", - "11": "e456899af55d88e871767dcec02e7f20a897932e9108f5e88502e47515821318", - "12": "9cad5b4ce654f5da3ce27bbecf87ce9ac0516141e5289b298378e17f73f276c8", - "13": "cb20fa6edb35cd989c4a0b96ec0e18873e317f3ba3769e6226c3c7abb2bb5f29", - "14": "0c929b1bac724521dea44e39c013ada67d3abeedb96614358f5fe075589b9b4f", - "15": "06c0101bed38c7085e415f5812082a855afdb2aa1f411ef47727f81894f48336", - "16": "347cc768d28888a1c8b42dbb697797aa6bd7cdc612d6524107e59f5c30bcb072", - "17": "58c397e0ba58940a349789f74e80e3e2956eb67290ca0512a96a4712485063d2", - "18": "31bee98dd66cd3eed8f343e9a3096f7cd858aca69085da59f6e355f8272cca31", - "19": "b0ed5c4cde858412a1aa7b089cbdc6f2730b9fd13166413b423755ddf3041285", - "2": "cb0787411e27a75f9266ee4ceced4359ab4306c1e58ffa678bb4edf4de11983d", - "20": "bd1b2e5650b7ad7aab51f90a179239afb76fca8ebb32ce10d7497eb3202b812b", - "21": "2971a9a29ef0af8ed3f9502304c1db760a8125d5c0fadb346c36df1bdc30af78", - "22": "98f4e442509433d8e326e031f0330ba86e965c4d848545767af64ad7b7ea38ab", - "23": "a38e85799ce755ada86385b5c13a05978c264f6a55a5a3e2f1cd1925369e6240", - "24": "d24937351f0f4f805749947e9cb9c316dccc99bdafd9e582d1788b0c2812d52c", - "25": "c2988f9ded7f9ee70d2339e4be9ff1fa250509110dec19f800ff6a634bd523f9", - "26": "78dee829ca7edd8cc65446cc5dd1c33d984abba8df95e67c5f5f71b1247f6084", - "27": "9ea99ac76b479b4068f4e20b35ab092e35a295066f27018eb8a6f3409862aaca", - "28": "af8991e00ae2f28cb1f73fbb93d4e3cce8b851e20fd33cc905171155b22258c0", - "29": "1732a5bfcbf3502490c5acfc757063071dfbdcacd3acac1f9f135f95301ce128", - "3": "cc5f27cf9ce15ca7167b8f1d3f65a772595f34dc5302103653d701a56a54edae", - "30": "777ef0c81dba5bd55c4e6955568510f60ed5befaa104e1c527f68228b8d7adbf", - "31": "3b4056d199c7e21b8e37d27a53f181f7a879dc16ec0d648b419f74a25cd9d9ae", - "32": "11113904298e60c600ee6f76747c9d226b7ad467a36f7024418c7119d706d842", - "33": "d8812e6e16be762a728601daa678d95c9c9cf0ff4a208a4031b4db6d339783b7", - "34": "4abffede6115e29bc7c8a32d179e935975ab036d2268e1868e74afab7f82a2db", - "35": "d94d15f27891facc73d25a3699574ffb47d447609dd429aa202f75da651ebae2", - "36": "6435a15b61ff3e922e478b295478d668ffb9e6cabfffa87263189b4db2c47506", - "37": "59cc08ac46b19ced63e8e75fafe154b85f38fe45351b3569f479e0edf042ed04", - "38": "bbff5e65a5e55f6cce0ca8f3fd3ad779c9ec56b6586b7f66ed305c6fcbe82cbe", - "39": "115a430bc25dd2e58f9cf619f13a929dc56ecd81f29800b4b761fe2c635038a2", - "4": "60bc67fe026faaf30d69797f81d80cf0107dffb8a694996583312c5f95329f74", - "40": "de06ce9faded8469be1f5fb32e1b5a83e7a618bdc58ecd702d05df47e19d80c4", - "41": "ef2c8be9b285696f2ce227349deb6f67f1f35dfe6c4c92e1022136266652e2a3", - "42": "4ed690bca98939732c27e0d2271d06aa533036846a5f2de943bf3c464741b7fc", - "43": "538d83ec3893c06b366378bcf5249639c044c92aa934026f3c905bf240c999e4", - "44": "a63b2ad0e5d99c281d8f1ac2323e791cc5469f8677b365fa0b5bcf20ff28824e", - "45": "f4159ec166ec889d0053cd6c3c3235346b0b46177f907b7dd9ce57956404f65f", - "46": "29d2f15c9e15bb7a3d6e86ac221d7029059cce53279c9c9281c14de905982463", - "47": "434ebdf81c0c447bddbb63bb1dd5537cd86a74d92510083a4e3bec4ec17044da", - "48": "97db71d4295499d8a21ef2d322d82e407bab887375e20748fff18501140dc5ae", - "49": "dbae149df4dbcf36d249c047a426a58a98b9a905c91f44b68609f1cf2ca87607", - "5": "a23121610e2ee010641af40da0fe72e60982e1d02ed04b4090bc56b87fcc3ee3", - "50": "e3e830cdcae0e6c300462f8ac12fa6c3924ce80ea05755a7f4438164a96c89a3", - "6": "918f7b2c278b457706f0d38e315109710f872c41f9ae7a016aa50417f8fd7d1c", - "7": "6cabf9aa26280550afa5656851b2ff1ee3326102a8514ec400bd48b36bdb307c", - "8": "f155c15be6a0b21470b664e23db10c52e34e05b0c56ef3c92d263548079a9ebd", - "9": "2c1635cdbf8c8314b0a5c15e076d6f248d883d85e78cc16d89f5e20146655aa0" + "0": "526db78835d2b7ea7fb558071166aac070777eb676a8f48f4b2a1b6c29d8c0a0", + "1": "8f13c3331d94040732c05efa4eb8a3998cc912edcd1a45ef2500ed494d674722", + "10": "19de77bf6335940e51a3ecc93f8a99a9ae0f9a5fe77ca4c8b8684b0f510caa51", + "11": "6bcd489c7a9d93d197108964c898c4afc23899ba16bdd8374c91a62b542ff5f4", + "12": "3e7770a9b6627b8fc40cfcbbc3ff615ff3901793d6918de22e10ef538da78f2d", + "13": "e4ffa2baa6816375b7b332ca5f3144feec864fb70a0d61dda2d28bd0d833741c", + "14": "01feef5d4ab4c76386cbd8e01cc5c054a12da2c8ac629105da8c11cfcf35fddf", + "15": "532f20d7920a096de2c3aa5909587fe57780ad649f40d774e9f569612e257001", + "16": "526db78835d2b7ea7fb558071166aac070777eb676a8f48f4b2a1b6c29d8c0a0", + "17": "526db78835d2b7ea7fb558071166aac070777eb676a8f48f4b2a1b6c29d8c0a0", + "18": "526db78835d2b7ea7fb558071166aac070777eb676a8f48f4b2a1b6c29d8c0a0", + "19": "0cb1d827d6464648426db3ccf0e91f9f40f307a5a0fc5b7c91003ce9a42a90f1", + "2": "c48d213c51a2cb49f42b8e2e483bae3d60cabea50358d34cac100a023754b4c3", + "20": "508c96b15f0e4feebbea38f924341ba6d6678439635d8f74f648defffe495731", + "22": "acc7d51ac2c6c950d0c2f086d14ebfdddfc5c9cad8101b00f2739d90ee9090d1", + "23": "93bd007f170e9fee09ac8d8cc421d7596c8a3698ba34d144c0c007b4555c6294", + "24": "01e6a1de28c7824d8089216ef73eb70713b01fccf81e5a7d3a0b1730b5035edd", + "25": "183482df8565d3c9da21caec978cee49a7dd92da8558b359521582d86dc5cabf", + "26": "de107409091a083cf955ad9ab068e24d65201ba8e8bdea3001ee4825d4ca1e7c", + "27": "58b3f98792bf2fdf9bfd6b830ea0642e6c960bedf4c2300d7d974df6326d6ed0", + "28": "c76aa25627566f8d3d8326391553752869483885391be4e94e13251517c3e8a1", + "29": "7ad49a3ae107c0b87aaae4da48e34eecc9d54543c85d2e6fe32cab7ce8604061", + "3": "7d901f92f73b4f178ef93aa085b24d243c6719517aa407f059857acf5e44f308", + "30": "939d923b3a793b9e7c07f8560aa5f49cb652d14a1f8193ba1fe1b7465e78d79a", + "31": "fe2b6dea09542723c729b0ef73c6b269be58c18d350ea0643d2099ddf95a4b3f", + "32": "526db78835d2b7ea7fb558071166aac070777eb676a8f48f4b2a1b6c29d8c0a0", + "33": "526db78835d2b7ea7fb558071166aac070777eb676a8f48f4b2a1b6c29d8c0a0", + "34": "526db78835d2b7ea7fb558071166aac070777eb676a8f48f4b2a1b6c29d8c0a0", + "35": "622120a44342e1666584a05d6f48bcf18ef0ce0d8b768b4d26e13806fcfd22b7", + "36": "10667b58833eade6a83ad5ca20daed04cf9886cb8202541d17dba7ae796c14e4", + "37": "4ea4595de026e0216751eb5e687d32176b6bb9a31f55cd6f51ae789431e318c1", + "38": "e9af6071d73cd8e75d8a5264694055c7fa605c1098c629a48f25ab9cc796afa5", + "39": "1c716b6fa0d23672c4e3d4f9fcfc9e58f26f013d4d7776a3ded61902d8af7e52", + "4": "89d69cbaa3a3ae2ae66ea48579908781c59d77f80e3b48983ee83d934aaa5ee0", + "40": "90cd6e974e47c9ff755a7f5cef8cd38379510d2bdfea369611ee657218083bf4", + "41": "99841f9dfc21f636f65fd34f8b193c5644d0850decee120d9cb9e37cc0189dde", + "42": "739dd9274a1c31f4bfaeeece56d3c861b4948c00eb2c7166c509c3e00bf0b2ff", + "43": "b7a9ee5a058e51e1005c915ff6bf2088a2eb69dfc3d43118dac47980610d722d", + "44": "035579a4a7b7ed94021838df92435f27677b50c3011f5365dd2a0344b69bc5ae", + "45": "a6f7671dc802bcd3376a057e6ff5373994ddf018ad4fe697305f1d38e0a19be8", + "46": "0083f521d47f3ae56300600bb53af31b7df7f9c6cd4c46eac47ceecbb4055f46", + "47": "4c31a6f3fb8e23c15b94eda0b72acfbfb40bcc921b64b432f1120f91a1a69723", + "48": "761b82fb11531f77270561c2275976b8ff0f68644a003ec6a66c4e3fdf54feb4", + "49": "29e9c6f096aa6db1aae2ed10f5412868ae1aff1598545e4c5f86587807bc8e29", + "5": "a0a634b02577fa98ccf178d0bd0b632cf7ae05bf488700d0b2f9b627718f8cfc", + "50": "fdf8e987215bd99a17880e0904a4ddbf87ce49c368e83e30a3c33d50c23d20f3", + "6": "850d5210aca47ec33dfff9693c48ba0fd54a32a14c3a799a74e4d55c93ba5a52", + "7": "0b1c42c032a0b200aea44defa8cc88d7a472d7dcc6ba052af1f0306b93396b28", + "8": "c07e9d89ff04e6cffc2ee36ee1814b44d22b8a010a1bc2fd0705ad4a0711be14", + "9": "1b04fa2470af0aa59452389cb41125418154ab3939758ed4a38789ce290b33ab" }, "gfx942_attention_tiled_3d": { - "0": "f61a3620614b51a54fccba438176c0e362dc492e3c3b6ce406d6c06dcf02083e", - "1": "6d7582d1419b89d70494ee63e2e05bde6c60f299f73440179d60e4afbf5ae8e4", - "10": "ba19d4dd3407ad9d67333362a27140567a70468654dc174667ed52fcabd35bb7", - "11": "0df4aa34224377bf944bb3062b546d5eee768e775e5e8f1f07a94ad4f1c8d4c6", - "12": "3ff0d802cd6023feac84bb1a07bc48a8689cbea24046138a956b4e66d5988ef4", - "13": "4db7db604d1a8f19325aaaecd11cc674aa0c2fb6d035caa33467e50ad5156c73", - "14": "02aea5df8e61523f8acb97d7e72aea32a2ac847544b55dfdffef362f21461ef0", - "15": "e9d7c0a309f76242a4ec4d1199cf8d05c7a7fc80d88a398e68808fd158a8976b", - "16": "d44a86588284102e502dd7f329604faae765fabd32c134fca6a83a37681031dc", - "17": "3f7ceb0edd63bb22f911e8f14279a8b5be14b1c4f951a9ad42afa7c9f7c38acf", - "18": "8ea2760245c71f79c9b42ae1d10362ef2a07df38ba65e17de924d813592aef1b", - "19": "ae54f0e8c4ab82f253693ccf4077a4a30bcdd8bf4b0fe965a94357eb3b7136d4", - "2": "d776e66b764f75793fc25b17e2639b306c289592fff73953fcede2e76b189322", - "20": "7cd9da7ec31fff72f25b3c5fd8195535771ee48ed073fb72e89895e2495a3a99", - "21": "806b56ba66837f09e3f56f4ff9ec3603a6f9ecbca388d0da00fb5fd573320247", - "22": "197c0bbd20666e3b52b0edd1c5d2d0c08261b7aeec06c5c05c565fdbad24d9ac", - "3": "41f971cb9da04b95343d06bd7c15143ff7cf3e82eb9baee619b563bc672d3838", - "4": "5eff7adc1f3424caab3164b362e805f6561e65955071ba010deb46f6a8aa33a2", - "5": "26bb8e65bf6a98aa64a294fc00756b12910f3f2a2c4a3c7e12a7a05d5f0c0244", - "6": "806338a1f5ecaaef9aa176e6a8ba08cae73fd17ab9b65ca0cf742df738f73bf7", - "7": "5d12ced5cd4697a9a82c85407b2becb0c54e0385e0968b6de06924f4624d5a8c", - "8": "ce51f9cb5d66e24e641647e9fdc7490bff294a358b9258ff2a86b9512c710740", - "9": "124c151ef4a8be0963321ca6f6e49e4efde00256aba1414da772d5ac3d074690" + "0": "d94d49bbb05d2df758da221ad23ee5f680233b40bf7408903d8e3625fa955f39", + "1": "09752826b00ca337e7315807d33fd8aa0dc51b6ded27bd4a472f77a972b6864d", + "10": "1a15b979a525eea328760b7cbb2a346a7d9060bce4a0cd8f2f6058e6674e76fd", + "11": "ca406f9c288b7f671ef4ade88db8aee72c7f46cba9594ae271f83b60c5874c46", + "12": "1db13e53d9e4d8bc04f935b4705b11646c10b39ffd833b7e8e7b9c7ecd298b4f", + "13": "c46889d1715ab6a2f89f632b904aba71f970b8d638616cac468ab797ca595d2c", + "14": "d0068ca273a3714dbc9920f8aa11e131e647617eb775c48720502eaa3cf57406", + "15": "3ac273b458497f3df8ef5fc73202e88316cc92fa74c680e5ffae51ab7c312718", + "16": "70c3cf16e608cff4c036354a7c5a88fe88a12376938d7399bababc587078d3fa", + "17": "a478df9215919e6ee5f5bf7ab2efdafeb77becd737bce7a78493baeec0b3ef9b", + "18": "fa30b40fc5370b53eec8d7170d18bea30cf261c01e382ebe9691e48c17bc6e9f", + "19": "07f52f10fc07c8524d3476fb97a3b1fd62f971be9f55fa157bd1e768dd9b889f", + "2": "9976606b42a736348651dd1f709a3b74a2d1306d5c1cf4a89921f85023b9bc13", + "20": "40be376758c89cffb631946b11546bde3a2636345a08c77f8f853b389d4119cd", + "21": "d0a7810fc120da3ed3c58fe14b1218d6e112009c0d1c1e74d8c0bd16fdc4dba3", + "22": "3061be726c9cd6cd393f474e021c29685f12cd1fae382b7c561df1e172ee4ccd", + "3": "e977cee9184c8bd95675b33a9b34e7b11218471ced98a709f711f2821deaabf7", + "4": "0cd0c07d6a2c87caa5bba1c2720ea9e9a703299f077a18668a9e971a1fc8dda8", + "5": "33804a39dcf82487f3ed9b1d166c47be6ac4f9e768184a14b26d1fb089185440", + "6": "5973c29007a027cf1bfb84d6d6c2e4b94b03d7e3dec3bde6275e7ec71947bc6a", + "7": "18b7f6c6136b8cf4dd23bdef33013ae10aba43beab09282c5b03a5a3f756cef9", + "8": "a840ade391b97e553602f3cd32dabd909de586c92b2a9f44dd51db7975e74a01", + "9": "f9794098f04a1cfad57dad0a169e28e5915f03806cfe4f573c50602f04132a6b" }, "gfx950_attention_tiled_2d": { - "0": "aef1bd87cba9de4d2392009f350d964a11bb7eed66ac3ca3231d3c011322ab92", - "1": "e5a2eaed34eec34f704848b54d7f1fbaa73bdc0497f56ac81b047aa579b77e9e", - "10": "b29f827e6767c2b6b8cd79099318bfe5225d6e7b4bdb720b840b92c457905dc8", - "11": "c2285074ce9e3cf1ce4225770dff64ca327616fe4580261f319bf4103450811d", - "12": "0134bb21b15243d9fc234739c75cb49667b3d41d4d89290dab17147f32760c0b", - "13": "b4dbf219f8ada776d4709b463f12cb92a35910919e864ecaf1899ff42073f5fb", - "14": "9e17b015fdce50610f46303797e9b6a1d22ecc96e3c751211acb756ae7221b7e", - "15": "69391888188ab4d0d5726ca85c07ead59099044a5eefef231dde82d94910cb9e", - "16": "d519b75ba175a174597e67080e72843be91019f6f314c2e6c3dd015142cd2540", - "17": "94b7420adafba98d7d2bc836c5d991976b1f2141bed341536cd8ad480a1bad64", - "18": "16ef974a2cc277ef6163e5069bd11dc23a661874b77cf2689762da8cf08beb49", - "19": "f23a57f4c3eae8f014f5127896941acc48ac384c2186c911d40fe63b92c8fbbe", - "2": "a6995b48bed6926382429ab3d25bf0305be67e3d806fa541055c00db69499ce5", - "20": "4e15d2201491c1c10f5f500f7d79c0e261c161615b3e5d0749b2f6620478a063", - "21": "eae20eca05997df37b896304b573d19bd6b7a87dc96ace6233a6b90f55eac29d", - "22": "7cceef9f1dcd64ee134656dcff95a553fad3122843c3fa639b6a51efa213908f", - "23": "0d49a4adf0e15713ababe2aedfd3f2711a10c8a9ef337ffbe046b38c820176a6", - "24": "6bc56671c8f90591fd4a88bf65cde7d5d1d4301f1a73b9c768c3c6868c1a649e", - "25": "3045d6536b535c3d5c69aa12ca482a7f3afeabbba3ae3d0fc2999dc3ca4845cd", - "26": "f3dd944a5e30734491c11242cc3d8fef01ad93377f3661a6d274e23498b6b916", - "27": "934c6c1aae039928280cea97436c58b6cc44c59d87ed07f5fc52406fac0f3b77", - "28": "be378700b81d7e791b8d286f18e87e5e4d6bf143008f37c489deb2cad5cd531b", - "29": "a513c6f8d0cbb468f25c3aa5d70d2cb44f442ac3b880ac5586bd40480605e904", - "3": "9ddadc0a40dafd15cdf2cba2f95070d103d8d35cf8e053f939dc91467b62ee2a", - "30": "74f441dd0c4fe899020747c539ce445e530d2a5d9e03fd3384f063790a137c16", - "31": "e495865dec350a16d62e23e67d60a671b3fe2434010e69a423d6856e82f13bfb", - "32": "2113819d5fb4f4a880a1b05bd4e0196c931c499b913f6f91b556b45132be5ac9", - "33": "e2cd7c648daa809306071f69b565d5e72cdf63761a540ed44d134cba3e6c264d", - "34": "65138d071a53e1fe195cdf3727c9a8fcc07809b59bd43a5844f46edff00d945f", - "35": "a27f63c87bb4dd49bd340a190c4bcd9f00fc8b578aaf5fcb4bee0fcc85a5989e", - "36": "5f094ce80ee1cf4086b0f6e97d06f096d4af247323a0e58ca0c1ed068cd21c6c", - "37": "98cdc7fada82e1a3d82cb10cf465ffc7c4248a421bc7081770d39b1cb6ccfe66", - "38": "534bc70fa1d7670dc71ce291b317913b7197ea48c2faebffb8a8ecf89b9bf382", - "39": "7f84aa71dd55bdf60a8804ba234df24862cceb3928734741174f2bca13652991", - "4": "45b22d22474972867af29d287a02bfc7e4c762c6e8b2ead606f3b7e6b8ff0208", - "40": "6d4db194ca2c4b834e8b582ca52b81220903e76ba233bad8121264991e70673a", - "41": "26747d1036507fad8b75deea57612a6e734a35432ab6c24cc96931aa1b0c0c14", - "42": "7482b05bf53347600b2e07bfecccec0fd201f6323d797325b34b2748e5182589", - "43": "9d4cbb738780e8b64d63f025196fe1e87c29431b5de9e92c447670dd5de7378f", - "44": "a52f06d7b27ceed1bee714a1c41102041abdc84fb7510fdbdf91896d29e8baca", - "45": "f409691173fdb9577d8fa2abd69b2336f090e302fa5a1e232023914d9d45a2cd", - "46": "56b04774a22aff850bcdbdfb65904dc80eb25f35d411cb96c6fa5f79af961b0c", - "47": "7813e8fd03bfe8e866ec306064eb93403bc2448fe79bf1143716840b0d7c0ee4", - "48": "4ef3a7411b0b3f001f4b6e14baa4a3404f5d7c1a985e0a77d9c02392880b53bb", - "49": "d74e5566f4dc2b4b5181d5a812dfc4a5fc08d9d4f778f261997fea3dd99cfb68", - "5": "069c7cb994d1b2214415889e995d5469a4b2a355fd4b88500a175b7981b5ba67", - "50": "e0bc8d2ba792ece678e9f00ea92b553039c39cea63579bb38021e37772b129e8", - "51": "692c71aa740ddc66598df3d97ed8279d20abd942f225e218e9db2e9ae34f711d", - "52": "682c63540b26530aa8455bc3dbfee26d5b1853660d45174ca5a40c8226b0e062", - "53": "b81fc27fde0aa56cbd3e05e7be9077879c020c844df5428a3537fc0e036879da", - "54": "bb09632c2225d223d2daec4f586e9fa9a662502b11d1cb8bfb31a77b83a11e29", - "55": "f3944a91e9f9593a8ffcc91ed2f2212eafa3941211ba267114f392106ff6a269", - "56": "16e2f1f8e2b727827ab2a17d5e80d674bf49d3b2bd75b4805381b4394e3f40eb", - "57": "87f86231f8649d807655b4944a9352e7c3a1b7fca12233f4651409c3ea3319d7", - "58": "dfc189ac151073a5a495b7a4806b7c5a9739b42d0569fe9ac72436ff117d56c6", - "59": "0686baed4d10755d2a2391010d81f753c2cc0e6f31e7c8d582ade28aaa8ee5ab", - "6": "f66b88c6300ea3086197f51c28d657bce9a6986b6eb92dd7b13bcb30067375e4", - "60": "c44a8fc90d04b63ca678fa74b400ee0f2f25a15272fd486452f593874258c1cd", - "61": "c4b7fdfadfd92354463ad1fa13a4dd17d87b32acd92ba14b47be644cbc9412ce", - "62": "32d908604514f4fa2c094adfd67ab0313fedc953c0b3feba7ea19bcb2ae87dd7", - "63": "bc5d50db11afd598ac32da8b3b04a312828c6cd140c15dfec32112c79084bb3e", - "64": "da27a564535e6dd9991ae53a40410a5d83f47fe08cce29dfdcf49547109c9c49", - "65": "b6d959cef65dbb71e1c5ce48848a5da8c028917db52a140bcc696d9f69d3317c", - "66": "e76285b8456dfb69ad0da8dc5b033ae6d432b94437744bd5fd8cff0f17b6ea5f", - "67": "f8625956e6fae9b60d82881ad8784db21bc77a40f581640b45f2c62beac076a7", - "7": "4eb18730ab767621e3f1db59982e8eac0b9d1848a60fbc189bbd1e7d36505ce8", - "8": "cde3ea55f0b75c4b71116b40a1526efb225c2a7d60259fc8ddbd8e341846a71f", - "9": "fd95683fba10dd242ec3bcd73c7baeac1108368c265e2b5106d4f219e7d5b662" + "0": "4bdf8cf347690410dcaa17a551a71466cbd0db71e8f18b4adfd37bbdf13c3075", + "1": "a48d32faf593f74b812e47aa721b14e259aa81a13b0fed9aade63c66763590f5", + "10": "9f6d867227bd3855a18cf52df41930f7f6a10fe6441c6395820c301aace4e94e", + "11": "7ca14895465b50667e984ab0964b9a3c1b131f0a032e3cf9f7f1b79b3d2511ed", + "12": "0a9ba6aff53a1da67132e38cf5a3e6b841983b300187c0aed338fc2bbfc244ee", + "13": "a734f4398b98c11bb9037757640442b4cb518da420a9de3be438d739ac53159d", + "14": "2a3ff5f494f978e9497637797a6f6633151b52f8187beefd702c13aa4451fe40", + "15": "ba35a211e2aa4820c3b62e1d270c5c1f81e9617f3dd6e14a1539d0a7240b5211", + "16": "be11ca19a167b93140496f53b2c8fb2c8fe9151d8fe541ebedb469ac5270f719", + "17": "160efb90cec21e303ecd8c197fe52ad8c918f325d914bba42cffd9652a091d32", + "18": "e025620977a2e7fcb56b1dcda5983865ece086bc641eb25a950731506fc3dd6f", + "19": "bc8d4ad3aaa549b3344a8bd3c918bb98fb9a4789642273042d4b47d0760674a2", + "2": "f678233f21191b677e270d838370d085d7d121418058abbc12516c40d91a5569", + "20": "4139d67d372a6774f96a5e5d57f58f371fb70888a351c17312ba8e00c048b6e6", + "21": "ad113bbd9afb2236eece86a5b341632bcff092b81789b300a6ec69c6936e584a", + "22": "acd37246ff7bbf8b2b80af258c482799c5a7ff33aed03d1a30a47ad4afec76fc", + "23": "6a10fb9477130ae1967f5482235269ffe32978cbc8d7199488de18c168f5c337", + "24": "696afd50443365fc801e1c75dd23c5ea679dad1822f9c7b02e360a04f5ee569d", + "25": "7ca14895465b50667e984ab0964b9a3c1b131f0a032e3cf9f7f1b79b3d2511ed", + "26": "7ca14895465b50667e984ab0964b9a3c1b131f0a032e3cf9f7f1b79b3d2511ed", + "27": "2ef113883021e464e292cedaea5cbc5c277d172b7f99fe3c848233be837e2c60", + "28": "ee439f677c2c593a6ebe407256e3e359faed0169915dcad3c01a39e50eb20ba5", + "29": "68d03dcfed4d4563e203a7d5a78127818e9d73e01149a4d3667ba9ef29d4025f", + "3": "4d1467650e14ddfd7af8bcdcfa10d48d7070e9bf736329b0697eb22c61fb82a5", + "30": "2a8f5a22c806acda3fd56877831378456b6453873fdbc877ef45aaf501b360f0", + "31": "7ff62c2d3ca8cc0b928c413a9ca28e900e0db8cb3b01c2c6b76913f16c765330", + "32": "f2f45eacc1726d15c508311be40be7952d364570383cee6c06687aa448bd7d49", + "33": "6098e6eeec05288f52be6d1d5aeb6584cbd79b6f9b611c3ce580d779b8250623", + "34": "a30f18d0cc70bf9f08f911fd44ae1fd6d139746502a48eebb3fce03379045dbe", + "35": "c23911eee14cd839ab6034f847e9cc9348ae1e838b35b8f0fd2fed5e95a4c18a", + "36": "17c0b16cc91ce2fe9ee5de8e884d350da73f3b7c676147bb70b915aa3ac38276", + "37": "b24386ba9e206ab57fcfb153fa6aa57ba8518d4ebe1ce31d8d69e01bc2293769", + "38": "94e5f3f7b0d34f620b6512177b901dea14b8882a125fb6330fcef5acb97149f1", + "39": "7bea9a440296cb5e7b9b32fd08b96c1be1b886f3806bdcf7c8377488292bb93d", + "4": "0bc0386a84a9b0045d8dcec43c59116dee2069cb698e8bcdb6a6725fe82daed8", + "40": "abc15b6d5b9550861116330858c6d3d5918c21f212671e4c7157a29fef1d74bb", + "41": "2d7a42540c529c7ac299b652f6465b7ba2ba2e71c8e48d243d8d52653651627f", + "42": "5fd70c6fdf52943cd666a6c3fe46f51d6ae30d28ca8a4bc1360088d9d3bbb032", + "43": "0ea0d641c14c615ba5c044a7291f07da0819386e84e97095ad5aedbc13004d35", + "44": "0ce7cdb26f8b90be6d7fb890ea1bbd9449dad803e4f3562eeb1a3d2a56923918", + "45": "ce2be7771b01bda03ad888e5487f3921b50df7824327166509dda2b821812e7b", + "46": "03c5b7ea2cea96fe90b8b55ba417592a2d69cf41fb606788f97003cd9cef5f94", + "47": "a5c4a8c796d26157c98a65cce5caafc9a565225b99212a097562a86c605df9e3", + "48": "04e298bcb97f18227b4eb4f097c2bdcab9d002586bcdc6343aeba8971e9eb059", + "49": "91c49896355c8afc4e4ae7d5a94b1cbd8ea1d2b05fffa31bb2f83f807da47cb6", + "5": "7ca14895465b50667e984ab0964b9a3c1b131f0a032e3cf9f7f1b79b3d2511ed", + "50": "f6a6b13b3fcdb852ed477ad5d61a40934724c3ef1aff1165b4f0b900b944eef7", + "51": "bb6105d40fdd2d5e78fd425dbb91cee9b6af3c27a1ca94338a4d5a6b37d2dd1f", + "52": "6087975b3babd2eab931101f80e014f14192007e387bf998e9576a57c79b11e0", + "53": "3e7d8580c1e101b640063c74bd2262046857e0143680999ad54e14451f6b53f0", + "54": "d5845a417190c278ec214528e56ad209ba7f685a4cce554469082087d9b96526", + "55": "40192b8a73a7474443fcaf49d937a2009fdf45fb2cbcab597c7238992e14ba25", + "56": "434c2f9f7b232ada8efb4654e94780b7e564b4d27ada46071ff827a4df9a6c04", + "57": "a1e6ecd4d4a7d0dc38dfc51c209070eb7b74f78617e7ad9e0ef7460facf7e682", + "58": "5f84bca7bb5459241e29bd862e11e8a21c0b62e58fb94178c904dfe8b7631ec3", + "59": "df6b14c2a9878232e818371e7e19ccd0cb1a3110eb45fb95bed49184eaa7b013", + "6": "bb8a1c83af0c4f902c64f36e5fb9a139a1fe0ade58ee2839c7ba50b9ac4dc7e4", + "60": "afbb076e21472ba5c897936f52f9a97090a209c81d062e037224b0b4f9c3c792", + "61": "32a9b8facd0d1a0cdb37b53d46148a6d6ac3bdc4b8af55d56064b7d77cd64bdc", + "62": "f966d3471a7e93919550d4b0807d726f102db7d9033ec254b6ffc548f80e73c6", + "63": "5f463bab5a0321071c62667fdaf053b141d3c85a3a67933845f10e63da141902", + "64": "e0057fd28ab9f539124eb549f5bf19fd96d6cdb0390b181a8b75c012787d132b", + "65": "c484f3022e26a9f69f017dd88249885875919ea1e2b5f5e3e323b039231f1b1e", + "66": "7d63c895465776f5df0bbb8676b765efc0bf02a57b74225ce77e9bf6f17bce86", + "67": "fa3d5c61f81199a636e8f43257251b8ef6332f214a388ca4805860cc930d91ad", + "68": "0dcae40aa492ac2069cc8c7f4a61498b4cd5c3fef69142470ed2384602d6ee43", + "69": "63b458ff53037edd27126db2f8028159d48239199c9494522bfedc95fa54727f", + "7": "763832662414ae026dad9670f8f064ceea6145dd42ec2159941d37f3927ed2f7", + "70": "2a3870347ab70cbcb276925da1726e5c51dcb737771d9d2af5e742a8c22680fe", + "71": "2e6df5bd9a6d6430bdd7cec8fb061975b840cc885565eb058307102a26e52b8b", + "8": "436a62f0670b97e464439f466724c5d911a96b10e4bb035429b37d5be8d85243", + "9": "07eadd3cdcb25d33e4162b75d3543681a9f260e6a972c64ce4bfe3a28dee2bf9" }, "gfx950_attention_tiled_2d_fastkv_regp": { - "0": "add51e2d6b461034696d31a6679a1051d90268953b1235bf55289f701a65d382", - "1": "b90f8b2d10e38eb102f3fb8de2e2f92811aad09d8641a72feb2ab32bbea3dbe9", - "2": "fb726c7366cf45b6c6893fb5fa4610520f7aa61864ee6f2b69e44776588dd2fa", - "3": "50b3aca4dbd9b425ecb92bc9cf908be84e3ebc91dcb32879fa7c205d196c6681", - "4": "0bcce2d8ca0a9d51aab0a1e776dc7de5b6c2fe662180642a46cab54c5f5f255c", - "5": "7045a67c8fb900d078d3fa3ae38907661912ccacaf6b080a16958074eaea21e0", - "6": "570c4873db463d1152be29fe809068c5bfc0d7f8c3d574a28370a196614d3cdf" + "0": "52fef7be3f5b5752f4d62e09b0c9296df1abdaba39cc1d360d05e41e0b70edb3", + "1": "c1ddc54998b8fa9724912c653f1a6872e1bf6c466d80a9863816cf41a2c2f3fc", + "2": "13c67ca229c95745229264eb5908a6cf2ce401be7c2a430551dfdd2dce2fde9f", + "3": "8ed9b04568737124dc72ae75167039279da8e2e1ffcce8fe133bacb1fea56a02", + "4": "44d88166afe8b28d105f18b7b3c5497ca75c376098ea333cb297d516ce1f1a49", + "5": "1383435210b90de3bd597ba5ac430d5daae9913cf4c0555d4db535f5a3fea552", + "6": "7da154244605f364dd492c3f8ffe5337f8f006e2fbba80239abee64d30fe44dc", + "7": "275ddad230134a4af0d132e1f531d155db6777a2e489b7b85a1c36cc3324206b", + "8": "ad8ed389d00c79488218abdc33e8f45f86a465825c9cc13a26bd8430033e0a2c", + "9": "cdd1f912248b5378cbb6eb42ea2e845570cac8f193ffb4b2d014bfb0a3924c2a" }, "gfx950_attention_tiled_3d": { - "0": "211ac134586448e278d5662ad89a605338b755e91bf63d1584e267dfc79254bf", - "1": "d4edaafd12d2efec90c73cefb53184fcf0d82bec148db3cd292db16a342867a8", - "10": "8cd481553082c11d6302bde9d47605eb0aedc7f081c24d212e5c09c808fa5895", - "11": "da1378094b7054d0afd8811eaf12733e7aa899e4a675c25555fa93823f250269", - "12": "9460de8acdaea4eb6feb3489f338fc4ab9b0c8f56abc16221f4d68fe23f01943", - "13": "f9a6211224a477eae84fa9a316ad84dc0674756e7fff8dd663c536438b57d6b1", - "14": "d95a4ee4691a5d4498aca014e680cc4bb9dad85731e6f7b913aa005b7542e0ea", - "15": "5922ae9e30f10d84ba06293cf1b9f530872f26ac4b31d264ee0e7e41d96f85f2", - "16": "11c86b374923581855b8fbad8e2e0728c03c86085f9ad5e01f88bb0dc18cc510", - "17": "53d870b45bf1beb8bc838b6941a732fc425feccf36347b01a59ec93cca6c06c4", - "18": "89b0866d30ee69cf00927ffe5cf6f3117d8e99269185ed21de78d508d284b7f0", - "19": "360bd847b011636cbd63eb70c90cda173cd2c7c913a16abc881f53dfbbaacfc0", - "2": "43225f4bde00693107c9eba8bb29ede7589c65b60cee13b6e310bce7260bdce2", - "3": "011317d12e0cd324dfdf21438343e02c900930fb3c5ccd02ec2752f73188734b", - "4": "68468fbf2b4c26089c675cc417e8af43b308a6a7838e9604853845fa154e6dd4", - "5": "fe17ac0fd5f0ba6ee6ceda19ee333e679d2734a21d083aac0a84e306b2a66d49", - "6": "5a40a5a12f9bb3b9e0d83625990829bdbf4b8165dc92b8c50dcd3f8321907766", - "7": "6a4c575380ba8cc9c2ed2bf2b7c645cbc37bec336a11eb35d2c6c2b6d8f7eabe", - "8": "79ff0b255d637b2b899291ca7a8ec600d0cde17280932b2e0279fcc0b37f408e", - "9": "e125f3349708cf7b6e6e4066957438905b2a9ff7e93da177de8d0771d83e0658" + "0": "fe675929a9634bd88d72b450c85337132c928137776c495e32b5f28ca255d380", + "1": "57ab70bab54350aa49d3baba87fa087105836e6842425649c314ebaefe8c7732", + "10": "aff41145919896c345f3f878958e01c97e0813ad7d5002c5d0b8751f130ad8a0", + "11": "9ff1a50a1380881e4e300ff8dffa21b83509eb9719688096c2d236d9a00820c1", + "12": "fe0123e9b13702deccaf6953b12d74929850aa602555419e8e6385eed184b272", + "13": "cfcc2c5f687396f119a6dd8a1e252d7f73be717d230c0a2591210ff0274dc1ba", + "14": "b0bf7e57e30193170e54bde16a00c4bb2db9a395d3d7d479520f8934918ce9ba", + "15": "ebfbf262cbf7c31026eb26a20825d580de27864ae6a01c46de1c5fb47f34f00f", + "16": "26cdb1a43dbd68bdab8cd379396e8c71dd318b6e495652951e245bc9ccdf5178", + "17": "4f21944a3a849cd9ae0f6350780c23209e2d6fb6373a1532a142209954a98438", + "18": "7dbf7924bcf5faffb0642a37313d441933d562edd526a6117e812b7196c6f968", + "19": "a8071f2bd763e910591c2eceb4b0b66a106a3b8272dd2ad7e50b1f487068cca6", + "2": "d894f9af92f6e93f9c3c31ea1eae12bb5e41a0a48d553e8c4d4f78897522161b", + "3": "a23f992ad7619d51697ff9fe98320985bbd24dde1dc0119cc4e86e508561f3bb", + "4": "106ae0d2ec8b80c3ea740f952f09de97420489008902dd0d8dd99d26d48ad098", + "5": "ce56b406998b94386798028bd17a420415cc79f5e36536db08766d6ea2815434", + "6": "37d8ec6157885dab20d3ada0a61745c9741e5d0f26c475359b48321e6611d0d9", + "7": "3ea23963d10f6d71a867a5aab0dc65813f5f6f38728c9f575dfc4b8a5834fd01", + "8": "9046803e1f3e588a2e41b45f7d885fced25771097d95f0598ea2f7b4856ab544", + "9": "135d33061aeb2d7494e23c685581fce8caf78a539420fac27cf3936aca4fcac5" }, "gfx950_block_scaled_gemm": { "0": "66bb0b6afc125672530de28db30992a4984e6aaf449efbfa50712adae21d59b2", @@ -3746,20 +3758,20 @@ "5": "c1623e50052e670b53bc35ceaa5c2e1375c0a49a40c351cee09f67f2f72e6471" }, "attention_strided_kv": { - "0": "bcef03e3b5555c8bbaaa03188901c1dfc8bee0cdfa80da32026367e82e7c727b", - "1": "9becf0d4f1b8eedc388e818afb01a40607a27efe2d415772ba1f6352c2ba35d7", - "10": "35992a4d15f09b54bd668d4b21556a2c871fa8c6f8aa39a89169736e303476ea", - "11": "ec52f6ee37cff049c8b81eb579b71c32be965bb4730fc7b30bd477196499727f", - "12": "002dfd5d5ce7d67b6992178da77d629eb26e80c5343c95a16a0e2f89462cf752", - "13": "f71e2da33c81aacfd3a6690c0fa2c540aa3a83586e0aa8ed0f1ed638023677db", - "2": "7fc95348ccfd43e6afef1b5bc83b0e17b55c0879d5d8c752d40e6f631d0b9c77", - "3": "c88f5e1109c70e0c4f132b05578c6abf440d04bccb52bd774b492d67d3acc9bb", - "4": "3e5522113635114b994bb40cf19bc51a417765ac827cbfa879acb45cb0e62796", - "5": "3c03d60f49863bc856426e1431cb98e8e682da4b6818f1df320536fab12b1314", - "6": "2437aa43c1700bf2af3731b08d30869d68cf9110b2fbed6114fae27b78a4e0f0", - "7": "b1503cc02eed262c4d08b02d3802a16f3ee74ff0700f756e79e9a1c17580ff71", - "8": "8942a331f8dd8285230d4a53c02bca59ee5158284edaf12d7cbbd50900a5bfe5", - "9": "43cbf9fc03696e3fcf0a132e31a769d92cd48267eaea3526e2cc7c46c35225da" + "0": "14ccc0e86cb9a96163a8c32812cd718dc454f726dacced64fa3cf07db5c70c90", + "1": "62086a1cb544ba32772dc112e252813ffff1961bc2db23401bf0003bce718b73", + "10": "0c4719c51595a01254f8a7c7c26bcf7cf10f7164fe54dc0c697fe36b938bc1b9", + "11": "c8bdc92a7fafee7c1187375671ea8caab00edceaf7b5f82c844d8343767f22b3", + "12": "b543ef77d6fd6bcdeab9b00f9c7dd8d9823777987f2e9e09b79195f3d230be37", + "13": "a2a213edfcbe00373fe8d3962f4b20a7216c9489a0c5c039536d63483f5bae57", + "2": "f2eb0fd45eb3693671f6d11bdce0eac92295e784e74efffe21351150e2807039", + "3": "3a8fe959ee0737f990adf184e812dfe983b24e0f50cfdbcad667aeda89eade72", + "4": "2f21d041985082346f34105cef0ed4b58df655eed4c600a176deaaec0350cfc9", + "5": "d2770b39913bb089531c1db4d9fc31db4114053922f0bdbca3b34cdcad122a69", + "6": "b4720021fe24f67cfdd7b6854a25f4b371cd5c086d7d804e4a1a451d8f885c82", + "7": "281cf49847fec9bf2f0133d0b0f83c9ff72e99e3f9bfd846d2d056289604609b", + "8": "8e1697deca33346efb263397db405b14b835a617e0fb8a9d6e5dd19958d767e4", + "9": "5f423f4b85911405a7548c7641e955fcdb485e040872744970c16c92326ae3e1" }, "attention_unified": { "0": "d6a5d8b21b5450ca0a62b370d061c63542bfe0ab168da244c80451c0162a7852", @@ -4917,203 +4929,209 @@ "5": "0ca6d5f7108bf00a9acbccd47bafbe6a067bbf7cb7d4c64438010f406175cd7f" }, "gfx942_attention_tiled_2d": { - "0": "d028bf1c22fc9d291c911b8c08cec2c8abd036ba20be67a77907c859291f9f7d", - "1": "65f08a82d3c11cc9cfec54326f0415b50a69152b33511bcd8ab41d17a294504f", - "10": "47cf6a4b2fb1c7302b4eaa5b0738e84a2c6219a610d55d3deabbe9e19feded80", - "11": "93a0abc05900e4dedfda9e3937894abfd80b73dbf6aa3eb20fafc1331b50a6b1", - "12": "c11599d44791da28410a5af9bf6eed88d3a77bb88e602969ca7aa8eeb3f9d2cc", - "13": "b550bff27d546914222eda35bef90306ae128d60f01db4b6290ce61524451747", - "14": "a7c78d9eb2df50c53c6246b1b8baa1379e6a39cdfa4d8e2d221fa91f035314bf", - "15": "17494678dbd9a4f25126df82c23994f0b001367f056a730141069fbc3e8ba0ee", - "16": "150b2145696469125d6efe3d1dd461fcd080822c0d05f8586ec6702be1130ba8", - "17": "c0aa4da20d38b6922a2495e8804f7385af85a82e7213e7fef086fa9c3a953ebd", - "2": "34fe3ea223d45fc886d63e8e0df04a3c7927c25c1066fd9d71a982a5d750a9ab", - "3": "d4c54dcc59b75ba65eab89bc02de5af9c8a0bbb54fbda8c91bb910d40025fcff", - "4": "d623775900c06600492019ae7297a0d24451258f05c3d53e9fad32ab4add280e", - "5": "afc0e52d8ba3e498cb318311d93adc08e065e66d80a7b3285cee6649ba10c13e", - "6": "6159eec66b11fa6741de0704192620a7786290a00ed6abc205b241981e3ee153", - "7": "e76ba89008060a838f26f485302891dadfab7e40ad6d4557481854ce2aa51ae1", - "8": "12f3d5683891d162b6b942257a2614d037cff5289188955078427d7ca251be11", - "9": "ddd096839369e8d8d633a1ec8d8d21df9e733dbf4f3652db08a6ea94b2e8c1aa" + "0": "6f3d7370e78f1dbdae597abc516b6d19f43c4f608f9188a9710462d37b1747bc", + "1": "fbb60c6e33a0531dd76f250e3ae8ad72c7ba2b1742772ed7a286a4c7fbc10e1e", + "10": "6c47202dfc70e77858d01a53b94b4cf952bdeff7e389f46d7eb6340b71148be8", + "11": "908ae1ded8ac6530ddc5fa761321fab86a3c027c64d383b2024bb8261f09a130", + "12": "077699a26a1f458fc9046955d1a34521692df5e0992e86ed4a1f367f820fec24", + "13": "e838a8e80852e5e3f20cfced3e5b829cc94c21a20be57a2bf685ca5c4d14570e", + "14": "692d076c8b0643577996fa3e13b3aafce1309aab75c4da28cd8edaeb1f431964", + "15": "21584b5bab475b3c8a85282e6f41d56e4a0db9aa4a85118593c98d8f8e8f13b2", + "16": "0786687ac2cf034ab3082a1b994059fc45f25fc170ec7abab89f7f0eec26a870", + "17": "83898d9d553b5f3e7eb784cf85b7fba8e0406f1619d38fdcf510eadc4a2670d9", + "2": "9b7b8d4ad0d7c67143c5887f561e08182cf9a095bed1a76ed53b406cbd45f768", + "3": "04366ee133dbfc67fba244a8d80ef386386c288436c373fd2ba3fb34581287e7", + "4": "f7627f88969eff7df1add5b6a54937fff1c2eb1f869cf4db71f51ab1657c72f8", + "5": "aec8d1aea59b7fd151266b0152bd10ea089a81f19b631629d94afed8468787bd", + "6": "7017e1ed71a580a82b998ec4a2f9fe6ffaeeba3d73d31259de0f604d22ab072a", + "7": "dc94853ae5351d5d1b20ad86e4ce2fac67f72d6c7b44adf379f576462692e323", + "8": "35a250041ae4c82cc94410201cb1517428b5d243e495d72e4df2ecf3127389f5", + "9": "1d7a4d4084b64b417d8e401d2fd5c6ebe74aef0c4eb791dff2b67956cb295ec5" }, "gfx942_attention_tiled_2d_stress": { - "0": "21c6e10bc9413eda9002487cfdfdf7b1858d2e7db132f63ea2f82f378defd4a8", - "1": "14283f82aea8538ca45f6230e0799a4d1fdfe24b92489f23e2bb6910a009ab05", - "10": "edf16cc47c470a67e830f7a8d18e650727d7db8bc2c849bfab9f3e53fb9a1aaf", - "11": "de72516951cb293869e122d2906b3218d6b6ff4b4535e90f561f3479d4a7197a", - "12": "ea22a5fd9ebe984cadd8650aefae4720eb207ba919d8888b1e9a43862997c26a", - "13": "2c2b2341e626fe15c7e4bf9f6ce15fca4274e853b5fd9c1bb9f3e155711e7b42", - "14": "b940974fc9458b0c985d61e2f5070da045beaa5cf4d11eab47d8a08f68fa4fae", - "15": "41000f4b84586d446a1a35ba0b54edc909476c68b792b571b44ae3245071f37d", - "16": "dd54d5f4d602688d2d2e7ac5d5ef247d6ccde2f5c9e4a55b9b261f0ddebe5360", - "17": "4ad8fa35eff3b2106b5d39ba777ca854a2aeae00a3da202515bf115221fd866e", - "18": "284d45a1cdd43fe72421326d60366e4ce51d03c8f50467d9b0118981e59f7c76", - "19": "740bcc00f33534964475a92e01f29fc62f8d0c262a181fb47330fcbecaded9c8", - "2": "ab8c7a68bd6a2294e936e52c211dbeeebd169189891d4e2b13431424400d469a", - "20": "dea89f3cecc634cf6791b884c1e178e408e9d9fc44ed31a1a5924de1ae8be79a", - "21": "e3f8f150f1e6b832924db47616551935cb6710bc7637f8f8b85d7b65dbf3bd89", - "22": "68c05f7e2633d82a79acb526b388257b50619922bb80e9d6b2feae6452720a78", - "23": "140462db6dd4c8d0b1199547ac23b66986d7474cf4ec5ea0ec5da98dc4b758bd", - "24": "1ecab1acabf3386f1131ee4c8860c89c79be5a256021bd9a33d215a97857d6fd", - "25": "467c2591e2ca05faa6bface9374512f09963fb9d0d18b137fd4b3347aeae6afe", - "26": "79e1f1966042f93a09af07b1b84124a3dfd70b7738460aa65a77f5fdadba11f5", - "27": "630700b61d4f7d8f3d8ed99410f093975cf1f44aa7581d6093be75c4633bc19f", - "28": "8a54b30e2bb12276fe80f5cc5cd2dd0b1230743ca806320ab6fba455f5b99973", - "29": "1f30105b5b6d14fd1bc4b262b55e287287551f46cc4f9bb23a7a8512da447c6b", - "3": "cc0accbf3651387e869bc43e478b14bc2075edfe7a6509576555f46e7a3c6091", - "30": "668bb2163fbc39ed5ee73e463acdbe412b7c6d94a296fbafef7304c5e084c42e", - "31": "3279259d6ae11b04e5748fc2ae2189ec15c6c2d910eeddb6971a787be01b8903", - "32": "be35babfd60fd2c146a240f71c3620e984aca31d157c16527d1dc9e31200f502", - "33": "a4a9cbbb9305812bc863f62456ed32893ee1a50681a3691420386a91cef4befc", - "34": "3fc0341d37130f8ba8661cfff93330801aa676176b4b35bef523bd15b43e8573", - "35": "bbc316d9d700c0fed9875c75da93367514d3a8e17cd11ae94a254dc64dc201c0", - "36": "39c8d22dce430e8f99f8d1aa6c5d2fa6cf7fdde711d7a2526bc8fb01ebef9965", - "37": "d738b16fe14a906d196f19342c4a6fae92e11cc242d3eb0d7146d7ca60f1e9ea", - "38": "aeade9aa01f8210f0a2c0126a3c6cbb5979ff81f1759349a22af6780351b452c", - "39": "99aeaa6ef49aa526067aebcd1878d997082e9398895cdd3b6197762285faa22d", - "4": "0ffc7a1684de24381ffff856536783ee7462f67c9cb385c760416ce05ab641cf", - "40": "c81decafc2a2e8b0b6ce1b37a8bcf63598947e2c27a9f8c7eb2d9d279d8847ff", - "41": "14f184287b90b419091d9554a63a583b51ca998c26cffd5a8fbc74f460fe1301", - "42": "f809069b3dbde1066ee941a5e19e32f34878a3bf8e0d5e8825624891f8fbd2ac", - "43": "0e9a7b1f7d1258e317a4d9140150464296a8fe85d53c3006ad01c30ed5b7566f", - "44": "d0f33e651764d8dcd437cafbd785d43367198a2b12947bcae0c8c118afbae360", - "45": "0a58e7baf63ec51ff391189633f52f04ad2d481eba02a5bfc9bb91643bc1acff", - "46": "16bb5ccb1bd0b5faa241ad86edb2339165023adf368b59317362e47293885bf1", - "47": "f56dff231b976b29921ed1637ece864685414ad8fb9a8d6258f9ff7ecc2eaf15", - "48": "472ffff9cea36de7cca3092d2c3e1af9e3cf2a2a38955f8443452f389f5059c4", - "49": "8e92bd61b6c07e3bf46fe17e8bdc909ca47456609bc64ccf1b9a104a0438420a", - "5": "42ad7c8b61ae6131fd82e30c109972caf1a282ffe9a0f7aa7296ab72bd38cd81", - "50": "fc546992f4d90a651eca567c9cee8a819cc395f31c36619ac55574d261f16685", - "6": "754486e716d33bf57c93e080ad38a12da70fcaa3d45dd298ae7a383760bdd5dd", - "7": "65f08a82d3c11cc9cfec54326f0415b50a69152b33511bcd8ab41d17a294504f", - "8": "4574dd9d17466d5b2058d0c4676892ce6e15e3b5f4ac9891c8628a8ff86d30d2", - "9": "d623775900c06600492019ae7297a0d24451258f05c3d53e9fad32ab4add280e" + "0": "704facb4229d18e773af4a426154e468313a93a8c7ca61d7e829dfaaf12b965c", + "1": "91350246d42c3bf1e407bee78994d4b65ef900659dc587e2f752ad25facf221b", + "10": "4b97164242eb34dd8081486b709d96e6655f1a4b7274e788ef18f11311710bc4", + "11": "dbe31881f0a8950889a1074a9064cc20b38e2be7c536d98d5f4482a135632af9", + "12": "63b42caa419ead7f474e77772e36ef4b1f7f707b1497accc07a055d71ee9aaab", + "13": "8677004a8bb5cbeae5c8cd64b0a69856c890a4efa167f745651fe38e3791313a", + "14": "4eaa31fe870d4d828cd4559bf7e0ec29bd831ba51d440ad6054dab45928cfef1", + "15": "3caffd13aa92c1002d7febc24604409d19723aca5f84cbd1ee5ad06304286af7", + "16": "704facb4229d18e773af4a426154e468313a93a8c7ca61d7e829dfaaf12b965c", + "17": "704facb4229d18e773af4a426154e468313a93a8c7ca61d7e829dfaaf12b965c", + "18": "704facb4229d18e773af4a426154e468313a93a8c7ca61d7e829dfaaf12b965c", + "19": "0bb698f1e7d115fc6e1b441348c9bc56f4a23a77fdc70b0f4f6dced8d77e5ecb", + "2": "6f3d7370e78f1dbdae597abc516b6d19f43c4f608f9188a9710462d37b1747bc", + "20": "84e4242ce5816d3dfa10bfa1ffab5095427764f228bfce69ffa758c875e397d9", + "22": "2da29508514cb9e860c70addc2e17ed2f45e4d805524c181679cb16c68e427a5", + "23": "fcf1ba8c03dcd55dbfdc62779924908131a15b5723cc43efc1ef73b5eab6cb4f", + "24": "9e680483e5d92516aeaa918728a6fa56cb0e642f6165dd0b1bbc22f6f1c40b12", + "25": "636278e3bcc7dd2f2d1bb4aabc1bec755f17a31aa897623e5fe6b65019a601a6", + "26": "f57dec7baf9b8edfee5c5c22a777111a87657c65e8a629f3fdd40b9f8d94c055", + "27": "c7d7d66e36b3f9d9ee723fbd831c2ba6447c24d2eb16e72687eaba4822487b54", + "28": "c3520f9961126e79b526edcef3c6ba54002e78945e6db6dd56d4f9c827b88f84", + "29": "01943992ddf17632feab4a07240fef6698fd2479b2123aa2b691a27d759d69d1", + "3": "21e2d1b138b0b26f314f1f5fe3b405bdd6a26f61af30cc35f3366bd83fab3b5a", + "30": "433f6839f693ed8e4f5b65532b77cbbef4b796a7bc3b20b79600defd185be31d", + "31": "365560866a39ba8115ae2edca41269c94b1f31a03afabdff14257fa0500c88f8", + "32": "704facb4229d18e773af4a426154e468313a93a8c7ca61d7e829dfaaf12b965c", + "33": "704facb4229d18e773af4a426154e468313a93a8c7ca61d7e829dfaaf12b965c", + "34": "704facb4229d18e773af4a426154e468313a93a8c7ca61d7e829dfaaf12b965c", + "35": "4af7e153838bf237157fadbab6587183ac66aef78f7e7f2ef2a3161ee6fbba34", + "36": "2a03afd42ec30b90d72c9dd122f1f3bb1acf65c35e3eefb387937e50bb2ea749", + "37": "71857370a85adcadf6781afc676ad0df2890eb36e802e4adac77110a26a6285a", + "38": "aeee3c7a2bf24c4556c47c0b44b896bcae528a25c53c3ca20b134a94cd1682eb", + "39": "05f637882207741e14d210d2b2fd0f49ecc2bd5b5018b28a81fb72f4252f13ea", + "4": "886a566c4bb12017bfa01e8a1215884c1236ef63532b8ac2e9d76d2bf35bc9cc", + "40": "e652634ea96af529c8e6a0679062c138809abecad13c7134ecf34f0a1d55963e", + "41": "3519497de385fd7b1019d09f13452314dd6cb5d67a935b31f9e89ee2933278c2", + "42": "140d229863b61a0578d708959bce13524bb262e628789c9ed40a2a1d1d6c8072", + "43": "91962c3cbe7f4fa81f056304757ab56dd25fce33db3514684c14bd46096ad9d6", + "44": "a30b80e73174c428397d48b6736554bbe278f0551937348141aa401218e5329a", + "45": "053eff8019fbd21b3d803daaf29259adc5a7388f5989b5196488603c1f75fe31", + "46": "0e2335bf5cf2b536fb2c9de258ac3c401ba47412d921877847e9fc0aebed084e", + "47": "e58c9a9e533e0a09c1fdb5d06c7ca5df4bb5f0b588dc3dc59e5e62ba50dc5c4b", + "48": "a30f1a41e6a580e0ae30e1fa483ca38f681a48c18d32ab51e066cbf584c57b26", + "49": "255b123ea3b780781245ec6d2ccdcea07b4d34d7bde00cd808a6e7d23d7736d1", + "5": "3c5a1aad52b214e0c3ea3b445cc3b7155cba25c5565a45bfe76fd5aae0b9e6be", + "50": "8d7578c73c1fff78416bc21314a7fbfed926e442852441a4364e7fe2df73b90b", + "6": "3f27a5f7c46771ae634d2278d204b715805d7e4e2bd7f99dd0b0440cc2adaa87", + "7": "fbb60c6e33a0531dd76f250e3ae8ad72c7ba2b1742772ed7a286a4c7fbc10e1e", + "8": "71a71530a2555af772e2dbf6fef58a6c01e6d4a3e003abc0d3914c5f14d049af", + "9": "f7627f88969eff7df1add5b6a54937fff1c2eb1f869cf4db71f51ab1657c72f8" }, "gfx942_attention_tiled_3d": { - "0": "cc3f2041d379a84dc3bf7b6cce098b0e09de51e2a53da24b8b005b76a88ef2a3", - "1": "0d08e69fd75f45e2213afabb749a795ef854d57fbcfa282185eefc167c6ecfdd", - "10": "b156fd210ce4b5c5dd7fc1acd1713dc8938fd1032387a061178d9e0ef7d165cd", - "11": "bbc6de3c202c39f2b5188698176f94638bc5489527b30e13484234a72e1ad6fd", - "12": "31888b625d263d42d838c7193cfbb8fd9a795aa565a8f5616cbad167158fe60f", - "13": "4dad26133f26869a0d87133e35a2d3a9f0d9732d763852474a30851882b9e453", - "14": "61a575de4d2bf56076920f7a9892b8809b2d1951290e229f560b5cb25bc0234f", - "15": "cd9141e0c9f06a5b405d0ed1f19625eed22c8a594a9c117aa97a6a20c66e8f69", - "16": "9d7f10dd845a8b7399a4dfd6ecaee077cc9c3831cc9e618be75a711cc9acfc4a", - "17": "6302c517f41d7cfcc32f3b1cf3843a121b25ac90e6451c0c4a71d9402f138b95", - "18": "6bbf65ce27b55559b83aa0de86b36c691b3792997081f047a48e00b575c974ae", - "19": "7b0041c22c25865cfb15ffe70640780a5fa79f5f6fe44781bd9858579d36b7eb", - "2": "56153f0bef9d8262404f7d0887de981b0bd919840bc2310b6473b98314df26aa", - "20": "a57d936897c12305b53746e75e566c805d8fbbce9d5eba992df17c89bb759fbc", - "21": "a29cfc841cbe873fd324a01cf60e0c35d255331577fdafaf40942bac9c35e05a", - "22": "b272e8028eadf023648b3cfbaec0119de8aa4621326a02ccdf2956c691e29997", - "3": "9398d2999e4caa2d1fa3ed7be10683b884fb813f22eb0404bb9cf8d472575904", - "4": "e5eddf3ecfd8fae5d62b08af12677b6d15fd9a1111225b4a08ea828cb12d0905", - "5": "523279f91929d6188a955c2490f810830fc15abf00df25dc055c093af8db6da5", - "6": "99e272ef0dff025b3e4b1afb9d3ec8de09fbd6b66ea2d2050755a3c854bbfabd", - "7": "ed5f5549086d0f47a9cd6623d625babf5fb2e6dfe6b5beef8818894049aab80a", - "8": "6e2e0a93bc06db1b2d9c5cdc6af832c212f04af161a6b2efd1197179b42bd5e5", - "9": "cd075c6ad036b6ca70f01126387f4bfc25baa6dc4cfb02752afe038ebb143850" + "0": "ad844ff28e6848c399f7147d29c5620e0546354621463ed36e76487f4cb982f4", + "1": "63e3953c0178ff760382c45c2fe3bb979b2dec3d440b5c143fc4ae45b8aa3683", + "10": "04bb65a8b60b9eb14429a482e53646ee23e16660b21b32870c802975b0c83867", + "11": "6af9bf0abdecfc56386a4de0eeced09ec2d6b52410aed5f5575455a869e01627", + "12": "a74a6d52055c3e08763f5f504ec70fbb84f5ad0bd790da4ddc7f6879933ec4d8", + "13": "05166ad2f3fcb437afed4f6e00553696f72e347b18df0df107e3aa2a5ceda845", + "14": "6b69a670ae8eaefd49e4e05d666be63d3bd4df8cf92f810f6dcf2d29c8950a8f", + "15": "41845128ddaff2006010b88d387299f3fc19b3b65312d77c07ee86953732c1ca", + "16": "c6ccbad3a8d6f43c723fcad24952032adcdbe227176a3f9b8ec9c7b3e25d6ce1", + "17": "a60e6a0999fb68e771ee6be08163ea410e4af178f7c8597382638f8e9194f61b", + "18": "731760eab373994b8d35ac82b148a2131be55d20a113b9c1618b756a0afb7eec", + "19": "025da440d7b247be168623ef962637e2a756e2f622bd3484363b5c0bd3b56c9e", + "2": "835fcee389e09b34d06562c485a598d812de675b4b61df933d9e925725668227", + "20": "eac3983ea702e6ee9a2c410d0a511f0fefc8a9a67e6004f023b455744b79cd01", + "21": "1b1a61154796016a3680a4a5baa59ab86fc9a806aad84d9c8eb4b352dcdabc83", + "22": "977161b8956658d30d658506d005addf3da2ee8853166d20950a130418b083a6", + "3": "c8bc9de015a452c5e154a9ffe4346b93a9fd90e6093b31e371efc6439c4f4749", + "4": "9dff59a8af03107f77a726caf069ace2298dd58cfef97779e1445dc92e14e0b5", + "5": "b32a6ed27c303eff3d023e6a1872cc7447f9eeb2d576bb22d2e78c4104f59073", + "6": "0763109a0f0ecb853339e0027645a591019a6c45b3ec4d8624e2ea1070c32830", + "7": "b4fd3d0ceebaf0a3d709b162472eaec7e02b8e8b896274cbe302faa3bce10595", + "8": "f0f518ad50fd102173d52d1f7f4096f5bf91a9e103837bc4e69a62fb7a49b4a0", + "9": "de3ca04e9008824143d5e906a0542a32367d1402fbbf49ffc33dc9ccaa0d333d" }, "gfx950_attention_tiled_2d": { - "0": "06d6398c896f69f5c7f516efc523574977d016662266da31bc987eaeabe7b8c4", - "1": "abebe8f6ab8d2c53d46c2b7dc3570f4950e2e88713eb5c03ac23a6ae994290e9", - "10": "beb202c65828098e7bf303e05cdfc4627e49a7c09392352fb8d436b7a7471411", - "11": "65064f54b84bf7bd4bb1857980d72dce87f2beec100b9824b73f2c4eda876731", - "12": "f13993e343ac9d722c99a7704b114f7c1bc429b4a3cb126d31a3206bf98f8032", - "13": "3458aa5ac83cc1a496e06018bdae4dbd9dd0e08e27b1ec72da3b2691b112e32b", - "14": "628747bfa27f831133d06b0cb50da25be449261db88a8f8b7a3fb8fe27140086", - "15": "b91f7408a957c92a528a9ebaebddf7eba7ea75485169b31e069237cb64f10c1c", - "16": "76b1157c1bdf889f6904d09553f65b8a9500a94c785624c8ab98cdb66ae044ff", - "17": "1cdcbc80e0669fdd9202f774f844346b9fa94bd562cccfa4f16ec5f85e1240ce", - "18": "840dbb64c7ee1cc839bd96cefae9d383607bf288636b18425a619f07a0cdaa64", - "19": "4fbb13e8a92c80510dae28a6ca93ddec991fed7ac83d06ccb28d494805aaf35f", - "2": "2365a2ced71294322eba96dcdb6b71d8ed23a9e07b04e745b3a9f98b4332d211", - "20": "882afd670f7dcd99856c8e56e33b4869a91e8969939c80926e3c7fa1bca79caa", - "21": "7d5c5aad5fa717b17aaaf0d156f9aa74c446120d089830c5ea2b82dea15275c7", - "22": "17f35f39ffa168849aaa764df20b407e06bd13d5c8af0869bd8a955e4ae5f2f3", - "23": "33aaffc0ea1de09eff1de40313695ac245fe3e9f3a9d46d72f37e7b19f711f7f", - "24": "be202901050e2a5d9fbf4ce3a1e18e80ff4b282e3dba703fb968a5f08b8da86a", - "25": "ce7cf62667c15ed57bb9dee04b078d203be961d99c06177f92fe8d6efafeddab", - "26": "58269c2f6f5c4f4d53bef63c6308731f82b8bb22fd0943c392c6e08f7141b040", - "27": "26be3938127ce61289d1f4e4ca2a4a85f21c5cefa493b4589e9d77fc03fb5b81", - "28": "8cf9fab3c4f73fb1681eaeb22d338a3fc4f881d205fe4ba074eba8dd53844b07", - "29": "a21881520667ecfb88e24441e5ddcb75a814a8716d869ea5493dc78237add346", - "3": "87814d4a7674e39e2be8e781b0c1e5f0dfb8d4054008b083cfb141a8953fa87d", - "30": "fe9ebd505134843e5de2037c561fe60196ec59446437cc6b6342f7ee7fc810b8", - "31": "a0a0be391d423d33b90c2cbc63d85d9aed8f464e8f603bcb756777b52005aac3", - "32": "a067a844f8fbe17bee7fc1cd0523a3b36cb974b3fab02df29ea581f9596dd251", - "33": "a8958d4571a8e2e8ee199ecf72f51f2c0ad53105a1ec154bacc733afe892588d", - "34": "ef4fc62aa7a24a6992219ad4b42e2e2920f87e4525bf8e0e4efa895b7564edaf", - "35": "b974000f49b2a1e918e2a3e118b43272d895ccaf2f510e97b570b20d9464d975", - "36": "c6999f495b5be8f308063d28fadd0b7a328b5ee44c1f6e0f047f8a2ef7acd5b3", - "37": "8c355e01721d24dd85aa6c1d5c431add07f9ff58f200075574f33ff8a2f5e2ae", - "38": "a850cb8de8260284350619559a89da52f1f1e38dbcf7853e1608a61dce185207", - "39": "e1330c5adb842284ff1c74e509265750b4c906f617691259fc94ef92e28aeabc", - "4": "fd580b4a7edfd1303fc34ebddbdf311b2206afd081a2f9e17b2a7268dd6b165b", - "40": "59093a397f21f1a8921f85b4125717a92d3ad21dedec8789006650852843f4fd", - "41": "2caf3376148486dd0b632a2118f3b5c7d1d24504113522d404395adde02e6440", - "42": "42e442321d99f07de2be545d538f3dca87f726c1e70ba72693b029335b922639", - "43": "73139e01e1e6d8c7cce7418ff6f5bbc3baa14bffdae91c21ba655f091b01b4be", - "44": "5f256121e9618e18cf5c946c6a1e591477c4086b53ec1fd72b3617a8a402643d", - "45": "44d8cbc5bef1a6235c8e01fea0322f72ec447922c36c95de604df34f98a8df18", - "46": "2dabbc9cf717e7ae301c355321b708fd066e219c7bec0ced5f2f8801aebbed07", - "47": "852b640bc62c878edbba641ade0376f5a30d2c2b37d5677454d7d8fe046bad8e", - "48": "7ea6df7e6c4b575eb6b7091156ccdff0c76d4ae3e6471a93918aa0a47cd64f03", - "49": "022f4b19b4010a5bb59b430e38b7b57baf290e5b154eb71f28e761147a18cbbc", - "5": "566bd28e3632a92b05dbfb2c2b6489da9f03195823adf7a27d55f4d6ab7a533b", - "50": "09eafe607034045a1db7a0e76ee367baa023b04e749cdf207c7610513abd9572", - "51": "6096e4edf254883676ce811c6d2125730cdbf4c4b4f9b28b1a64550a2cc0aeaf", - "52": "0d25fab6e9029190faa5e0dd69f1e688e3f1482ce114b29d93df6146212ac55a", - "53": "4511040ecd003129629ba51f030906b98d35cdfd4e50f1bb981d848797c0e786", - "54": "caf014e9bad532cde9c3820ed188cdc6ff56df20ed3f8d1796a1a81fa14f210f", - "55": "5094d1472b6fce2f4a40b866fa8bcf69b622b2ff03c042b8873b9000e2f03722", - "56": "4ab929aa37ea954f96061bfd39f69b257d28df223a2d3639c28bbbf8fa8df2bb", - "57": "01ee67b9870ecfeffa6fc08254b2a020e8fa52b9f44f3023106c1b008fe8cb78", - "58": "1ae2a5082b3239027b5ee3f4966bb1fce09da5c1f04641449cbb1921e038673b", - "59": "61f0fd6ec24968d198624afcc3dfde3388f3a0aff90149bf486f33fc7fc8f47a", - "6": "fcf6cc803feeef666a17f67b2586ef47892006249282b010501f61387b012731", - "60": "7af2f7c16e4923c5dc776e883cd18d1819fbd7a36c99a026a1a8f7a4f7bc78db", - "61": "1c2c15cc564ea3b516ef844207a7ca6f544fe153e8287c8619b2d6c8d6b04e41", - "62": "d1712525752acf3543932485fde5cc6e5f630128030aeee43f1ac94b20c03e95", - "63": "9a5c210e255be4934c64c9270927d6f6aa5e5571abeeb7575615ac10849269c0", - "64": "3ce7d597a2e9ecd6f1305a68f0c46e354cb12385e23a0a39e9c9f482481f76fd", - "65": "fcd7ccce7f6e9d174a78dafe3382e668b488c7be715653ac3308dd963176d3fc", - "66": "e1b19694eff3848b713edac16d2345028c19d83db7e7507edddb76c50fc5e2b3", - "67": "78cf352962b82eb16e4715a8aaf54058c5570f8c8b90270afac0e48abe4ebaa8", - "7": "44ffee499d2dcc882686fcd777f60b5d3a2172af2d755346511f10a0cf4fb5fa", - "8": "91bad86a927df04b143f7b67d968bfd368d9d7e49556c08e275d0ed683ba7bf9", - "9": "2c5b0d54477dfbd31b38cd4f18c8970199d19d28614d975d303ba38dd99a824d" + "0": "67f9e6ef716ddaa4f8f1291a18e9351864eae23e51dd7b5e12842f119387624d", + "1": "c8a4fcc7d6d87e36e6ac0d891db089572237b009cc8da6ed2ca2b46661140220", + "10": "039d44ac095fa880c9a83a5aef8636c8fa96deb206da2f0b60ffcc1f5c971673", + "11": "cb20a775ae57a92d2f6acbf3ad3f2741289afd20d965ca251f964acff3f044e4", + "12": "277ead55bf9148eec08e238e9ef10d604f26e5fa61a59a564b68bf120dd29e5d", + "13": "73fd1874b113ffc83c1a5cb6fb2341bbc13976fff35abaa5ac2b8dc358753bc6", + "14": "4da12282f5e100e38c220e9c33a15204ec3fb9278bf91a6906fbe11044208360", + "15": "250d336296f9e156df243bf40332fe66f747fc1086718b66e62923cc435b2ac6", + "16": "acb77501dfd34c2ac0981065e91b6ab112ee15a23701a911c0d40619a6745536", + "17": "06260311a25a8c0acc36f44cbe1d71f7f520d4c6dbc06739ab2bae639e4138bd", + "18": "82aba4adff64a31d8df7e885675b04f99f25194563ba5381c105e29c1f8ef10b", + "19": "ff42fb999f0054277178a93865b9a72964ce652915a9fbbc91ec692fcc8be2f8", + "2": "8a5996187173d3f60d312d0f6a69bc5cb38a4247c879979085cc9f3d1e732ae2", + "20": "9af2e72273a2e5dcc660b67e7f0e11d9825f10a683baa8ff3deef3c544d2bb99", + "21": "6d5caa6f21da5379a232f551fafcd8a0ddcf509f531decd47e0c093830e98bab", + "22": "5dbf3f719133ce1fc9f154e9f2418ef99b6de240c5f796eb297e3a1af512e851", + "23": "15ea756cb5d241a9ff2cb11b282a6ab4aaafa2be4b504d2a0293e3a50f392c6e", + "24": "4beff7886e03be07d89dfe4e4f920b56f8bbabc89544446bc536d94c6e1e3d8d", + "25": "cb20a775ae57a92d2f6acbf3ad3f2741289afd20d965ca251f964acff3f044e4", + "26": "cb20a775ae57a92d2f6acbf3ad3f2741289afd20d965ca251f964acff3f044e4", + "27": "7f2dd3803ec45303f02f4f3cf9bf83cbf0eb271eb00393243b0dd6f6ead4f37f", + "28": "01c7b8a3bd7c943889de20f15124dc69c0ca02ff65c98bf093f53e0839684dfd", + "29": "202211232550b0ff06d03eef660d64b8136bbfb62865a96be9aa025b1a3c4603", + "3": "e9affe1d0a56f738970366aeaa0874d0ddfdf7f750631093f9fa5f0a24d3b286", + "30": "1a2afd613bc416bcfb2d8e102dad6e9dbd4a9ca009cac2ec507cc172585b2642", + "31": "9f51b866d5407538674aa75c3fa9ae34907f27cb8e37c1d2704ee5e37389f1fe", + "32": "375345b5198a13d6cc51dfb1fd2539752fb9b4a07795b0a5d8c2ebf60c16cadc", + "33": "fc6667d38d7a960365b30860634c91c0fd1c7fd44d23ec623697a20146ed0425", + "34": "bcde5025a1cd0ca74f6c2406b261bc210d4e7fc62cf48640ab456e8858883e8d", + "35": "f664a6bf4b6049290de05915c8f640f74e3b9805e26efb08dda385d0144aa8be", + "36": "97943864c3cb8d9c232bd97c65b02a02d7026dd3dbb5b25218a6631978915f2c", + "37": "342302c5f25421175392d3bf0b430247094a8c93ef6ee87de2e11957f3d54365", + "38": "bab4e8200ee6bde3e6f7aa8394cbd3d10e6f1a06d074016d39a98fade67dd059", + "39": "084e1c8609a0ffc5cae837f4c30e7134669c986a29c25043c4e26de31b472d9e", + "4": "bd21c8e9bc19b8b350f10f079372e1d4a831404d533eacdf2d0f598dcfa5c03a", + "40": "3caf24ee22990f4073b0ba7ceb06663dac682d21669f716e59ea191280641030", + "41": "d1503b821909470f6127dd31476f16444f3b4d7f07b7dfbe28d9104dc105d5e6", + "42": "7ceae3f56cbccdc73905191a119e1fc87036a39ee7552d4fb9324fdb201689f5", + "43": "991a6c11933e9447a983f513c631b75a5c86bdc60c06fe583958a532320f8751", + "44": "3d6c613e37eae97c4f8c2ffa4830598a6f957e4af8caf610ec1b87ea11352a52", + "45": "1018b5d5a13725c92c8c89a70870405755fb6621110e11dc6eb22d1099f63fb7", + "46": "05a486f8df2157612c62cdd7538479c72fc50c07f18d626492a5e7010c420ed7", + "47": "56c68cf50e02e981712495a1460bfae81560fd7bc76b748b27f5c9c5034cc3eb", + "48": "f79342d6ccaccaa48fdc6b00500df3579278da2d6505eca6a1a8fa6529ea17b5", + "49": "5f3d605feba0c10ceaeced27deec8fa14ed3623fbb2788a32a36fdb89c880274", + "5": "cb20a775ae57a92d2f6acbf3ad3f2741289afd20d965ca251f964acff3f044e4", + "50": "cc659f196ea51ad59ab55343c08a8d704347fae79fcb271e12c386fcc49d36a9", + "51": "d6dc2f6fa59a9df0da679030a095819cabec2e30cbbd4d61fd8e91a08785e9e0", + "52": "0a4b6aadad85b4ac0bdb20f01b95d062946d47b80406a9ebc17794874a54788e", + "53": "37c52193adf9457331c5be183f693ea44255ebde98b37adc4be8b682efc3d0df", + "54": "231f98b8b174d680c2cb4e1f73086ab67e99c73e2c67503fd2f184d25d8e0e51", + "55": "ce82af853fce8157326230c4de2a1c8c0e27206a01c04f290177a4a4409df90a", + "56": "c00760ee6bd5679c88fd69dc2ddcc22201a24ff3bff4fb2673a5f12d86d6da91", + "57": "895c437684ed86c3189309885f171bad515434eda8e7b736e6a458869e26e84e", + "58": "42bfc3b17757a8ea5dca8ff16fc2526bbcbaa1117d5f1fd96e5851f39c08d1aa", + "59": "ceb3bf9543024994611b6ba66ad2b0c45b030933a4fff1eeaba91c680efac000", + "6": "7f71cb0640a6a22492ff7a6e205e6aa80b8a70bca105cae1e604666fe7db8004", + "60": "88906d3e51b487d3fc952d929a75c6ec8f883ece0fae8d8dd28ead61ea958960", + "61": "bbb06b3cd3c6a2be58f6df6d370b287a3ccdf830b34d34b4454451f452e86d95", + "62": "b6a81000db2076e71fd5976748091d59f2e85840b35bdfe766e30f0426b67199", + "63": "2fa7e7fcb6ff25b06011c6dcd277ac6999b007e82fb8cc944d510461fb75ca8e", + "64": "7d4d413db8214217b3eaf29c4b76e65983d84269cb78e39f6d1a6d9ea05c4818", + "65": "bcddc18990534aea373b4f03639610a110c14dabeec44b12c6158b27beda7841", + "66": "9a19cd00d9f2d6a2f24dcf287d1613a7860bf0f9903722d97964d7655b66c049", + "67": "69d7f24186153f6a15ce887865e50c1f0f157da37bb41a3fba3d36d4126a4008", + "68": "7f3c05f5adfb9bb1639b3531ac98abf38b18a0a2b950fa4827f1dbcfe6b9d615", + "69": "2045c5ff62259cd0fa0883a039ef937c7245952c9fd75520b8125f1cdd335ef7", + "7": "8212f1103d5177844f6a8f6ae2e4beb73c28486bb07d975da6aa94cbac407c41", + "70": "6a08f23a301f5f3025c4f0aa673f1848a6a91af57b81f853f3150e347b6fd164", + "71": "b8eff71e24ee9632e4c37ab8c589975411b851bc1abaaf0e47e4a2dada05ce10", + "8": "7d827babcb8192445726b06e4250deb674158b5a77d87ecdd9e873118dc0ec93", + "9": "5454f3658b3f3e436f29eb8a9f7184ed12eb0209155508c15d6d0b09eae9e2b0" }, "gfx950_attention_tiled_2d_fastkv_regp": { - "0": "8125c0336011ea91d7f3bee08710e09fe510034e1ee288bb11af2eaab168dbbd", - "1": "29d86bb72455517651361e3fa1bf1d1602b5a3ecd257c1be4f2303051dbca7f3", - "2": "251324d112d6c528566f26147e83d1d17500fb7b2d960c4f63e8a686501cd0bc", - "3": "38699dcda4a34bad75630e11de51cd1fd9105a8bc91967cd3e41897292bc6a10", - "4": "999975e55cbecdfcc0160fdb0eb2ae74b3ea84a36c30673580a464ea1ee2cf86", - "5": "8a030aa0a4d71c252cfc14adb5984ac2dbecd77c3bfcf1da1b9474db26cffc1c", - "6": "8c9b365749845364c858982a743c0b78ff94260b6a8d9172b3dc3900ed15b17f" + "0": "6f9c1088480f429bc5f275eac443b8bdeeb1bedc8113b3984c4abc00054969a1", + "1": "ca235f48fba43c86ef565d8e12710e04b4252157679d2d9fa3f3ee03f8a26c55", + "2": "ff01cb9487345e04a83f0c3a72e5ac536b45f6332fd3c9f8b88188ab5b165a1d", + "3": "f2d520db3325ca016969999eee222942a83517b46eca27a34c6c6f095e285427", + "4": "5b60a32e22ca58cf48c9b90583bb5fc543ad5e39b19663efd16c9ababf0ea771", + "5": "1ea229dc75d06f602cd796741ef43d8414a818c27811157d8cfff82f7b4f74fd", + "6": "069dc527897c969e165f470a3c748325fd7649b611ff6410bf52ecdf675ef8fb", + "7": "7f86cc3f74703c9e604a41d402585125cc423f53029b22fcd8e54df7c823d87e", + "8": "003664a49cf3529ee42b832f008358aa958e583262bd4f478cd5d7f53c922ca5", + "9": "43b5d6444c3e108e3c2e7b433161e731154fe6022ca16dfaa61506cafcd03757" }, "gfx950_attention_tiled_3d": { - "0": "22c3c50dcdeecfdd1706c2c3806786e493f4576336bd5960ed9c9029bb8ee919", - "1": "af5e9cd856f18112da7820b9d8a59caa361fce8ae3204658071db605101e60cd", - "10": "621a818700d6c87a14554b10f1d99367a4f3506b56b7dd4fcb35153f17f1e60d", - "11": "7f03d2d86ed48a324e9f40cf0da2a32e00d71bb1b1bcc6e563befd5014e63e39", - "12": "05b11a1897269a47d19a68662c4484557a920b1ddb4d8f8e482a3b2c78f9bf7d", - "13": "468671325ffc7a4e9d4d713fea1ba7373be4239e875b97cde89c95c8192d970a", - "14": "2429340adb1e6297cc6e3b7b8ad5f599459bf976d4f0e0973b7f4596709d9fb1", - "15": "af9726d24b47db9c198912f2d3f6ecfd2701cb3a720da6f3619d0949abd9f6bf", - "16": "6ed051c1af811770c2af798cdca13056c07fce94afc67d697e71f69796356fcc", - "17": "813cae2f8db1cde8de40cde864c8f67fb51e2e5b3cf3b5d0b128b306934c4695", - "18": "bb08fbbeb808eba61d61dc2b0c0be758a7ef326d5ffaf6cd25cc42c9d9a2381d", - "19": "ccbb69e46dbefc3b1b36c7d2a838206534a4afe856c29adfa96ff31f8fe7e558", - "2": "f0a777ee162213e55e259419d46d1791a975efb8fb0bf272d7536cd9d9efaeca", - "3": "f92162681e4e1b1c3bd8ff591baefe4dcfa3a840ade95e46c4b3ae47d3498880", - "4": "41bf0207a6966ba320af8c251bfc51a0791e82f3782b204b3cc1e07e2c265a57", - "5": "01ebc828573b3f80c1477b713a26688d6be326c0471338acc4e1f4425bec1bd2", - "6": "7ea68a213e1fd528c99eb137b420e4a638913c1dc16271bbe5d7a0f1aad4c6a4", - "7": "f208f171bb95f08b72962196681ca36bccf1a43a1d201d92dfcb1d5df79d1d73", - "8": "4f26d93dba745e47c56d09b5e28e04a486a07bb98dcc6bf133efa042003aca8d", - "9": "8cb90ae44cf07b9dd7f14d5c5bc39c7f82a9d1786b010dbf9be5ed204a957968" + "0": "9c2eb707c5e6f816cb8a8cde2043486839be7744ba14fab9ec1c2a9040423f3a", + "1": "d2d475c2f77f51a66e7e89e3dd8ff1382b24287a8e1f7cc67916972f71a2bcf0", + "10": "5b2639db7625c46c6f3674c99405dba02cc4edddd01de074a429ebb655bca80d", + "11": "4980053626a2642194170dfda1e0100b42233c3d326581255ab0c91a51c69228", + "12": "3bae7978951e52b791ce2e2717b6b14ffbc627e64bed3772c6bf3fbfa411d29c", + "13": "87a148b6f9549fdd82fa70d06df5b8bb10d9f02cd5f7926738ffd9558c8a6e9d", + "14": "9c30ffc9aa7961ea59c6131e97fefb8b2e2a1e65bd154cc17f3e194f41eb102c", + "15": "797161b0cc40f517d50369b8dd509fe3111e7e8985e96814cc84ff2dad60fe2c", + "16": "92a0078ee9b03c3406127e074aaa9166cdab27e3408e900f62c3cede46eccffe", + "17": "2420cb7ded3b220bb16597e6e2e2e705504e96a6042fef887e78542aa3b27815", + "18": "55acc7683eff383268cd3cec8b57020bd8d393d4a87e99982becbb1861f46d32", + "19": "ca0c841d75b1ec27c7450dd6c2aba1a00f927bbec393eb876af9a14583276039", + "2": "ed7166fa1dda465ceff4f41aae8d4f83234a4c267b63377a681ece2a09169118", + "3": "8160d94c1f921b2464387efb27f2697ca3b9a812cad6d9625bc6b68917f8c897", + "4": "f3a1153ea9c1d6e847538c6deccbada4f65ba44841ff72ed5712c50581465806", + "5": "88c518ff4d4c3490ac01bf456a8e801c1ba3fa3f399ac84361084b0e0bf82e4e", + "6": "0176fe74a827d2f881405f0ce2f7175979eccfac3e82e6f7989d4ed93e9bbfeb", + "7": "fae6dd54d2a3430fe83677da1710c609b951684908c3391e892bb342a3314f28", + "8": "33d3cbde3a27b309661b5605df4acc0829d7277d10771981961373d641a5d9d4", + "9": "9181d129aac71a08956752065d4746be5dfa024168ea37b59dbbc473939b2690" }, "gfx950_block_scaled_gemm": { "0": "e5e987f27ee79ef21e95d40eb956f71f784be46d0284512d94da6a08abf0b1d1", @@ -5543,20 +5561,20 @@ "5": "c8eff38fc6979cc723b3bd8e276884108a0655d1284a4e548bab6ecce69116ce" }, "attention_strided_kv": { - "0": "67e7f5bd0d8861dc16e6cff53316d780d40f6232716a5a3ea25f06055ac48442", - "1": "a0b5229967c3892d492cbacb9db2d15f06ce919b1f15c8de8d298e0d74306e46", - "10": "400f75d835f4bb4d37dfe0039c609deb5e172c77e5a83b87b89071ed3b0017bf", - "11": "0e441b0ae432165d00da483191b9470dcfc043c2bacd386b86fe6bef8f05266f", - "12": "03eb993466369d2ca4aabaff7880f23138bf08a0f71154159a0c9e4627e17df3", - "13": "74d14799531a8e079e822a46af3a22f1875a85a46c6c561ef26427fd44ffd748", - "2": "4cded6921b9951b733d1dbfdfe93015aa7b78c5bd5def3dfbe503175949af968", - "3": "393df7f6bb892e49b3c3031ff696d11addb1da00ad2088aa881ea37074c7db77", - "4": "db62e65f3aa74588eb95d2bf69935aa6331d0a57d42996bbe81c7510e146f119", - "5": "ee8248f21f5631e005750601f275daf2fd57d87f102a241c2fee76caa4567d33", - "6": "92c9560cfd6ae383a059972d0f4863e3699b56beba16aab1736d06bb41946143", - "7": "1b3cb6e2c817fce85c1bd189f7a7a01eb7e089c37ef1824cabdfbe5bd6915b47", - "8": "4b0b553b289d0de60dd9401f61f97d2f36cb201b6343437e28e5fbdc370cf529", - "9": "cb2426e5ff5ce00a3094ea4217df160031006f22cfe058568e4c959f1ecff086" + "0": "7ce6e33c41e2752f1863c4ef79eef519c4748cc9f1c3035cab5cab030f386d55", + "1": "38fa323beea8da252380810dbbe76686f3cff17bc15aca23c32a73c11f6a70e4", + "10": "59ba76e31fd186ba00c06d841b8a038c8941a8419e56b8fa46daf2ea9eb61028", + "11": "a30d1148dddca335071fee50a5cedd26bb5323693c278b14d22580d6cc5df811", + "12": "0ec51a3f92220ca3dad8979ef3dbe976f89ab95e28de5a178eaee7477661b8d4", + "13": "9bb3761cb7e928a0efc7789cda47a011260003ba29f7db4cdde89af0a0bda125", + "2": "5e39245a8752ce0229c9229aba2b11fc8cb07241e93ebaafa051cae24aab4514", + "3": "76bca1175f651389730a2856cf8bb9cf9d79998445425cde57ac37700dd1d2ef", + "4": "0ea423ae0c4cdd6d4b72ed754b1a18dab37fb3f6b9fbad492b1bcd036529c823", + "5": "29304ac08dddbce980440041340d863e3996b72fa786fecd5f739ca8c14bd150", + "6": "b3add7e87f7a74a750718e45156d80bc9a5725c33a8a25fadc55b71bb600af89", + "7": "94ea80d639a380a3cf0069ff8679e6f197fa1c40c9a4885413d747341964ef9c", + "8": "d8a78f5237a1c466494c27808c41590153f9ae47d899a5d6ad240510e680d89f", + "9": "67bfeb3341425741eb76f57e12f56c47a87bbdb73b1e5641c4a4f256f2647591" }, "attention_unified": { "0": "eb791027895b4a7db0aa0742569d6777b1331c559c87097d562f72aa2b6d41e3", @@ -6868,203 +6886,209 @@ "99": "056fa9ad97d50744d41853632710588920df00099f28006244673116483888d2" }, "gfx942_attention_tiled_2d": { - "0": "e7d4566fb615309b0345ccfd7c20287bbd8cb7a725fbf97ecb8ebf25c5d0b687", - "1": "87e091dc53f52b4666c1b46997e2ddca6e07adcec7c7c8394d3c9dba0aab263a", - "10": "9d4a2883a29cd2d073eafee6c0e6c72d7f85d6e5e430adad4b0215d315f3bd60", - "11": "00c3e90a002e86b091e5da4898991502eb7343ed9ddd5a1639d6a46806de1aea", - "12": "8327bf97823a7823a95536e84d14ca722bf5b8c32f7118d5f310d5ae1eeb98a9", - "13": "a0720903a7d354e085d4c68189279ee6eacfff930b865499707d59d91fe178bd", - "14": "487415ff6b42e500ee274f222930de09569f4d396996d9dcc1aa168e38798797", - "15": "d3a693c511a915d5706a56d83611d666a8b5a97746f59d124fd3345c012327b2", - "16": "ff028f89ab6dd968ae11ba15f0aa8f0ffeaaee3f16fed65e4f523cd03f137629", - "17": "b46f1819760ea0ccf0799759b1ca6580ca27127224e706f8f031af271a46fe3e", - "2": "2a322dec5ee9ada916ef06b47c4b0c555d1e7dd9e31142b1e9524ff5a4cc4901", - "3": "4b47445faee5557133dbc3286dfd8cc25751ba0568de4ad534bca1fd62d93a08", - "4": "0bcea1e69e4465b10a2554fe3dafd535544a31956c598f196d2ffbd95ca2a4e4", - "5": "e022360d8c005125cf4a375f274be893ea496228893e0eb182428de1ac7976fb", - "6": "8772ae5e0dba21cba23517661319285401e1b4ca0a5f6d8b4b1f6fe13f77ea4d", - "7": "4776e8d28fe58a4befb92306bd75f4ffdb2c452c74be4f48c40ec791260d5573", - "8": "8f579d55c882032526a27b20537904835ea0ef45d4b7d26d02cbc0272088a4a6", - "9": "1db87021061733bb5244d6fa7650ddf55407332921ce24d538aa96f9f030092d" + "0": "d2287389d440d609f01fcde8f6a5eafd6a52175118b1722d175a92044cd859ce", + "1": "4343006204506a5c0a08c5d9c1e4420adfc118806e241458b2496d8e3758b15f", + "10": "09f3f8a6dda2f5f9892a76040244308decef90eeb4594d77a5713332f487764c", + "11": "253b4d6b46377521d8301752b97f2d38bb3bbb0121f1c36290a009564f1b88e0", + "12": "f03fb8d960412fbbf96481ce32bf0f00f5af32b1b67bbaf7a3b8cb1bcda700f1", + "13": "64fff6a1232aba46257363c1b2aa3be6f5514c092e15ecc235e2cdc3ea42ff6b", + "14": "4016e10226975201cb8fcd11e0c37a5b82eab820c7a0a96b81c9066d0e2e3996", + "15": "5b9dcf6b30a3655f3c42dada636af5b4ee586f7b78f9ae8ba37ab5a0a79e70b9", + "16": "d11dad8902f8247c8122e2e2321155bc74aa009b2c9e148c5734eeda74a274eb", + "17": "8a89a840474fb264584bd0cdb49fea64c1bfbcde5a69b5fb1abfcd43eb0781f5", + "2": "55c737ed7f81060d5f9202dd7f3fabe25917c8497e7fa03edd921e2e552be9e9", + "3": "bc950eea70f3c80d7e64c6fdfad5c96ba3fb4da49a316168cfd1bf20f6b20f13", + "4": "dbad6405c1fb6e15d364d61db3ea7a9aae57e32c8cb64cd0fabb96862c823d69", + "5": "3c769ccf2cb2140c5659782fc886cf15bffa290fd1ea5739ca1389887ccf779c", + "6": "42ca2a9e7955596c87b29d334b563e51ef824635a47c43499f6175b69bb01021", + "7": "416fd50cde6c20af03f65e9b9a030b21dfb1b130561f0528fdf8ad5873607a57", + "8": "0c04ce3c73547c94af145fd9f1fc0be1445a4facd0a264683da0aebf68c8cbc3", + "9": "4c293e9bea4baa711fed44c04c0cdbe8ba950c4764ba74204498fa4fe7072c50" }, "gfx942_attention_tiled_2d_stress": { - "0": "4266276021f9b83e5c76f6a712c546877a3fa6ec123d89f04d8ca6f17dd9522d", - "1": "cde1c38c981ee52532af9ecd8335becf149b3dc01b91c687bd6eb502184e9288", - "10": "fc46983b7a750c4319f1b83ce6cd645b57bf953574bb8b3e6130e2961da10396", - "11": "398f7ce0c0b59c7b93c292ac95da48052fd84a58b74df70f989b980cc16c641d", - "12": "714f37a821d3e431a5740938a97697f020846cad27bded909eaeaec67c19b53e", - "13": "40644174eb0aafcbd54729da9224019a6ac9e5eb9d9b5add812d6233e8a4c674", - "14": "a0c38a32e87e85bc69db1510948106e5b5e39c05cb44a365eca0bdaecf34d320", - "15": "500593dcb043bf2d3d4efc34d9e5ddad3612e3b9fa22059efc08b4c4d55eecee", - "16": "cd602204c2af62fed1aa5222567210b401d1e5c8974f2bdcf07ddc64a1ec3e1e", - "17": "0f07afc0987992740e217e270fe19b649adaa1ae76ae54e99067408ac2ea49f6", - "18": "7c61051750ceae238c097cd07c3320f2a1944ca26e5e5088c6bb97e58be27e60", - "19": "e98889ab577f8e878be2fa8c49be393891bdfd3962e77f8827f33492235d4349", - "2": "7dc8e045b1333355ae2a2aea67d74066bc8b6a7dd44cbdfa44ed9ca93d5f040a", - "20": "6a6216c52ff238a879b27b997748c37544693038dab959fde4794a7f66e0b3c4", - "21": "aa84edd7b92afc5bbef9adb4c967451252f72ab6f0c0ac9002d089cd66f76426", - "22": "5eb0fcfb90045613f03fe7e27761fa88aa286908e8d2441f57bfe8f562077df5", - "23": "453e1e0a1c7b347e5ca6d3005f6c6436b04151eed6c1a35915ca8111dbe8091e", - "24": "8f1a8dc5e88a8d8daddeed48224e6e5addad729f7c8546d3be696317453fb59f", - "25": "7c631e5a2f3794229e6120b88b3f5db8fbf2108c4479b64e0aced3cbfe1fc7b1", - "26": "c5f2f63e6e5a6ca0f457eb4729e2f2724f1a09288130f02755278044480edd03", - "27": "fee8cc344d6bd87ba829ac687eaf4cf47432bcfd4dd053c722d9395883016693", - "28": "e539ec06774ac35051e060b7077c702ef67e14335b387d782974bed8fd52f08a", - "29": "d594bdf5072ac1f36a7569c5e47d23da82a2f1473dfa45775a1debc7729587f0", - "3": "06721c535b76347bbbf3576a367ff970af8e59d54d65be8fa8305a11cb69f888", - "30": "ab1a117d36ed41bcbfc253493ff0277b9555384f2ddb152fbf0316f3796aeb37", - "31": "e95c0c7340697de9bc1c44d14ac08e8ca2d2c573d2219d0aa0b571fdec0f8355", - "32": "9a0e9e541f0fe59e7925b2fc2eaadd89da4d5138e26119955c7a12edafe0c22f", - "33": "939f998f20bf57cf48f1155ea3881ec4bf986b626b7329b970b2d2bbfe72897d", - "34": "d738286df4257fac394cddfdc82c1ab649698b19a209d6f996d24130ef60a6cc", - "35": "ac0c39f68b9d129a5c604f27445fcc4a46ce214ccf35ec0be1a5c201d4073620", - "36": "bcf0246cd81bb46aeb057419908b888550e7072fc13431bf9c91744f4951efcc", - "37": "c995ba16b24eb173f6abfe9ace42b7aa517eef82f1929ff2d575b8566c6b5d8b", - "38": "acacec1b618a8342a9bc2c454040b0765283a86fd5fc66d1495c89191ad356da", - "39": "40144ef00a6010246c43f81fcd743992434f20a4d52546370fc802fce398115b", - "4": "b2fa1d60c7d9ef12556a81644aa8e21294b480051e24460deba2ef33fc2e411e", - "40": "16db9a6a0d16350941ff3f207c5d96fca130cf60fda24e40165784e3dd7df1c4", - "41": "50e7264fe294c77c3efce21d519aebb73b3cffbd440539c3352f403020154f1e", - "42": "98cb5963c0bd139386ead88ff6dffb852c629a983d027647f48dd019b915a463", - "43": "5a8ed0d73a605e76ba021bb7057c9654a1bd57cad9a55cee9dc610ba0bddc4fb", - "44": "df7d72a34394824d0fd8a58c10dfd44d0d63fee882010529018ea1b8808e01c6", - "45": "d81409743da466df8eb90e3493c8b32d6292f8b1f26ad88a2e90903fbefd138b", - "46": "841623c17c125da2784cfbba439aa140f87e27732c67727384a872c722947e3e", - "47": "d8332368280657368da191f588fc0a55191f142a77ce907658139bc9ee0da513", - "48": "cdf7698a0b93ed09066aea275ad718740f19d12e6e7465adefce3d88441fdcc1", - "49": "5cc885d581a32c95f139ad06168c748be86173ae7f9fb72224e8e91149abff9d", - "5": "6ea9cf149a3ed479b4e97f7e39c519d1aedbd6e57f014a814176d34c0868b3c3", - "50": "f2f2bd7fec4b42e68afe75bf47580e21bad89b27374802bd0766febc22d362c3", - "6": "96abd7c9b4b037dfb5a9287e61c0513150c407557ac450d26484db06de880136", - "7": "87e091dc53f52b4666c1b46997e2ddca6e07adcec7c7c8394d3c9dba0aab263a", - "8": "a7a246397ee9f315c8b62695559de57ea35ba5d4a0473a3c336f8d41192ebda4", - "9": "0bcea1e69e4465b10a2554fe3dafd535544a31956c598f196d2ffbd95ca2a4e4" + "0": "5774de07420a5078930346ddf5819f183d5e7474d2ec1fced5163d8bd9c1201e", + "1": "f4e55c7280f694a4f9f7211b31b83f8ac4ed76479ee3afd93b79c84e204547a7", + "10": "2f15ec34dc1a56bafb25a67cb4a8b4be0b1c30ff72834d85407179803a352760", + "11": "0b874769fac2df0f1c858a03752cb3f4e069721fbb9ae9b1e20f1143ef432a42", + "12": "58631c0cab4c1408cd33a3297de56facf4dcda36b97e006eca0782987edc1574", + "13": "a815b0aed0a812cd9eaba108a60d1605d13f577f14709c3acbf78b33322bd3fd", + "14": "5b7bfc7b3479a8ef23c56f5f70401514470a01b287e719c0bc89f67ce6ea14e5", + "15": "35340ced0cc1a7ed3ad0bde4d4ff3e45cc21d66af9eb946cfe2f5e6326738f72", + "16": "5774de07420a5078930346ddf5819f183d5e7474d2ec1fced5163d8bd9c1201e", + "17": "5774de07420a5078930346ddf5819f183d5e7474d2ec1fced5163d8bd9c1201e", + "18": "5774de07420a5078930346ddf5819f183d5e7474d2ec1fced5163d8bd9c1201e", + "19": "9270dc04118c3c54288bb0f26595dee9627c0193a553f23d6ae0e583f063ed0f", + "2": "d2287389d440d609f01fcde8f6a5eafd6a52175118b1722d175a92044cd859ce", + "20": "d31da8b6037b65e794d03ec8763e392787cdc6e453bd1a005261c42ff29d0fc0", + "22": "8b0386f0372e9380c7878b110e94c69e95b63acbcc9de6819982d331a1e6a5b1", + "23": "0de5ae39bff1832101f673c9e89bc157e6ecc82be7bc67cd97b7897e43d99ce4", + "24": "98b7af693c24567312660088a8846a5bcb52e8f235e9cfefc302a19ce5951b7f", + "25": "11a62432b01efd7173bab2434a70fb64f02d9693d7d27d803511092c1f5c0c41", + "26": "574b4ea1ca253fc86dbaca356b7bd03a65d010eafadfbd54ad2cca9db73d7d3b", + "27": "97920d6f5ddbc30e5aa43e4c8a1d52696bfbec9250bf6983514b58f6b7fbc1e9", + "28": "b93ddaaf245c307b3bc571eec7c4f3fecde618b1235d9ecaa068318ac31a3667", + "29": "7872137503f74234743cd56fd5393050c3780649356bf28ef1b53dd8579c3e6a", + "3": "60c312ea26665d60f34e1e3f6c5573dd09209983ffa221a82ba1a1c8c2b88b1a", + "30": "0160240546bd7607b57caa729749f63e337a0279439858e1971910716a834463", + "31": "81258553948ee548be34e33d34c3ed8ba9fc4e896af22096fa7dfe76c8dab77c", + "32": "5774de07420a5078930346ddf5819f183d5e7474d2ec1fced5163d8bd9c1201e", + "33": "5774de07420a5078930346ddf5819f183d5e7474d2ec1fced5163d8bd9c1201e", + "34": "5774de07420a5078930346ddf5819f183d5e7474d2ec1fced5163d8bd9c1201e", + "35": "6cdc7e95609274291d6d1393d5986a6ca9d76b7ec18ff20170614cc0e4ac0d1d", + "36": "0a34f8b6cf16fdd8ea32fd150281496da56a2d3b664077905799adad2011ac8e", + "37": "6b63550b13462beb572579476ce8001953cd745aa741c69f0391157f49d0bce3", + "38": "21f531fcdffa8b55bd0fb0dd134529a4a3bc0fb32ec864fdd00e46e99f95d791", + "39": "047c1c356b8f9146078039f83cf5e000be213d119fdde9ae474cf5e3fd5fe4fe", + "4": "247918ae4fdc46423d142d59f8d0e4272342621eabed139b0bace42117f7380e", + "40": "ed7d10992f947289ed3e1c8c36867660393b85a36f878eae70ff2c12ca81c1ef", + "41": "a0da2f4cf161112c1c53fe015eea8ec3099b9bbf0a228f059477ab48c62d1a2e", + "42": "089c77acd138dd3e8c08b9377fc285992ca3ef96986cb17b41f40c87d1d2576e", + "43": "84d266ae0dabf9a6179f5ee237f2d77c7b7a61549e8e749bea44bc625a975b45", + "44": "feccb1655e6f985d9f9d4627a3c286164a3a8b1e09b7a272607fdab85c77cc29", + "45": "56cab000e5e72efd67930375a7a6b9adee7baf3a7f1b57a959b74619f1e14fd6", + "46": "06aa3987d4c72ca196901d2cdfdf5dcd484757168099556f8332d1d6d536914f", + "47": "42fbd5d2648d86a3b567c897d843c488e08b5bd4f934069090cf0d13676a5493", + "48": "22b5dca9d654768a49814db36e5a30038a192d0b1a43bf8d477ff6c71264686f", + "49": "6cbd01602000ca6eefe3d06a13215dfdcf8c4079d3b56a79e4f4422e129e721b", + "5": "e20940fbba8a136c702c3d812426a106f13b9734c96740c638ac09268479ba07", + "50": "7268d2a983e5b9ed98381029c8c299e2b2bf7ae77d0ca1b796bdb394f757a4b3", + "6": "8e6c05844f449f1dab1a5c7a8d277575da316906e2a258cdae6de51619bae770", + "7": "4343006204506a5c0a08c5d9c1e4420adfc118806e241458b2496d8e3758b15f", + "8": "3f93e8bfdb78b80da9856df61c45ebab1a8a32a03e3ecf6ff215ec4735f7decf", + "9": "dbad6405c1fb6e15d364d61db3ea7a9aae57e32c8cb64cd0fabb96862c823d69" }, "gfx942_attention_tiled_3d": { - "0": "57ca5ee38d86fc8b4e651192c70ba44ac718c9b2fe30e999ec9e71f91eed1135", - "1": "226af192115e35b584bfbd1b7b8b9ffd8ebdeef9dbded9a5a09e2c9057477e26", - "10": "f1183de138bcf10016206d28a24a162e62f131f56f17c20833daa7c91e55ae30", - "11": "bfef7228e0bdd07aa26811572456b8f7cd9ce439b6c3b43495d2444f6f8eb693", - "12": "41d82ee2d11ab4d157092deb75442f377119feebd48d2a2a03f33b9620c6aae3", - "13": "0a8e2378f921147722c87e3157dfbb13ca751664547b11a9ff80ae0e0670552c", - "14": "70f152fdc8e454549e258c41687d6d9a674bab828876496506af2d88544a6ec8", - "15": "aa50f643e437fa9ed1cf32e4691c7fbdf3a0fe69ecbe1075695ff40b62103f57", - "16": "48a1e5d30c4d66a9cddcdeb9a701854ae76a3e7f00019ab02c22c42bc1535ad3", - "17": "baad169f072e48314087b69b424e32a40a09e96ffe17d4fb0e5779a1ef7c6d3a", - "18": "8d593285bfb858ad7385ab547d1d809c626986a8aafdc2d54d88d30ce4a3863d", - "19": "65858a9f795e872b15652d116a9f1ebea6d8e3b9e8c981910785ed1913fe5256", - "2": "6f4de2c5041fdca3d24abc5d74cb661146e716ad06b113600cefd059875f26c1", - "20": "13475d55ceeaa04216a6a73d39395ba262ce1fe0a285fda57bb4c969e1ffc93a", - "21": "8f5e4de1a6939da30edda46461e70e3c8293e45ae6bcd85974a8b4b93962087b", - "22": "3c69e5d2b5f6c817885851dccd04ecc66192a8264720c1ea0e0063407b02a994", - "3": "e8153af8740777bab3d5613e33d52b40b4616cf4b1890a0559b76c0eb31e2d85", - "4": "3e17cdf6ffb2daa7a7c457c1741f3a6350a5939897d5d7b6e9bc21fa64ca60e0", - "5": "dff48e7a6319aae6c1f0a67bc8eea2e79686a4caaf34b9cb1704cc62d2574f67", - "6": "aa8ab4e1c38d7a1f31eed51bcf6ae1ce78daac1fcb309e60106f96190f36103f", - "7": "d1233449ba68ce5d12e96a6201f487536db220eb32f9290b15940b2d53a56195", - "8": "c38439219df6ba61583c59378ee8842bfb5b62fab6576b3592570c8b52cee17b", - "9": "20855a3cdd3d37414ef6e184050ec341cec92f0e332e47543ec2c3e8310ecc05" + "0": "05f9208d14e20625c2355d08fead09242c01f3cfb6e692c305e4b53e06864f09", + "1": "57c6d9ec3cab9ad101daa297ed465cd9ec8369dad8d838e4fa6b7b38abb72bf1", + "10": "c9b6044bd6ad5613cac0e1271686587923ed527944e2bf551653ca45a4dc2193", + "11": "a95831cc76d51ac3a373bb21eb52c3f2d70f96b7e916e663e569a0498843a38d", + "12": "c23a71e2dfd74511e3b90c3db3a32cb712012e08950c836c5fbead37f67fb40f", + "13": "9743c4e3857b3e4d0ca7fcea728942f3cb4b5921e3f5c1171def03bf5c7cf490", + "14": "44eb3c2c08af758651044a93fe35e868724141edbd74166500744bb12ec207ab", + "15": "ab732e91ba542852f9b8d8d69c99b2cdf743bb9d5391d4ada0f31fe5d261af36", + "16": "6b4ed78e7af2fa4eaa0a2bf72c035401116e480af9e57bc257149a50b577a559", + "17": "3c93a4c7f94f8d4850c5a2c49a7d0bcc11a121dd635f1c687c1060f73742f429", + "18": "12c6db4a0432f1e03c69a5aaf92a408aa7dbc476e8943265a1509fc198295d09", + "19": "6a677a6825d0877a297442709e34dd9482e391b1815e40578a6942a039451835", + "2": "640e6b3eb547b74221c2c3d4d2c1dc5b91da62f92be5e6e570e99978b4961ce3", + "20": "10170faff0bdcf51a03b2491035a89ecd9b52c5e97a20c3ec7d1a16d48006bd3", + "21": "8b7dcc34edc0617b83e20dc9e6fc99f93deb86dcffdd47821b3d939849191e19", + "22": "6edcdb1a7e60b94407d341625a0867f88dfd39dd0a79d06ff25f36843f3ac5c5", + "3": "14931a184dcdd75b8c70d2fa7fcb7486a215f10cccd37867e0b95f9ae45b6566", + "4": "47309bd8554215f56236b38fb0b9c37d3253eb0d54b4437b94244fb27437a3b7", + "5": "1946bc1f67069ffda2dc161d2a8d11daededa922c1458942b266f481c97bec08", + "6": "3403e4686fcba6f26cb2ab502b028205f18c651d9ca0a84b961ac08add09f1f1", + "7": "6f45ac9918b237fee5ef9f51734569ff2eff1e6871f4500965a0a0ab01de94d1", + "8": "ccd6c331cc69c39faab3dae1759c3fc90aa63d5383f68f0493bb055e612edccd", + "9": "0780bfda4117929b13f56941e11205d93fc75afe332d5ef49b91c55d76d4bf34" }, "gfx950_attention_tiled_2d": { - "0": "0db726bfbde1796b0d4b7ed1e33241cfc349865034f68f419472058d6c0afbff", - "1": "9ad938a7499ba5efea5d4f53c5b8ba4a9725738148d0e856a9dd8796e75ad166", - "10": "d017d94ad9b728852f840a0759ebd3fdc937e7612c555c95404cc966ed70626b", - "11": "9bd19453c7f4f8d444c19ef34db82b5842b4bf042856aed872fbd210a094fa5b", - "12": "6ec3707fa5ea7d1a1b2d7a8f271bdb3cc9d5997d97d737d30b97d986766b7b7e", - "13": "65195b87a8e3104a1c26572be6b9108b8ff6f9e827f4d0dc58232610cf3a3377", - "14": "de24e2382261f00ad6cbcb595f458357d763e44238a2a3f56fd9319f07152439", - "15": "688cd9fbc1831238738a07966dd54e474a59a59e42cd1d6efb51ff1db951ee18", - "16": "d67f839fa4a3984a9fa2c5f1783f187a50274abb2346dfe2fcbf3adba18b24d6", - "17": "eabe2fb9cfd8280dfa89fd0f1541e1c34951d16ebde562c8ab6fbe1511322eec", - "18": "8dd30fdf84d76f599151b900bf0ad652586093e3c7d8c70775187a9ce53aa893", - "19": "17b9575bfa19d084235cd2b2dd059388838a4db622b376dedc664659ca47675b", - "2": "5cf276b1f9cf7bde862c9564adc4ad3ae3e89f73c022899eefccaa64bdc8d3f2", - "20": "e45439179f27a76963bda359a525f259ba75cd610d78931a20534df29be197a9", - "21": "b080c423c960d6e4ea1edf4c206890efb7c6bf1b73785ac310187f64f08c7e80", - "22": "83572334ac10bafbbd59ead6efc21b65be7c15a00caeb7f04ffd7c75d90b254a", - "23": "40c9a00ca4902e1cc0e77f4142d9fae180e52593dff0f72380ac836a90eb43cf", - "24": "97a32b20a0e1c7affe7a6db87f7bacbda28376384277c997d98ebae59645481f", - "25": "facae875616e6099e21ed4187adc72cc7b39cde52192886ebb95b6f3cfe31203", - "26": "60a379b295143c234ce0010491c15f9bed02bf1ebcf0497a9c7b9dfa1798b516", - "27": "5d6bca94022ac4fc888ef10f67040e51df60ad2254c2edcb37cb481b5f8494f1", - "28": "28bf2fa0d1224dc440d4a5c073c896fa3fb482566c29c9a41144b31327f7b3fc", - "29": "eaecd7456e96c8742e36244ed7b976ce9f9fccf52c408b354f9936225a9413b0", - "3": "eb6bd74e52f0e74a020b7cdd70fc840db874caac6b6cbbcf9ebce7cdbacc0561", - "30": "5f2c99e107406b25b1cdd40b219f1d79fdca599c084ba5794e6c8b12fadd9449", - "31": "9ddf994b8116bae7aa42ce63ffe08def14446c161c2cfcfbd65a2be7f106c210", - "32": "c16880e0996e0148f5e97fb9aff67e47e57ec9a1ddeb8730ccae4ea7d08a8fc7", - "33": "edfe36db820b5893f274af230e5a7a52edb2910a2f8679de25db1b840894a02b", - "34": "286fbddf2407170bc8fd4775c3f6b0a4cd5f4c7a7e269cea15bcce98b80dcaad", - "35": "0e1200286193519f20c41369e3571b1efbc88fecd1bbd372ce5b4a9d585f214c", - "36": "3ff32dd991095a55e95cda6781888c48ca4c546cf6d7674fc66b0567e692ec7c", - "37": "d84853584603030ec7827ba4fcc64a1ba8bc71725dd6d960ecf72a311dc4b712", - "38": "406be63b29f23aefd76e120fcde654cf974eef30a58bd2bb5c3a939e559ac3ae", - "39": "4bcf650cf96ea606f17643621715d61421b7e6d13dd456b41d51db76e91e6413", - "4": "c9626052f9b05a098d47cf8bbbe3e9abcf4f63b8c05bda3e449d96012da904c5", - "40": "03fef24ac7b009c2aad32efa7ce9f03906008c8bb3d138ecb1070e05069b1f01", - "41": "d633dcf482d90dad760d5e1ce799b113ad0ab1124dd970b69f7e5c77b49e09fc", - "42": "7efda102f67229c76f3a4c31a40a46df6bf26c9b65fef68d021e43736bf2fbd5", - "43": "5868052d86d78f09494d8dbe666b16d35f6a186cd8f7660f9e302b64beb6d522", - "44": "f96b52c82dbf6e1b1a1be50aa0ecc12f57a16a41b78eacc889a38d0105d0e29e", - "45": "4b2d41f37c5e6e2df10cf78956c0620239eb98c8d8a985d9f657df2aa16b05e9", - "46": "6c15dad11cbecf06c6d8ba4ee8b92ff40f6728eda0907174dcbe47c9e76d34df", - "47": "8dc2996616afbc035d83c093624b86e8627d0ec2912aec9d04cd9a3f98b85746", - "48": "970e2375d1449b525d9e1448a31fff063bd9994ac3e0b5e5394a5601ba37f68c", - "49": "2ec4923e1b9199f5aafb1cd90cab3ddb4b61b1fe27230babd6d3e3d5a99437a9", - "5": "ab1b43f9773d3c8814fe703ab076598676994f497f7b518efc7c73287366488a", - "50": "071e5fcae402f4fe84e8182c7ed175c24951b9b3b3d41e70a612356a4832a8ab", - "51": "16ac35130dcfe4009551eb6e3c7bedc758da7af7c41dcc40845bd5c9bed1db04", - "52": "31a17176f2865f70dc8d641563c55672014c8c37c11b834689d60f49600e733c", - "53": "04f52e3aa82ddc22536589d23eec3e5d313dc21d2a1493b1ee6240480a7d0f76", - "54": "2f9446585481aa76c6a8074e5212a1408958d62ff44bf3344cdb5f165d7584e8", - "55": "2a13a454cc9be2c505b156253f1104668eebdb724dc1330155131b46a3e48a79", - "56": "c9c12b31a8aa28739a618587a3915bd21457775387d69cd30cd8e7b2f86e866b", - "57": "b1d47c4552b93fa76df85fcd34b6f771095f7f00f3fe323c325c9f2e4b249438", - "58": "fb6f6443fb0c836f7ce4999c800a29648bdc4b9f6d69c01d1664631f5a1f33d1", - "59": "af09f6198225a9cc2dfd4ef10d4276c0ef5580c004b01547de2cf5e5e534a0a6", - "6": "c4ccc38dd0ac0e64736e2f21502990a5cda9791e68150c7ef688d050792cb37e", - "60": "e33c99378c66237a90297a05be33e67ebbf3a0d1d1d61ae36237005fa9945194", - "61": "bf31c9c41afd790d5400e98f31dbc19da1d5f22dd1cb62d6ecc3bdf4ca480b9d", - "62": "1b5ec55411b01a7a7e00ebfedc73dfa66ad01ffc2501410f671893aea4c9e6c1", - "63": "8c6ca9745a20689ecbb23f2b3a51dd4a4ec33d8da3f09a5342c704b45a374d54", - "64": "f4c6ce377965f2fb53d583e7ce80923b9dfd462e325fd9e6c92d24f2a03d110e", - "65": "378c535e5425529c682f6c5bb87675241a425bc7e4334b7a50a94e5d673be96a", - "66": "fec6d6e091d2451388628985c4ad70f450125da2a9584a85691fa5b5ba88fd6a", - "67": "c95ada5fb892c66dce20797100f3c09ecccea38bb713dc00fb53f50f23c84992", - "7": "eed7359db9622d3414b709a29a3fe612bef17fa997146df06bb2a280396e24a9", - "8": "9cf4aea8ae9949ae25ac095655fbe0d70dbd7fda64e3c9c6adda2b17a4027ad8", - "9": "263a8e66cd5802c731a74812b123d600e90696da2b2b35a0ddfc652bf8b62f71" + "0": "9701ca27a9122202a8accbe2d887d2b4e4d94505f2c59ddc2c3c5d11fcf4a650", + "1": "bc0e1b8161bdd21fe6aae9bf926dededbdb68e8dcff531c372bada44e282e422", + "10": "837007f048151146fd2cf6c6056831964afb7d5e3f2aedd69b3537b786e2cad3", + "11": "d588c9d2f9087b4d13ff55c0ed78c21d6c7afd1cbda35d7e5b2e6cf26a2e719d", + "12": "36f795569fbf68acfa8755fa79bd2aaa7e4310cd63fc86e17f8a0e45634a4cc8", + "13": "0ada2e4afbaaf39cffc71f398d800e1ea1088e7b3ee6500bde4a395be71f4037", + "14": "264d961b08f22c5c25b86f160b04491b005e63139682f1396f2cc668d6429a62", + "15": "66bdfa73724e929309e85e493085b0b8e7a9a5789d5136947e9918159d63eb8f", + "16": "3c99d6eba3829f625ffdbccb8019cd2c180c9153a8cdf42d871c04c8eb170016", + "17": "44628e1309353b6d554af771f711905417f860d624b6b2562724925549301975", + "18": "319083fae4e47202bf3ff8cd5a07ebec2fe9a75750f70bfff80a2b46df25ac73", + "19": "71fac1cc337d732e20badfbee796dc556fc16479d6f390d164fb18e9f8c1cfce", + "2": "d1461ce64be0cab265113a2846223f7e7d62b3d28fd87dfa46d9f90d9906c49f", + "20": "dce561895528101c25ce8e8dde0ac119777020996263e8f02a4290c24ba4d8d2", + "21": "03cdbf24e818fc08a8d4773b1d795e1e87797c60361761cedebb188f1c9a41d5", + "22": "39d0946c5dbef24354f5b910047c55c75d5fce7b45a098816da24945ee246152", + "23": "10d551e2b2a3cf8657ee0d9a4a85384141df1ea5e28996b7e3b0d0fa1d69d35c", + "24": "eda54a8df595fb46dc6b49476e6adfd3ebaaf70f89b73d93990ccab3ab0f21b7", + "25": "d588c9d2f9087b4d13ff55c0ed78c21d6c7afd1cbda35d7e5b2e6cf26a2e719d", + "26": "d588c9d2f9087b4d13ff55c0ed78c21d6c7afd1cbda35d7e5b2e6cf26a2e719d", + "27": "907af9acf3710435a198eee4bef055aafb93a8ddb1d9e2b333baf3630958bdfa", + "28": "c5aa15364b33402a19478d79b34ca373bd30062ab52a58003ceff880f062988e", + "29": "83be0c145dd5bc48d36ed5c0a3148947d8ed7886ee186710a632a851fd8a6874", + "3": "e395f40b24fc4cdad4cfc46861420f3cba58c6cb09dd7187343cfa07f2e1e696", + "30": "05465e9b494f36a9ec35ae6211a8d91be5b0988996e206aa7ea9de38972f75e9", + "31": "c69983a4bc236fb8d132382f9126e770090875a241dceba7a3c91280ad1947e8", + "32": "0892df2a7a26ee65301a72cd784dd57aaeac4396a64e90be4e51142bdaf3fe3a", + "33": "b3c8ccd0ace81886865d5485728f6777c78211a35c69c00f866903d7055c83f2", + "34": "b9a97eefe3302a8966ef259e8005bdb268e0fbc71b7f15272b0945a93e68d81c", + "35": "303667207ad3f77cd7158069d639935c8c779660a48baee4f1f26635d52c2668", + "36": "67ecdbbfb92cfd9b55bd960717643a9740eb753870de1cae867666c9d60be1dc", + "37": "0a214d78f8220b28d05841502751b1ab7c99f43325787f54c0e31b040bdeefe2", + "38": "4980672786c3b89652ff6e93b6eabe80c3ddc4447256ed10a14895913c0b0986", + "39": "33e4f6424379b1a897ad90efaa0e0bf657223f63080113587b3891d0ada881c1", + "4": "d7ee147cecd7538d9ac3f6087f820a85fdbfb1dfa48044d7c3eb3d396baab560", + "40": "18b06ec0d37b0730dfe3c7b884d256a79555326c49755c37a482939bcb0a9435", + "41": "3dcd0a79f12f974b896c20e7b86d0ce0ecc10778b0af6d864ebc04bb3055a973", + "42": "999fedee2b1de9ff2da3edff00931d54f29a5fc4f47c550dbc52feaa368936b2", + "43": "307d5e4420e680542cd77d53a537eef0e9f73dccc92e0d6d28f7bbc75b2d90ba", + "44": "9b5aa388806791e41045eac81533b363e035bb26e46992340ad4665c2c5a1207", + "45": "da8de494dd607d08f01740cfe76025c532c1cdc6ce9f6345e84ed0765b18ce7d", + "46": "3bc777a99145570691675e905669fbbfd141f1c6e41396e4841b725cc02f19e7", + "47": "d2ba1a9b75946542d127a8d4c0c4b1e54aedc0f263c96d8a71890977d0d29ee1", + "48": "ae61f8fff6f666305fc1edc1ed6acc86e428877dd09f55f78c33d8a4c1c7bcd3", + "49": "91999d68afcd28a11e589944e8371d516a0bc6d0c48b9e27e43479c6828b0fc9", + "5": "d588c9d2f9087b4d13ff55c0ed78c21d6c7afd1cbda35d7e5b2e6cf26a2e719d", + "50": "e857cc8491ea707c791b41447cac59f3808f73c2b047400c1063c4c102b7ecdf", + "51": "adac76b1caa3a852bd1a47457cfeaf0f2a3ac6d9f04a93f5e86d7bad828fa905", + "52": "fff7bb8f877450e2638efd464119367c1e23c83071c06bde9b8c904f6de6742a", + "53": "306a985e6d07bc019c89a9851bf987f1e58caabe3538875db65be1d4db45bb1b", + "54": "0e1a84c5614990fa68907c703ccbd6e969a694791ab0d420114f47ad649a6373", + "55": "3e6d6bf7936e31fde226ac9122963d6aaa6aeb9028bee4d2ed23adf48e3536b6", + "56": "10f3f6d65cabfcaf6a5e9213a208c90930f9f6e86a4f17c8dd5b574d03a67323", + "57": "5b7a42cbc64aee9b5d7742faf6ebdf02704521a9f9de03e5900c4bff2d5d59be", + "58": "683250919a8e4f11c0009c6b86109f2ed881386420bb3099222a8fc81da64e62", + "59": "bc0f103913224d56bcbb037f75b1901cf12396ae1c1b8d2c4bff37645627ad6a", + "6": "e024a8b228b32bbd7fe9f7a2fce2cdf3b7ea93ceb6568cb20b1a6efee0542cd6", + "60": "c3f4bb8449658bbeb4bd02c134aac836b111d54698514d429c7aeca2911d1246", + "61": "0bb37048ce7745bfa88de737554ab5b75222bd5ecbab283885c4eb11f706f42c", + "62": "5bc30997585d0c078cfa6d9eab5158d3ac903fe16e6bb9f0d19246e0b344d96e", + "63": "869820c42229284ba199ab77d9b6c16ee1687e85020ad43f75a1fcceae92bb47", + "64": "1e3845d629eed51e85354dc3d31646354f767a349e3a08390b18b28d28b23525", + "65": "89523051b07e3f703887191e391d047aa5688b58e2083094e4d82cdc52859fc4", + "66": "884619cd327d081936f482f6e53217db56efe8b9032c36543752a8e13f3f6dd2", + "67": "f79be17ef75f8af025435f5dd686d2fa4d13f96af28797914cda674b1292a8f4", + "68": "f506a72086d0e3847d2c7bb55bda83fb14afd77e7d7190d62690429e828a4a36", + "69": "1b2746a0d11d9d22993c5f58314b7eaf913926a7b80f3c28ac96c4df8824b22f", + "7": "9f8857f7a94b9b8524bd631b0b1fcd3c89d8733d317d2ccfd9e26af3ff72e3ea", + "70": "2d702df8314f65d553d8f0e1d72d0d202625361d26ec2a361f7ff3dd012ea947", + "71": "a1e8c2259d9baa180cd31a07c3974ff01d000af87bc24a21a7e4b3a157e939ab", + "8": "9a4431d34fde7e376552fc757ff75be86b1c4c6194b3f95c93c0ab9fbbf9f2a4", + "9": "6267b3845952589a6f3594a1e07902bfd52ec3ecfaa85d398c4b002570f37e07" }, "gfx950_attention_tiled_2d_fastkv_regp": { - "0": "c57d0d3c94014b8fb4f2970dcea22e1d5fa1c59eae53c488477c1b4cbe7e1fa6", - "1": "5dbf1f1f94eafbf60dff8773ca8beb6025f4db7cbbbcdba67aca696ff4ea3401", - "2": "361a82466d9c33eb0c50d5d511c6e0c831ae3c317f192752c31bea4bd715829b", - "3": "1af7b32a44343599433ae0a320f28437014e4f00344578a4602d97e36a4a0466", - "4": "b7deee184c7b806981600cc7300b89d9653426bb3a85dd220dde96af788fb874", - "5": "7e8f4056da548709d5ab9d17edee524ba06ff2e16497198f9f0aae8a76735e2c", - "6": "4c890c3b268497e172919e8be183533f933d4602cbe9f0ca4f8b46aee0736a4f" + "0": "e1fadd21fc2e1e2701d08d91d04fbb52034b6bccb6205c6d28b42fb210a5d6ba", + "1": "a5e206ff0c428abb923eabdc858d3a5ebc5bc5102c932dd36ab7e939945c5361", + "2": "c18b047312b1d0f138ddb2781f4d84a0d6675b65f7586117d7525309e8de9680", + "3": "edb0cc4d37cb741f8df02610bca8c7a983103c94fa257baec5478cc019097e83", + "4": "d7533887bfa73579fe814b43a4a3cd9f4f69d12875770bce14bcfa20bd9806a6", + "5": "dccae9d102fe08468f192b138dc0d8f3df4ed9bcfd2564b288d33a1e8c3b33c3", + "6": "8b0d8fdf6de830b3cab247ca4f34e2aa97d6e82646dd4634a4d1b6cf4eb26495", + "7": "e7ea0c26ca0f5ccf4e40fd43f678844b31652ac912ab542c957cb87da52f23a9", + "8": "ce6c66a32ecaf2a4ab0745d05feecdb34d1cb6f9a0caa4fe8e1555e02a636a4c", + "9": "f66721e89813567bf21c5491bb483575d0868a98b4a47a7093d6a72666bab4c5" }, "gfx950_attention_tiled_3d": { - "0": "2d15cfdb485bc112c4364d6b000586b3da7cfa0403b170a7f4c4662a3003ab84", - "1": "0476ad71ba35fe47953f586a272750b88fba23a5f66aecabcec34062a1a5a7b7", - "10": "b0d08ee5431786ce546dce7a44e221a76b0ac9cdcb5afa2c8313fb2c2e0e1b44", - "11": "4870f2163281ea108cb8ae6383de76d6fb427ec80e558f6b6c076a607da18d6f", - "12": "9680fd6292de1079d9387a0e9947acbf0831cb64de2858ce4f19f879e5379d8e", - "13": "d34f0c7de3fd5d0467d139803dfe5bd17dad4c26628bd868f4245c05e292d0f4", - "14": "7a984f9f07137c77ea2f0704ad7a65a350e9fc96ab759497cba626b3fb766f52", - "15": "942c3c72cc504c539d4abc6201db2bab548871427f35a335258eee0614c1491e", - "16": "5276267bc23de9579470828f4d0452b249d301e70727437f315544ae9e3ed9c0", - "17": "99acbf37277a473e4cedd12d0595fc6b304f7af70c39fae62f08c0eb1a0a13a6", - "18": "52f9076af863cd1f6062f112db555f90d28a58df1a610dde09067b2fdc52f66a", - "19": "6d33f51f39c8d4a54c1be3ca2098dea8cce826d7395a0ecb8a09417e80d98f64", - "2": "1680787ea992fe51bc10f0d663dce835be3c0dc742b4a36023964a9c1e671ebf", - "3": "d543f94c41159c5af6822c0656497b16bcada09f475aa2205c2ed937becaa526", - "4": "e3c11f0bea0406fd826b858b25c0273db7685e0fc1d04cc8f90c4789fa474a9f", - "5": "d43dcd71e7156cfd4f15bec3022fd26ecbe83352068d1313fc824a1c17475de5", - "6": "f44fa8e200198874f65206294b327a10f1fcc62ec4b6f36e78efd52654299b44", - "7": "efaa5a76dca2b7fafcdd6a775d13559f38cbcabfea66b05e2cb94e86da519a7a", - "8": "2c72ff6b89e1d4a1883e7d3e82ca01a7d4de9dd98d54c1142bd40de8012fcf1a", - "9": "62a5bae4eef3aff5335a242803f79ef319a5e1a6ff80e984709c0b51e241be11" + "0": "185fe233b422855899789edbb842c03f215fad00c1695917e21cad755a16de46", + "1": "a13ef2bebf3c8d27861c800653beb21b0efdec7e1865146ad540b5e4333521cb", + "10": "bc305699e45ba27ddc4d9c747ef9f3557dcc34346e558e8c07405467ef11535a", + "11": "9753247ce408ad7f445143ca04a20c3ea3a6edd86a823386109750a3b38f44f1", + "12": "06e34a54b23416e64f2b9b995f2cfceb7e18e6bac98bfa5fa6245df6ee8b70ed", + "13": "4c9f99009b4997667cd8f5d378869782f35ef55664924a42094859a696df3138", + "14": "d657f73cff7d7c5ca2f8ff5c6143e5471564608be25a8e55bb0a39c7638d67fb", + "15": "23ecb293b7e44b601d6db8bc37f89de539e9089940561c836d8b832c81bb0d39", + "16": "9621c5fff3a88548ae2af919629d4cb69414e888bf79b4fdbd00fd809c1314ef", + "17": "1f9ca3d620ff2cfdce2738538c68c14088c585589e5da11b5b71aa4fb2708694", + "18": "1a7376af8650d21ffd67a3ee829bce1bbdc409d7c6faeccda29c9ec8582eed5c", + "19": "9d6bf157a0bfc34f90d79bac8d5f3dcbeb5c6d339552e13a3c85c51329978a1c", + "2": "fc583225c3e1a96067496e8ab7afa37fa6c6eb993b7bbc3d2f237fd57607f063", + "3": "7104dd650ee025a86f1830a1805541c315425bbac5c24b25c4e115874f50feff", + "4": "3bb308c0df37f7f375ebb7cfa77188a3f847cdd40627db54591a3a64f51ea694", + "5": "5a6f8e3e40767268e22d73f6dbe7a1db0d682130f795e49bc46fc8e3e18836eb", + "6": "dbc33c5d0880dd8a476682f38346ab2305dfc211c9511f0f252a8e978419cc6b", + "7": "f95f155e633137d51def2007f23ed9b0f5123e0521f43fd712d4ce02b3d0850a", + "8": "048a04ffe10fda0e1a274e78ec9ac2a06c216e6415d38e9e309478ae57d6d7ae", + "9": "51c6643e3d5d31cf6ae87b54e8c40b66ac2fd0d6f6651a6117f09505ad0ae60f" }, "gfx950_block_scaled_gemm": { "0": "a550fa8769391a2bf246ee73ad70769826d1f902db0a49bb60a80e7c0efda41e", @@ -7494,20 +7518,20 @@ "5": "c8eff38fc6979cc723b3bd8e276884108a0655d1284a4e548bab6ecce69116ce" }, "attention_strided_kv": { - "0": "67e7f5bd0d8861dc16e6cff53316d780d40f6232716a5a3ea25f06055ac48442", - "1": "a0b5229967c3892d492cbacb9db2d15f06ce919b1f15c8de8d298e0d74306e46", - "10": "400f75d835f4bb4d37dfe0039c609deb5e172c77e5a83b87b89071ed3b0017bf", - "11": "0e441b0ae432165d00da483191b9470dcfc043c2bacd386b86fe6bef8f05266f", - "12": "03eb993466369d2ca4aabaff7880f23138bf08a0f71154159a0c9e4627e17df3", - "13": "74d14799531a8e079e822a46af3a22f1875a85a46c6c561ef26427fd44ffd748", - "2": "4cded6921b9951b733d1dbfdfe93015aa7b78c5bd5def3dfbe503175949af968", - "3": "393df7f6bb892e49b3c3031ff696d11addb1da00ad2088aa881ea37074c7db77", - "4": "db62e65f3aa74588eb95d2bf69935aa6331d0a57d42996bbe81c7510e146f119", - "5": "ee8248f21f5631e005750601f275daf2fd57d87f102a241c2fee76caa4567d33", - "6": "92c9560cfd6ae383a059972d0f4863e3699b56beba16aab1736d06bb41946143", - "7": "1b3cb6e2c817fce85c1bd189f7a7a01eb7e089c37ef1824cabdfbe5bd6915b47", - "8": "4b0b553b289d0de60dd9401f61f97d2f36cb201b6343437e28e5fbdc370cf529", - "9": "cb2426e5ff5ce00a3094ea4217df160031006f22cfe058568e4c959f1ecff086" + "0": "7ce6e33c41e2752f1863c4ef79eef519c4748cc9f1c3035cab5cab030f386d55", + "1": "38fa323beea8da252380810dbbe76686f3cff17bc15aca23c32a73c11f6a70e4", + "10": "59ba76e31fd186ba00c06d841b8a038c8941a8419e56b8fa46daf2ea9eb61028", + "11": "a30d1148dddca335071fee50a5cedd26bb5323693c278b14d22580d6cc5df811", + "12": "0ec51a3f92220ca3dad8979ef3dbe976f89ab95e28de5a178eaee7477661b8d4", + "13": "9bb3761cb7e928a0efc7789cda47a011260003ba29f7db4cdde89af0a0bda125", + "2": "5e39245a8752ce0229c9229aba2b11fc8cb07241e93ebaafa051cae24aab4514", + "3": "76bca1175f651389730a2856cf8bb9cf9d79998445425cde57ac37700dd1d2ef", + "4": "0ea423ae0c4cdd6d4b72ed754b1a18dab37fb3f6b9fbad492b1bcd036529c823", + "5": "29304ac08dddbce980440041340d863e3996b72fa786fecd5f739ca8c14bd150", + "6": "b3add7e87f7a74a750718e45156d80bc9a5725c33a8a25fadc55b71bb600af89", + "7": "94ea80d639a380a3cf0069ff8679e6f197fa1c40c9a4885413d747341964ef9c", + "8": "d8a78f5237a1c466494c27808c41590153f9ae47d899a5d6ad240510e680d89f", + "9": "67bfeb3341425741eb76f57e12f56c47a87bbdb73b1e5641c4a4f256f2647591" }, "attention_unified": { "0": "eb791027895b4a7db0aa0742569d6777b1331c559c87097d562f72aa2b6d41e3", @@ -9129,198 +9153,209 @@ "99": "056fa9ad97d50744d41853632710588920df00099f28006244673116483888d2" }, "gfx942_attention_tiled_2d": { - "0": "e7d4566fb615309b0345ccfd7c20287bbd8cb7a725fbf97ecb8ebf25c5d0b687", - "1": "87e091dc53f52b4666c1b46997e2ddca6e07adcec7c7c8394d3c9dba0aab263a", - "10": "9d4a2883a29cd2d073eafee6c0e6c72d7f85d6e5e430adad4b0215d315f3bd60", - "11": "00c3e90a002e86b091e5da4898991502eb7343ed9ddd5a1639d6a46806de1aea", - "12": "8327bf97823a7823a95536e84d14ca722bf5b8c32f7118d5f310d5ae1eeb98a9", - "13": "a0720903a7d354e085d4c68189279ee6eacfff930b865499707d59d91fe178bd", - "14": "487415ff6b42e500ee274f222930de09569f4d396996d9dcc1aa168e38798797", - "15": "d3a693c511a915d5706a56d83611d666a8b5a97746f59d124fd3345c012327b2", - "16": "ff028f89ab6dd968ae11ba15f0aa8f0ffeaaee3f16fed65e4f523cd03f137629", - "17": "b46f1819760ea0ccf0799759b1ca6580ca27127224e706f8f031af271a46fe3e", - "2": "2a322dec5ee9ada916ef06b47c4b0c555d1e7dd9e31142b1e9524ff5a4cc4901", - "3": "4b47445faee5557133dbc3286dfd8cc25751ba0568de4ad534bca1fd62d93a08", - "4": "0bcea1e69e4465b10a2554fe3dafd535544a31956c598f196d2ffbd95ca2a4e4", - "5": "e022360d8c005125cf4a375f274be893ea496228893e0eb182428de1ac7976fb", - "6": "8772ae5e0dba21cba23517661319285401e1b4ca0a5f6d8b4b1f6fe13f77ea4d", - "7": "4776e8d28fe58a4befb92306bd75f4ffdb2c452c74be4f48c40ec791260d5573", - "8": "8f579d55c882032526a27b20537904835ea0ef45d4b7d26d02cbc0272088a4a6", - "9": "1db87021061733bb5244d6fa7650ddf55407332921ce24d538aa96f9f030092d" + "0": "d2287389d440d609f01fcde8f6a5eafd6a52175118b1722d175a92044cd859ce", + "1": "4343006204506a5c0a08c5d9c1e4420adfc118806e241458b2496d8e3758b15f", + "10": "09f3f8a6dda2f5f9892a76040244308decef90eeb4594d77a5713332f487764c", + "11": "253b4d6b46377521d8301752b97f2d38bb3bbb0121f1c36290a009564f1b88e0", + "12": "f03fb8d960412fbbf96481ce32bf0f00f5af32b1b67bbaf7a3b8cb1bcda700f1", + "13": "64fff6a1232aba46257363c1b2aa3be6f5514c092e15ecc235e2cdc3ea42ff6b", + "14": "4016e10226975201cb8fcd11e0c37a5b82eab820c7a0a96b81c9066d0e2e3996", + "15": "5b9dcf6b30a3655f3c42dada636af5b4ee586f7b78f9ae8ba37ab5a0a79e70b9", + "16": "d11dad8902f8247c8122e2e2321155bc74aa009b2c9e148c5734eeda74a274eb", + "17": "8a89a840474fb264584bd0cdb49fea64c1bfbcde5a69b5fb1abfcd43eb0781f5", + "2": "55c737ed7f81060d5f9202dd7f3fabe25917c8497e7fa03edd921e2e552be9e9", + "3": "bc950eea70f3c80d7e64c6fdfad5c96ba3fb4da49a316168cfd1bf20f6b20f13", + "4": "dbad6405c1fb6e15d364d61db3ea7a9aae57e32c8cb64cd0fabb96862c823d69", + "5": "3c769ccf2cb2140c5659782fc886cf15bffa290fd1ea5739ca1389887ccf779c", + "6": "42ca2a9e7955596c87b29d334b563e51ef824635a47c43499f6175b69bb01021", + "7": "416fd50cde6c20af03f65e9b9a030b21dfb1b130561f0528fdf8ad5873607a57", + "8": "0c04ce3c73547c94af145fd9f1fc0be1445a4facd0a264683da0aebf68c8cbc3", + "9": "4c293e9bea4baa711fed44c04c0cdbe8ba950c4764ba74204498fa4fe7072c50" }, "gfx942_attention_tiled_2d_stress": { - "0": "4266276021f9b83e5c76f6a712c546877a3fa6ec123d89f04d8ca6f17dd9522d", - "1": "cde1c38c981ee52532af9ecd8335becf149b3dc01b91c687bd6eb502184e9288", - "10": "fc46983b7a750c4319f1b83ce6cd645b57bf953574bb8b3e6130e2961da10396", - "11": "398f7ce0c0b59c7b93c292ac95da48052fd84a58b74df70f989b980cc16c641d", - "12": "714f37a821d3e431a5740938a97697f020846cad27bded909eaeaec67c19b53e", - "13": "40644174eb0aafcbd54729da9224019a6ac9e5eb9d9b5add812d6233e8a4c674", - "14": "a0c38a32e87e85bc69db1510948106e5b5e39c05cb44a365eca0bdaecf34d320", - "15": "500593dcb043bf2d3d4efc34d9e5ddad3612e3b9fa22059efc08b4c4d55eecee", - "16": "cd602204c2af62fed1aa5222567210b401d1e5c8974f2bdcf07ddc64a1ec3e1e", - "17": "0f07afc0987992740e217e270fe19b649adaa1ae76ae54e99067408ac2ea49f6", - "18": "7c61051750ceae238c097cd07c3320f2a1944ca26e5e5088c6bb97e58be27e60", - "19": "e98889ab577f8e878be2fa8c49be393891bdfd3962e77f8827f33492235d4349", - "2": "7dc8e045b1333355ae2a2aea67d74066bc8b6a7dd44cbdfa44ed9ca93d5f040a", - "20": "6a6216c52ff238a879b27b997748c37544693038dab959fde4794a7f66e0b3c4", - "21": "aa84edd7b92afc5bbef9adb4c967451252f72ab6f0c0ac9002d089cd66f76426", - "22": "5eb0fcfb90045613f03fe7e27761fa88aa286908e8d2441f57bfe8f562077df5", - "23": "453e1e0a1c7b347e5ca6d3005f6c6436b04151eed6c1a35915ca8111dbe8091e", - "24": "8f1a8dc5e88a8d8daddeed48224e6e5addad729f7c8546d3be696317453fb59f", - "25": "7c631e5a2f3794229e6120b88b3f5db8fbf2108c4479b64e0aced3cbfe1fc7b1", - "26": "c5f2f63e6e5a6ca0f457eb4729e2f2724f1a09288130f02755278044480edd03", - "27": "fee8cc344d6bd87ba829ac687eaf4cf47432bcfd4dd053c722d9395883016693", - "28": "e539ec06774ac35051e060b7077c702ef67e14335b387d782974bed8fd52f08a", - "29": "d594bdf5072ac1f36a7569c5e47d23da82a2f1473dfa45775a1debc7729587f0", - "3": "06721c535b76347bbbf3576a367ff970af8e59d54d65be8fa8305a11cb69f888", - "30": "ab1a117d36ed41bcbfc253493ff0277b9555384f2ddb152fbf0316f3796aeb37", - "31": "e95c0c7340697de9bc1c44d14ac08e8ca2d2c573d2219d0aa0b571fdec0f8355", - "32": "9a0e9e541f0fe59e7925b2fc2eaadd89da4d5138e26119955c7a12edafe0c22f", - "33": "939f998f20bf57cf48f1155ea3881ec4bf986b626b7329b970b2d2bbfe72897d", - "34": "d738286df4257fac394cddfdc82c1ab649698b19a209d6f996d24130ef60a6cc", - "35": "ac0c39f68b9d129a5c604f27445fcc4a46ce214ccf35ec0be1a5c201d4073620", - "36": "bcf0246cd81bb46aeb057419908b888550e7072fc13431bf9c91744f4951efcc", - "37": "c995ba16b24eb173f6abfe9ace42b7aa517eef82f1929ff2d575b8566c6b5d8b", - "38": "acacec1b618a8342a9bc2c454040b0765283a86fd5fc66d1495c89191ad356da", - "39": "40144ef00a6010246c43f81fcd743992434f20a4d52546370fc802fce398115b", - "4": "b2fa1d60c7d9ef12556a81644aa8e21294b480051e24460deba2ef33fc2e411e", - "40": "16db9a6a0d16350941ff3f207c5d96fca130cf60fda24e40165784e3dd7df1c4", - "41": "50e7264fe294c77c3efce21d519aebb73b3cffbd440539c3352f403020154f1e", - "42": "98cb5963c0bd139386ead88ff6dffb852c629a983d027647f48dd019b915a463", - "43": "5a8ed0d73a605e76ba021bb7057c9654a1bd57cad9a55cee9dc610ba0bddc4fb", - "44": "df7d72a34394824d0fd8a58c10dfd44d0d63fee882010529018ea1b8808e01c6", - "45": "d81409743da466df8eb90e3493c8b32d6292f8b1f26ad88a2e90903fbefd138b", - "46": "841623c17c125da2784cfbba439aa140f87e27732c67727384a872c722947e3e", - "47": "d8332368280657368da191f588fc0a55191f142a77ce907658139bc9ee0da513", - "48": "cdf7698a0b93ed09066aea275ad718740f19d12e6e7465adefce3d88441fdcc1", - "49": "5cc885d581a32c95f139ad06168c748be86173ae7f9fb72224e8e91149abff9d", - "5": "6ea9cf149a3ed479b4e97f7e39c519d1aedbd6e57f014a814176d34c0868b3c3", - "50": "f2f2bd7fec4b42e68afe75bf47580e21bad89b27374802bd0766febc22d362c3", - "6": "96abd7c9b4b037dfb5a9287e61c0513150c407557ac450d26484db06de880136", - "7": "87e091dc53f52b4666c1b46997e2ddca6e07adcec7c7c8394d3c9dba0aab263a", - "8": "a7a246397ee9f315c8b62695559de57ea35ba5d4a0473a3c336f8d41192ebda4", - "9": "0bcea1e69e4465b10a2554fe3dafd535544a31956c598f196d2ffbd95ca2a4e4" + "0": "5774de07420a5078930346ddf5819f183d5e7474d2ec1fced5163d8bd9c1201e", + "1": "f4e55c7280f694a4f9f7211b31b83f8ac4ed76479ee3afd93b79c84e204547a7", + "10": "2f15ec34dc1a56bafb25a67cb4a8b4be0b1c30ff72834d85407179803a352760", + "11": "0b874769fac2df0f1c858a03752cb3f4e069721fbb9ae9b1e20f1143ef432a42", + "12": "58631c0cab4c1408cd33a3297de56facf4dcda36b97e006eca0782987edc1574", + "13": "a815b0aed0a812cd9eaba108a60d1605d13f577f14709c3acbf78b33322bd3fd", + "14": "5b7bfc7b3479a8ef23c56f5f70401514470a01b287e719c0bc89f67ce6ea14e5", + "15": "35340ced0cc1a7ed3ad0bde4d4ff3e45cc21d66af9eb946cfe2f5e6326738f72", + "16": "5774de07420a5078930346ddf5819f183d5e7474d2ec1fced5163d8bd9c1201e", + "17": "5774de07420a5078930346ddf5819f183d5e7474d2ec1fced5163d8bd9c1201e", + "18": "5774de07420a5078930346ddf5819f183d5e7474d2ec1fced5163d8bd9c1201e", + "19": "9270dc04118c3c54288bb0f26595dee9627c0193a553f23d6ae0e583f063ed0f", + "2": "d2287389d440d609f01fcde8f6a5eafd6a52175118b1722d175a92044cd859ce", + "20": "d31da8b6037b65e794d03ec8763e392787cdc6e453bd1a005261c42ff29d0fc0", + "22": "8b0386f0372e9380c7878b110e94c69e95b63acbcc9de6819982d331a1e6a5b1", + "23": "0de5ae39bff1832101f673c9e89bc157e6ecc82be7bc67cd97b7897e43d99ce4", + "24": "98b7af693c24567312660088a8846a5bcb52e8f235e9cfefc302a19ce5951b7f", + "25": "11a62432b01efd7173bab2434a70fb64f02d9693d7d27d803511092c1f5c0c41", + "26": "574b4ea1ca253fc86dbaca356b7bd03a65d010eafadfbd54ad2cca9db73d7d3b", + "27": "97920d6f5ddbc30e5aa43e4c8a1d52696bfbec9250bf6983514b58f6b7fbc1e9", + "28": "b93ddaaf245c307b3bc571eec7c4f3fecde618b1235d9ecaa068318ac31a3667", + "29": "7872137503f74234743cd56fd5393050c3780649356bf28ef1b53dd8579c3e6a", + "3": "60c312ea26665d60f34e1e3f6c5573dd09209983ffa221a82ba1a1c8c2b88b1a", + "30": "0160240546bd7607b57caa729749f63e337a0279439858e1971910716a834463", + "31": "81258553948ee548be34e33d34c3ed8ba9fc4e896af22096fa7dfe76c8dab77c", + "32": "5774de07420a5078930346ddf5819f183d5e7474d2ec1fced5163d8bd9c1201e", + "33": "5774de07420a5078930346ddf5819f183d5e7474d2ec1fced5163d8bd9c1201e", + "34": "5774de07420a5078930346ddf5819f183d5e7474d2ec1fced5163d8bd9c1201e", + "35": "6cdc7e95609274291d6d1393d5986a6ca9d76b7ec18ff20170614cc0e4ac0d1d", + "36": "0a34f8b6cf16fdd8ea32fd150281496da56a2d3b664077905799adad2011ac8e", + "37": "6b63550b13462beb572579476ce8001953cd745aa741c69f0391157f49d0bce3", + "38": "21f531fcdffa8b55bd0fb0dd134529a4a3bc0fb32ec864fdd00e46e99f95d791", + "39": "047c1c356b8f9146078039f83cf5e000be213d119fdde9ae474cf5e3fd5fe4fe", + "4": "247918ae4fdc46423d142d59f8d0e4272342621eabed139b0bace42117f7380e", + "40": "ed7d10992f947289ed3e1c8c36867660393b85a36f878eae70ff2c12ca81c1ef", + "41": "a0da2f4cf161112c1c53fe015eea8ec3099b9bbf0a228f059477ab48c62d1a2e", + "42": "089c77acd138dd3e8c08b9377fc285992ca3ef96986cb17b41f40c87d1d2576e", + "43": "84d266ae0dabf9a6179f5ee237f2d77c7b7a61549e8e749bea44bc625a975b45", + "44": "feccb1655e6f985d9f9d4627a3c286164a3a8b1e09b7a272607fdab85c77cc29", + "45": "56cab000e5e72efd67930375a7a6b9adee7baf3a7f1b57a959b74619f1e14fd6", + "46": "06aa3987d4c72ca196901d2cdfdf5dcd484757168099556f8332d1d6d536914f", + "47": "42fbd5d2648d86a3b567c897d843c488e08b5bd4f934069090cf0d13676a5493", + "48": "22b5dca9d654768a49814db36e5a30038a192d0b1a43bf8d477ff6c71264686f", + "49": "6cbd01602000ca6eefe3d06a13215dfdcf8c4079d3b56a79e4f4422e129e721b", + "5": "e20940fbba8a136c702c3d812426a106f13b9734c96740c638ac09268479ba07", + "50": "7268d2a983e5b9ed98381029c8c299e2b2bf7ae77d0ca1b796bdb394f757a4b3", + "6": "8e6c05844f449f1dab1a5c7a8d277575da316906e2a258cdae6de51619bae770", + "7": "4343006204506a5c0a08c5d9c1e4420adfc118806e241458b2496d8e3758b15f", + "8": "3f93e8bfdb78b80da9856df61c45ebab1a8a32a03e3ecf6ff215ec4735f7decf", + "9": "dbad6405c1fb6e15d364d61db3ea7a9aae57e32c8cb64cd0fabb96862c823d69" }, "gfx942_attention_tiled_3d": { - "0": "57ca5ee38d86fc8b4e651192c70ba44ac718c9b2fe30e999ec9e71f91eed1135", - "1": "226af192115e35b584bfbd1b7b8b9ffd8ebdeef9dbded9a5a09e2c9057477e26", - "10": "f1183de138bcf10016206d28a24a162e62f131f56f17c20833daa7c91e55ae30", - "11": "bfef7228e0bdd07aa26811572456b8f7cd9ce439b6c3b43495d2444f6f8eb693", - "12": "41d82ee2d11ab4d157092deb75442f377119feebd48d2a2a03f33b9620c6aae3", - "13": "0a8e2378f921147722c87e3157dfbb13ca751664547b11a9ff80ae0e0670552c", - "14": "70f152fdc8e454549e258c41687d6d9a674bab828876496506af2d88544a6ec8", - "15": "aa50f643e437fa9ed1cf32e4691c7fbdf3a0fe69ecbe1075695ff40b62103f57", - "16": "48a1e5d30c4d66a9cddcdeb9a701854ae76a3e7f00019ab02c22c42bc1535ad3", - "17": "baad169f072e48314087b69b424e32a40a09e96ffe17d4fb0e5779a1ef7c6d3a", - "18": "8d593285bfb858ad7385ab547d1d809c626986a8aafdc2d54d88d30ce4a3863d", - "19": "65858a9f795e872b15652d116a9f1ebea6d8e3b9e8c981910785ed1913fe5256", - "2": "6f4de2c5041fdca3d24abc5d74cb661146e716ad06b113600cefd059875f26c1", - "3": "e8153af8740777bab3d5613e33d52b40b4616cf4b1890a0559b76c0eb31e2d85", - "4": "3e17cdf6ffb2daa7a7c457c1741f3a6350a5939897d5d7b6e9bc21fa64ca60e0", - "5": "dff48e7a6319aae6c1f0a67bc8eea2e79686a4caaf34b9cb1704cc62d2574f67", - "6": "aa8ab4e1c38d7a1f31eed51bcf6ae1ce78daac1fcb309e60106f96190f36103f", - "7": "d1233449ba68ce5d12e96a6201f487536db220eb32f9290b15940b2d53a56195", - "8": "c38439219df6ba61583c59378ee8842bfb5b62fab6576b3592570c8b52cee17b", - "9": "20855a3cdd3d37414ef6e184050ec341cec92f0e332e47543ec2c3e8310ecc05" + "0": "05f9208d14e20625c2355d08fead09242c01f3cfb6e692c305e4b53e06864f09", + "1": "57c6d9ec3cab9ad101daa297ed465cd9ec8369dad8d838e4fa6b7b38abb72bf1", + "10": "c9b6044bd6ad5613cac0e1271686587923ed527944e2bf551653ca45a4dc2193", + "11": "a95831cc76d51ac3a373bb21eb52c3f2d70f96b7e916e663e569a0498843a38d", + "12": "c23a71e2dfd74511e3b90c3db3a32cb712012e08950c836c5fbead37f67fb40f", + "13": "9743c4e3857b3e4d0ca7fcea728942f3cb4b5921e3f5c1171def03bf5c7cf490", + "14": "44eb3c2c08af758651044a93fe35e868724141edbd74166500744bb12ec207ab", + "15": "ab732e91ba542852f9b8d8d69c99b2cdf743bb9d5391d4ada0f31fe5d261af36", + "16": "6b4ed78e7af2fa4eaa0a2bf72c035401116e480af9e57bc257149a50b577a559", + "17": "3c93a4c7f94f8d4850c5a2c49a7d0bcc11a121dd635f1c687c1060f73742f429", + "18": "12c6db4a0432f1e03c69a5aaf92a408aa7dbc476e8943265a1509fc198295d09", + "19": "6a677a6825d0877a297442709e34dd9482e391b1815e40578a6942a039451835", + "2": "640e6b3eb547b74221c2c3d4d2c1dc5b91da62f92be5e6e570e99978b4961ce3", + "20": "10170faff0bdcf51a03b2491035a89ecd9b52c5e97a20c3ec7d1a16d48006bd3", + "21": "8b7dcc34edc0617b83e20dc9e6fc99f93deb86dcffdd47821b3d939849191e19", + "22": "6edcdb1a7e60b94407d341625a0867f88dfd39dd0a79d06ff25f36843f3ac5c5", + "3": "14931a184dcdd75b8c70d2fa7fcb7486a215f10cccd37867e0b95f9ae45b6566", + "4": "47309bd8554215f56236b38fb0b9c37d3253eb0d54b4437b94244fb27437a3b7", + "5": "1946bc1f67069ffda2dc161d2a8d11daededa922c1458942b266f481c97bec08", + "6": "3403e4686fcba6f26cb2ab502b028205f18c651d9ca0a84b961ac08add09f1f1", + "7": "6f45ac9918b237fee5ef9f51734569ff2eff1e6871f4500965a0a0ab01de94d1", + "8": "ccd6c331cc69c39faab3dae1759c3fc90aa63d5383f68f0493bb055e612edccd", + "9": "0780bfda4117929b13f56941e11205d93fc75afe332d5ef49b91c55d76d4bf34" }, "gfx950_attention_tiled_2d": { - "0": "0db726bfbde1796b0d4b7ed1e33241cfc349865034f68f419472058d6c0afbff", - "1": "9ad938a7499ba5efea5d4f53c5b8ba4a9725738148d0e856a9dd8796e75ad166", - "10": "d017d94ad9b728852f840a0759ebd3fdc937e7612c555c95404cc966ed70626b", - "11": "9bd19453c7f4f8d444c19ef34db82b5842b4bf042856aed872fbd210a094fa5b", - "12": "6ec3707fa5ea7d1a1b2d7a8f271bdb3cc9d5997d97d737d30b97d986766b7b7e", - "13": "65195b87a8e3104a1c26572be6b9108b8ff6f9e827f4d0dc58232610cf3a3377", - "14": "de24e2382261f00ad6cbcb595f458357d763e44238a2a3f56fd9319f07152439", - "15": "688cd9fbc1831238738a07966dd54e474a59a59e42cd1d6efb51ff1db951ee18", - "16": "d67f839fa4a3984a9fa2c5f1783f187a50274abb2346dfe2fcbf3adba18b24d6", - "17": "eabe2fb9cfd8280dfa89fd0f1541e1c34951d16ebde562c8ab6fbe1511322eec", - "18": "8dd30fdf84d76f599151b900bf0ad652586093e3c7d8c70775187a9ce53aa893", - "19": "17b9575bfa19d084235cd2b2dd059388838a4db622b376dedc664659ca47675b", - "2": "5cf276b1f9cf7bde862c9564adc4ad3ae3e89f73c022899eefccaa64bdc8d3f2", - "20": "e45439179f27a76963bda359a525f259ba75cd610d78931a20534df29be197a9", - "21": "b080c423c960d6e4ea1edf4c206890efb7c6bf1b73785ac310187f64f08c7e80", - "22": "83572334ac10bafbbd59ead6efc21b65be7c15a00caeb7f04ffd7c75d90b254a", - "23": "40c9a00ca4902e1cc0e77f4142d9fae180e52593dff0f72380ac836a90eb43cf", - "24": "97a32b20a0e1c7affe7a6db87f7bacbda28376384277c997d98ebae59645481f", - "25": "facae875616e6099e21ed4187adc72cc7b39cde52192886ebb95b6f3cfe31203", - "26": "60a379b295143c234ce0010491c15f9bed02bf1ebcf0497a9c7b9dfa1798b516", - "27": "5d6bca94022ac4fc888ef10f67040e51df60ad2254c2edcb37cb481b5f8494f1", - "28": "28bf2fa0d1224dc440d4a5c073c896fa3fb482566c29c9a41144b31327f7b3fc", - "29": "eaecd7456e96c8742e36244ed7b976ce9f9fccf52c408b354f9936225a9413b0", - "3": "eb6bd74e52f0e74a020b7cdd70fc840db874caac6b6cbbcf9ebce7cdbacc0561", - "30": "5f2c99e107406b25b1cdd40b219f1d79fdca599c084ba5794e6c8b12fadd9449", - "31": "9ddf994b8116bae7aa42ce63ffe08def14446c161c2cfcfbd65a2be7f106c210", - "32": "c16880e0996e0148f5e97fb9aff67e47e57ec9a1ddeb8730ccae4ea7d08a8fc7", - "33": "edfe36db820b5893f274af230e5a7a52edb2910a2f8679de25db1b840894a02b", - "34": "286fbddf2407170bc8fd4775c3f6b0a4cd5f4c7a7e269cea15bcce98b80dcaad", - "35": "0e1200286193519f20c41369e3571b1efbc88fecd1bbd372ce5b4a9d585f214c", - "36": "3ff32dd991095a55e95cda6781888c48ca4c546cf6d7674fc66b0567e692ec7c", - "37": "d84853584603030ec7827ba4fcc64a1ba8bc71725dd6d960ecf72a311dc4b712", - "38": "406be63b29f23aefd76e120fcde654cf974eef30a58bd2bb5c3a939e559ac3ae", - "39": "4bcf650cf96ea606f17643621715d61421b7e6d13dd456b41d51db76e91e6413", - "4": "c9626052f9b05a098d47cf8bbbe3e9abcf4f63b8c05bda3e449d96012da904c5", - "40": "03fef24ac7b009c2aad32efa7ce9f03906008c8bb3d138ecb1070e05069b1f01", - "41": "d633dcf482d90dad760d5e1ce799b113ad0ab1124dd970b69f7e5c77b49e09fc", - "42": "7efda102f67229c76f3a4c31a40a46df6bf26c9b65fef68d021e43736bf2fbd5", - "43": "5868052d86d78f09494d8dbe666b16d35f6a186cd8f7660f9e302b64beb6d522", - "44": "f96b52c82dbf6e1b1a1be50aa0ecc12f57a16a41b78eacc889a38d0105d0e29e", - "45": "4b2d41f37c5e6e2df10cf78956c0620239eb98c8d8a985d9f657df2aa16b05e9", - "46": "6c15dad11cbecf06c6d8ba4ee8b92ff40f6728eda0907174dcbe47c9e76d34df", - "47": "8dc2996616afbc035d83c093624b86e8627d0ec2912aec9d04cd9a3f98b85746", - "48": "970e2375d1449b525d9e1448a31fff063bd9994ac3e0b5e5394a5601ba37f68c", - "49": "2ec4923e1b9199f5aafb1cd90cab3ddb4b61b1fe27230babd6d3e3d5a99437a9", - "5": "ab1b43f9773d3c8814fe703ab076598676994f497f7b518efc7c73287366488a", - "50": "071e5fcae402f4fe84e8182c7ed175c24951b9b3b3d41e70a612356a4832a8ab", - "51": "16ac35130dcfe4009551eb6e3c7bedc758da7af7c41dcc40845bd5c9bed1db04", - "52": "31a17176f2865f70dc8d641563c55672014c8c37c11b834689d60f49600e733c", - "53": "04f52e3aa82ddc22536589d23eec3e5d313dc21d2a1493b1ee6240480a7d0f76", - "54": "2f9446585481aa76c6a8074e5212a1408958d62ff44bf3344cdb5f165d7584e8", - "55": "2a13a454cc9be2c505b156253f1104668eebdb724dc1330155131b46a3e48a79", - "56": "c9c12b31a8aa28739a618587a3915bd21457775387d69cd30cd8e7b2f86e866b", - "57": "b1d47c4552b93fa76df85fcd34b6f771095f7f00f3fe323c325c9f2e4b249438", - "58": "fb6f6443fb0c836f7ce4999c800a29648bdc4b9f6d69c01d1664631f5a1f33d1", - "59": "af09f6198225a9cc2dfd4ef10d4276c0ef5580c004b01547de2cf5e5e534a0a6", - "6": "c4ccc38dd0ac0e64736e2f21502990a5cda9791e68150c7ef688d050792cb37e", - "60": "e33c99378c66237a90297a05be33e67ebbf3a0d1d1d61ae36237005fa9945194", - "61": "bf31c9c41afd790d5400e98f31dbc19da1d5f22dd1cb62d6ecc3bdf4ca480b9d", - "62": "1b5ec55411b01a7a7e00ebfedc73dfa66ad01ffc2501410f671893aea4c9e6c1", - "63": "8c6ca9745a20689ecbb23f2b3a51dd4a4ec33d8da3f09a5342c704b45a374d54", - "64": "f4c6ce377965f2fb53d583e7ce80923b9dfd462e325fd9e6c92d24f2a03d110e", - "65": "378c535e5425529c682f6c5bb87675241a425bc7e4334b7a50a94e5d673be96a", - "66": "fec6d6e091d2451388628985c4ad70f450125da2a9584a85691fa5b5ba88fd6a", - "67": "c95ada5fb892c66dce20797100f3c09ecccea38bb713dc00fb53f50f23c84992", - "7": "eed7359db9622d3414b709a29a3fe612bef17fa997146df06bb2a280396e24a9", - "8": "9cf4aea8ae9949ae25ac095655fbe0d70dbd7fda64e3c9c6adda2b17a4027ad8", - "9": "263a8e66cd5802c731a74812b123d600e90696da2b2b35a0ddfc652bf8b62f71" + "0": "9701ca27a9122202a8accbe2d887d2b4e4d94505f2c59ddc2c3c5d11fcf4a650", + "1": "bc0e1b8161bdd21fe6aae9bf926dededbdb68e8dcff531c372bada44e282e422", + "10": "837007f048151146fd2cf6c6056831964afb7d5e3f2aedd69b3537b786e2cad3", + "11": "d588c9d2f9087b4d13ff55c0ed78c21d6c7afd1cbda35d7e5b2e6cf26a2e719d", + "12": "36f795569fbf68acfa8755fa79bd2aaa7e4310cd63fc86e17f8a0e45634a4cc8", + "13": "0ada2e4afbaaf39cffc71f398d800e1ea1088e7b3ee6500bde4a395be71f4037", + "14": "264d961b08f22c5c25b86f160b04491b005e63139682f1396f2cc668d6429a62", + "15": "66bdfa73724e929309e85e493085b0b8e7a9a5789d5136947e9918159d63eb8f", + "16": "3c99d6eba3829f625ffdbccb8019cd2c180c9153a8cdf42d871c04c8eb170016", + "17": "44628e1309353b6d554af771f711905417f860d624b6b2562724925549301975", + "18": "319083fae4e47202bf3ff8cd5a07ebec2fe9a75750f70bfff80a2b46df25ac73", + "19": "71fac1cc337d732e20badfbee796dc556fc16479d6f390d164fb18e9f8c1cfce", + "2": "d1461ce64be0cab265113a2846223f7e7d62b3d28fd87dfa46d9f90d9906c49f", + "20": "dce561895528101c25ce8e8dde0ac119777020996263e8f02a4290c24ba4d8d2", + "21": "03cdbf24e818fc08a8d4773b1d795e1e87797c60361761cedebb188f1c9a41d5", + "22": "39d0946c5dbef24354f5b910047c55c75d5fce7b45a098816da24945ee246152", + "23": "10d551e2b2a3cf8657ee0d9a4a85384141df1ea5e28996b7e3b0d0fa1d69d35c", + "24": "eda54a8df595fb46dc6b49476e6adfd3ebaaf70f89b73d93990ccab3ab0f21b7", + "25": "d588c9d2f9087b4d13ff55c0ed78c21d6c7afd1cbda35d7e5b2e6cf26a2e719d", + "26": "d588c9d2f9087b4d13ff55c0ed78c21d6c7afd1cbda35d7e5b2e6cf26a2e719d", + "27": "907af9acf3710435a198eee4bef055aafb93a8ddb1d9e2b333baf3630958bdfa", + "28": "c5aa15364b33402a19478d79b34ca373bd30062ab52a58003ceff880f062988e", + "29": "83be0c145dd5bc48d36ed5c0a3148947d8ed7886ee186710a632a851fd8a6874", + "3": "e395f40b24fc4cdad4cfc46861420f3cba58c6cb09dd7187343cfa07f2e1e696", + "30": "05465e9b494f36a9ec35ae6211a8d91be5b0988996e206aa7ea9de38972f75e9", + "31": "c69983a4bc236fb8d132382f9126e770090875a241dceba7a3c91280ad1947e8", + "32": "0892df2a7a26ee65301a72cd784dd57aaeac4396a64e90be4e51142bdaf3fe3a", + "33": "b3c8ccd0ace81886865d5485728f6777c78211a35c69c00f866903d7055c83f2", + "34": "b9a97eefe3302a8966ef259e8005bdb268e0fbc71b7f15272b0945a93e68d81c", + "35": "303667207ad3f77cd7158069d639935c8c779660a48baee4f1f26635d52c2668", + "36": "67ecdbbfb92cfd9b55bd960717643a9740eb753870de1cae867666c9d60be1dc", + "37": "0a214d78f8220b28d05841502751b1ab7c99f43325787f54c0e31b040bdeefe2", + "38": "4980672786c3b89652ff6e93b6eabe80c3ddc4447256ed10a14895913c0b0986", + "39": "33e4f6424379b1a897ad90efaa0e0bf657223f63080113587b3891d0ada881c1", + "4": "d7ee147cecd7538d9ac3f6087f820a85fdbfb1dfa48044d7c3eb3d396baab560", + "40": "18b06ec0d37b0730dfe3c7b884d256a79555326c49755c37a482939bcb0a9435", + "41": "3dcd0a79f12f974b896c20e7b86d0ce0ecc10778b0af6d864ebc04bb3055a973", + "42": "999fedee2b1de9ff2da3edff00931d54f29a5fc4f47c550dbc52feaa368936b2", + "43": "307d5e4420e680542cd77d53a537eef0e9f73dccc92e0d6d28f7bbc75b2d90ba", + "44": "9b5aa388806791e41045eac81533b363e035bb26e46992340ad4665c2c5a1207", + "45": "da8de494dd607d08f01740cfe76025c532c1cdc6ce9f6345e84ed0765b18ce7d", + "46": "3bc777a99145570691675e905669fbbfd141f1c6e41396e4841b725cc02f19e7", + "47": "d2ba1a9b75946542d127a8d4c0c4b1e54aedc0f263c96d8a71890977d0d29ee1", + "48": "ae61f8fff6f666305fc1edc1ed6acc86e428877dd09f55f78c33d8a4c1c7bcd3", + "49": "91999d68afcd28a11e589944e8371d516a0bc6d0c48b9e27e43479c6828b0fc9", + "5": "d588c9d2f9087b4d13ff55c0ed78c21d6c7afd1cbda35d7e5b2e6cf26a2e719d", + "50": "e857cc8491ea707c791b41447cac59f3808f73c2b047400c1063c4c102b7ecdf", + "51": "adac76b1caa3a852bd1a47457cfeaf0f2a3ac6d9f04a93f5e86d7bad828fa905", + "52": "fff7bb8f877450e2638efd464119367c1e23c83071c06bde9b8c904f6de6742a", + "53": "306a985e6d07bc019c89a9851bf987f1e58caabe3538875db65be1d4db45bb1b", + "54": "0e1a84c5614990fa68907c703ccbd6e969a694791ab0d420114f47ad649a6373", + "55": "3e6d6bf7936e31fde226ac9122963d6aaa6aeb9028bee4d2ed23adf48e3536b6", + "56": "10f3f6d65cabfcaf6a5e9213a208c90930f9f6e86a4f17c8dd5b574d03a67323", + "57": "5b7a42cbc64aee9b5d7742faf6ebdf02704521a9f9de03e5900c4bff2d5d59be", + "58": "683250919a8e4f11c0009c6b86109f2ed881386420bb3099222a8fc81da64e62", + "59": "bc0f103913224d56bcbb037f75b1901cf12396ae1c1b8d2c4bff37645627ad6a", + "6": "e024a8b228b32bbd7fe9f7a2fce2cdf3b7ea93ceb6568cb20b1a6efee0542cd6", + "60": "c3f4bb8449658bbeb4bd02c134aac836b111d54698514d429c7aeca2911d1246", + "61": "0bb37048ce7745bfa88de737554ab5b75222bd5ecbab283885c4eb11f706f42c", + "62": "5bc30997585d0c078cfa6d9eab5158d3ac903fe16e6bb9f0d19246e0b344d96e", + "63": "869820c42229284ba199ab77d9b6c16ee1687e85020ad43f75a1fcceae92bb47", + "64": "1e3845d629eed51e85354dc3d31646354f767a349e3a08390b18b28d28b23525", + "65": "89523051b07e3f703887191e391d047aa5688b58e2083094e4d82cdc52859fc4", + "66": "884619cd327d081936f482f6e53217db56efe8b9032c36543752a8e13f3f6dd2", + "67": "f79be17ef75f8af025435f5dd686d2fa4d13f96af28797914cda674b1292a8f4", + "68": "f506a72086d0e3847d2c7bb55bda83fb14afd77e7d7190d62690429e828a4a36", + "69": "1b2746a0d11d9d22993c5f58314b7eaf913926a7b80f3c28ac96c4df8824b22f", + "7": "9f8857f7a94b9b8524bd631b0b1fcd3c89d8733d317d2ccfd9e26af3ff72e3ea", + "70": "2d702df8314f65d553d8f0e1d72d0d202625361d26ec2a361f7ff3dd012ea947", + "71": "a1e8c2259d9baa180cd31a07c3974ff01d000af87bc24a21a7e4b3a157e939ab", + "8": "9a4431d34fde7e376552fc757ff75be86b1c4c6194b3f95c93c0ab9fbbf9f2a4", + "9": "6267b3845952589a6f3594a1e07902bfd52ec3ecfaa85d398c4b002570f37e07" }, "gfx950_attention_tiled_2d_fastkv_regp": { - "0": "c57d0d3c94014b8fb4f2970dcea22e1d5fa1c59eae53c488477c1b4cbe7e1fa6", - "1": "5dbf1f1f94eafbf60dff8773ca8beb6025f4db7cbbbcdba67aca696ff4ea3401", - "2": "361a82466d9c33eb0c50d5d511c6e0c831ae3c317f192752c31bea4bd715829b", - "3": "1af7b32a44343599433ae0a320f28437014e4f00344578a4602d97e36a4a0466", - "4": "b7deee184c7b806981600cc7300b89d9653426bb3a85dd220dde96af788fb874", - "5": "7e8f4056da548709d5ab9d17edee524ba06ff2e16497198f9f0aae8a76735e2c", - "6": "4c890c3b268497e172919e8be183533f933d4602cbe9f0ca4f8b46aee0736a4f" + "0": "e1fadd21fc2e1e2701d08d91d04fbb52034b6bccb6205c6d28b42fb210a5d6ba", + "1": "a5e206ff0c428abb923eabdc858d3a5ebc5bc5102c932dd36ab7e939945c5361", + "2": "c18b047312b1d0f138ddb2781f4d84a0d6675b65f7586117d7525309e8de9680", + "3": "edb0cc4d37cb741f8df02610bca8c7a983103c94fa257baec5478cc019097e83", + "4": "d7533887bfa73579fe814b43a4a3cd9f4f69d12875770bce14bcfa20bd9806a6", + "5": "dccae9d102fe08468f192b138dc0d8f3df4ed9bcfd2564b288d33a1e8c3b33c3", + "6": "8b0d8fdf6de830b3cab247ca4f34e2aa97d6e82646dd4634a4d1b6cf4eb26495", + "7": "e7ea0c26ca0f5ccf4e40fd43f678844b31652ac912ab542c957cb87da52f23a9", + "8": "ce6c66a32ecaf2a4ab0745d05feecdb34d1cb6f9a0caa4fe8e1555e02a636a4c", + "9": "f66721e89813567bf21c5491bb483575d0868a98b4a47a7093d6a72666bab4c5" }, "gfx950_attention_tiled_3d": { - "0": "2d15cfdb485bc112c4364d6b000586b3da7cfa0403b170a7f4c4662a3003ab84", - "1": "0476ad71ba35fe47953f586a272750b88fba23a5f66aecabcec34062a1a5a7b7", - "10": "b0d08ee5431786ce546dce7a44e221a76b0ac9cdcb5afa2c8313fb2c2e0e1b44", - "11": "4870f2163281ea108cb8ae6383de76d6fb427ec80e558f6b6c076a607da18d6f", - "12": "9680fd6292de1079d9387a0e9947acbf0831cb64de2858ce4f19f879e5379d8e", - "13": "d34f0c7de3fd5d0467d139803dfe5bd17dad4c26628bd868f4245c05e292d0f4", - "14": "7a984f9f07137c77ea2f0704ad7a65a350e9fc96ab759497cba626b3fb766f52", - "15": "942c3c72cc504c539d4abc6201db2bab548871427f35a335258eee0614c1491e", - "16": "5276267bc23de9579470828f4d0452b249d301e70727437f315544ae9e3ed9c0", - "17": "99acbf37277a473e4cedd12d0595fc6b304f7af70c39fae62f08c0eb1a0a13a6", - "2": "1680787ea992fe51bc10f0d663dce835be3c0dc742b4a36023964a9c1e671ebf", - "3": "d543f94c41159c5af6822c0656497b16bcada09f475aa2205c2ed937becaa526", - "4": "e3c11f0bea0406fd826b858b25c0273db7685e0fc1d04cc8f90c4789fa474a9f", - "5": "d43dcd71e7156cfd4f15bec3022fd26ecbe83352068d1313fc824a1c17475de5", - "6": "f44fa8e200198874f65206294b327a10f1fcc62ec4b6f36e78efd52654299b44", - "7": "efaa5a76dca2b7fafcdd6a775d13559f38cbcabfea66b05e2cb94e86da519a7a", - "8": "2c72ff6b89e1d4a1883e7d3e82ca01a7d4de9dd98d54c1142bd40de8012fcf1a", - "9": "62a5bae4eef3aff5335a242803f79ef319a5e1a6ff80e984709c0b51e241be11" + "0": "185fe233b422855899789edbb842c03f215fad00c1695917e21cad755a16de46", + "1": "a13ef2bebf3c8d27861c800653beb21b0efdec7e1865146ad540b5e4333521cb", + "10": "bc305699e45ba27ddc4d9c747ef9f3557dcc34346e558e8c07405467ef11535a", + "11": "9753247ce408ad7f445143ca04a20c3ea3a6edd86a823386109750a3b38f44f1", + "12": "06e34a54b23416e64f2b9b995f2cfceb7e18e6bac98bfa5fa6245df6ee8b70ed", + "13": "4c9f99009b4997667cd8f5d378869782f35ef55664924a42094859a696df3138", + "14": "d657f73cff7d7c5ca2f8ff5c6143e5471564608be25a8e55bb0a39c7638d67fb", + "15": "23ecb293b7e44b601d6db8bc37f89de539e9089940561c836d8b832c81bb0d39", + "16": "9621c5fff3a88548ae2af919629d4cb69414e888bf79b4fdbd00fd809c1314ef", + "17": "1f9ca3d620ff2cfdce2738538c68c14088c585589e5da11b5b71aa4fb2708694", + "18": "1a7376af8650d21ffd67a3ee829bce1bbdc409d7c6faeccda29c9ec8582eed5c", + "19": "9d6bf157a0bfc34f90d79bac8d5f3dcbeb5c6d339552e13a3c85c51329978a1c", + "2": "fc583225c3e1a96067496e8ab7afa37fa6c6eb993b7bbc3d2f237fd57607f063", + "3": "7104dd650ee025a86f1830a1805541c315425bbac5c24b25c4e115874f50feff", + "4": "3bb308c0df37f7f375ebb7cfa77188a3f847cdd40627db54591a3a64f51ea694", + "5": "5a6f8e3e40767268e22d73f6dbe7a1db0d682130f795e49bc46fc8e3e18836eb", + "6": "dbc33c5d0880dd8a476682f38346ab2305dfc211c9511f0f252a8e978419cc6b", + "7": "f95f155e633137d51def2007f23ed9b0f5123e0521f43fd712d4ce02b3d0850a", + "8": "048a04ffe10fda0e1a274e78ec9ac2a06c216e6415d38e9e309478ae57d6d7ae", + "9": "51c6643e3d5d31cf6ae87b54e8c40b66ac2fd0d6f6651a6117f09505ad0ae60f" }, "gfx950_block_scaled_gemm": { "0": "a550fa8769391a2bf246ee73ad70769826d1f902db0a49bb60a80e7c0efda41e", diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/dynamic_helpers_emit.c b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/dynamic_helpers_emit.c index 0d5cbfdf1f15..c35a45c88620 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/dynamic_helpers_emit.c +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/dynamic_helpers_emit.c @@ -337,6 +337,37 @@ static void build_unmerge_mixed(rocke_ir_builder_t* b) rocke_b_ret(b); } +/* Python build_mixed_strides: create() with a Value, an int and an int-1 + * stride, through offset and offset_i64_split. */ +static void build_mixed_strides(rocke_ir_builder_t* b) +{ + rocke_value_t* o = ptr_param(b, "O", rocke_i32(), false); + rocke_value_t* o64 = ptr_param(b, "O64", rocke_i64(), false); + rocke_value_t* p_sb = rocke_b_param(b, "p_sB", rocke_i32(), NULL); + rocke_value_t* tid = rocke_b_thread_id_x(b); + + const char* base_names[3] = {"blk", "tok", "d"}; + rocke_value_t* dyn[3] = {p_sb, NULL, NULL}; + const int stat[3] = {0, 64, 1}; + rocke_tensor_descriptor_t* desc + = rocke_tensor_descriptor_naive_dynamic_mixed(b, "kv_mixed", base_names, 3, dyn, stat); + + rocke_value_t* in_values[3] = {tid, tid, tid}; + rocke_value_t* off = NULL; + rocke_value_t* valid = NULL; + if(!rocke_transforms_descriptor_offset(b, desc, base_names, in_values, 3, &off, &valid)) + return; + rocke_b_global_store(b, o, tid, off, 1); + rocke_value_t* base_i64 = NULL; + rocke_value_t* within = NULL; + if(!rocke_transforms_descriptor_offset_i64_split( + b, desc, "blk", base_names, in_values, 3, &base_i64, &within, &valid)) + return; + rocke_b_global_store(b, o64, tid, base_i64, 1); + rocke_b_global_store(b, o, within, tid, 1); + rocke_b_ret(b); +} + static void build_pad_dynamic(rocke_ir_builder_t* b) { rocke_value_t* x = ptr_param(b, "X", rocke_f16(), true); @@ -431,6 +462,7 @@ static const build_fn_t CONFIGS[] = { build_unmerge_mixed, build_chiplet_runtime_default, build_chiplet_runtime_chunk, + build_mixed_strides, }; static const int NUM_CONFIGS = (int)(sizeof(CONFIGS) / sizeof(CONFIGS[0])); diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/dynamic_helpers_emit.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/dynamic_helpers_emit.py index c2dc9aa0f6b4..292b7e4136a2 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/dynamic_helpers_emit.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/parity/dynamic_helpers_emit.py @@ -18,7 +18,7 @@ # under different names. # # arch = gfx950, llvm_flavor = AUTO, matching the C side. -from rocke.core.ir import F16, F32, I32, IRBuilder, KernelDef, PtrType +from rocke.core.ir import F16, F32, I32, I64, IRBuilder, KernelDef, PtrType from rocke.helpers.grid import chiplet_transform_chunked_runtime from rocke.helpers.pipeline import SoftwarePipeline from rocke.helpers.schedule import SchedulePolicy @@ -207,6 +207,28 @@ def build_unmerge_mixed(b: IRBuilder) -> None: b.ret() +def build_mixed_strides(b: IRBuilder) -> None: + """DynamicTensorDescriptor.create with a Value, an int and an int-1 stride. + + The tiled attention K/V descriptor's shape: the page stride is a kernarg, + the token stride a constant, the head dim contiguous. Covers both stride + hooks: offset (i32 terms) and offset_i64_split (the i64 page term). + """ + o = _ptr(b, "O", I32) + o64 = _ptr(b, "O64", I64) + p_sb = b.param("p_sB", I32) + tid = b.thread_id_x() + desc = DynamicTensorDescriptor.create( + "kv_mixed", coord_names=("blk", "tok", "d"), strides=(p_sb, 64, 1) + ) + off, _ = desc.offset(b, blk=tid, tok=tid, d=tid) + b.global_store(o, tid, off) + base_i64, within, _ = desc.offset_i64_split(b, "blk", blk=tid, tok=tid, d=tid) + b.global_store(o64, tid, base_i64) + b.global_store(o, within, tid) + b.ret() + + def build_pad_dynamic(b: IRBuilder) -> None: """embed_dynamic with every argument a Value, plus pad_dynamic variants. @@ -318,6 +340,8 @@ def build(b: IRBuilder) -> None: _chiplet_runtime(explicit_chunk=False), # 9: chiplet_transform_chunked_runtime with an explicit runtime chunk. _chiplet_runtime(explicit_chunk=True), + # 10: DynamicTensorDescriptor with Value / int / int-1 strides, i32 and i64 paths. + build_mixed_strides, ] diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/tiled_attention_entry_guards.cpp b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/tiled_attention_entry_guards.cpp new file mode 100644 index 000000000000..62ada3bf0c58 --- /dev/null +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/instances/tiled_attention_entry_guards.cpp @@ -0,0 +1,205 @@ +// Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT +/* + * Host-only test that the C tiled-attention entries refuse what the Python + * builders refuse, with the Python reason text, instead of crashing: + * + * - a zero or non-dividing head pair, on the gfx942/gfx950 2D and 3D lowers + * and the gfx950 fastKV register-P lower (the kernel names carry the GQA + * ratio, and the shared 2D scaffold divides by num_kv_heads); + * - fastKV register-P on gfx942, on its lower and its support predicate (the + * body is gfx950-only). + * + * The expected texts are the Python ones: kernels.common._fmha_common + * check_head_counts and the fastKV module's _NOT_GFX950. The pytest twin, + * test_attention_tiled_cpp_entry_guards.py, compares the two engines directly + * where the rocke_engine binding is built. + */ +#include +#include +#include + +#include "rocke/instance_gfx942_attention_tiled_2d.h" +#include "rocke/instance_gfx942_attention_tiled_3d.h" +#include "rocke/instance_gfx950_attention_tiled_2d.h" +#include "rocke/instance_gfx950_attention_tiled_2d_fastkv_regp.h" +#include "rocke/instance_gfx950_attention_tiled_3d.h" +#include "rocke/lower_llvm.h" + +static const char* const NOT_POSITIVE = "num_query_heads / num_kv_heads must be > 0 (got 32, 0)"; +static const char* const NOT_DIVIDING + = "num_query_heads (32) must be divisible by num_kv_heads (6) for GQA / MQA"; +static const char* const NOT_GFX950 + = "fastKV register-P is a gfx950 kernel; gfx942 is not supported"; + +typedef struct head_pair +{ + int num_query_heads; + int num_kv_heads; + const char* reason; +} head_pair_t; + +static const head_pair_t BAD_PAIRS[] = {{32, 0, NOT_POSITIVE}, {32, 6, NOT_DIVIDING}}; + +/* Fails unless the call was refused (non-OK, no output) with `reason` in its + * diagnostic. Frees any output it was handed. */ +static int expect_refused( + const char* what, rocke_status_t st, char* ll, const char* err, const char* reason) +{ + const bool refused = st != ROCKE_OK && ll == NULL; + const bool says_why = strstr(err, reason) != NULL; + free(ll); + if(refused && says_why) + { + printf("%-40s refused: %s\n", what, err); + return 0; + } + fprintf(stderr, + "%-40s expected a refusal with \"%s\", got status=%d ll=%s err=\"%s\"\n", + what, + reason, + (int)st, + ll != NULL ? "" : "NULL", + err); + return 1; +} + +static rocke_attention_tiled_2d_spec_t spec_2d(int num_query_heads, int num_kv_heads) +{ + rocke_attention_tiled_2d_spec_t s = rocke_attention_tiled_2d_spec_default(); + s.head_size = 128; + s.block_size = 16; + s.num_query_heads = num_query_heads; + s.num_kv_heads = num_kv_heads; + s.dtype = "bf16"; + s.use_sinks = false; + s.sliding_window = 0; + s.has_softcap = false; + return s; +} + +/* make_fastkv_register_p_spec over the experiment's bf16 d64_b32 shape. */ +static rocke_attention_tiled_2d_spec_t spec_fastkv(int num_query_heads, int num_kv_heads) +{ + rocke_attention_tiled_2d_spec_t s = spec_2d(num_query_heads, num_kv_heads); + s.head_size = 64; + s.block_size = 32; + s.use_fast_paged_kv_desc = true; + s.use_mfma_32x32 = true; + s.use_transposed_qk_32x32 = true; + return s; +} + +static rocke_unified_attention_3d_tiled_spec_t spec_3d(int num_query_heads, int num_kv_heads) +{ + rocke_unified_attention_3d_tiled_spec_t s = rocke_unified_attention_3d_tiled_spec_default(); + s.head_size = 128; + s.block_size = 16; + s.num_query_heads = num_query_heads; + s.num_kv_heads = num_kv_heads; + s.dtype = "bf16"; + s.use_sinks = false; + s.sliding_window = 0; + s.has_softcap = false; + s.num_segments = 16; + return s; +} + +static int check_head_counts(void) +{ + int failures = 0; + for(size_t i = 0; i < sizeof(BAD_PAIRS) / sizeof(BAD_PAIRS[0]); ++i) + { + const head_pair_t* p = &BAD_PAIRS[i]; + char what[64]; + char err[512]; + char* ll; + rocke_status_t st; + + const rocke_attention_tiled_2d_spec_t s2 = spec_2d(p->num_query_heads, p->num_kv_heads); + const rocke_unified_attention_3d_tiled_spec_t s3 + = spec_3d(p->num_query_heads, p->num_kv_heads); + const rocke_attention_tiled_2d_spec_t sf = spec_fastkv(p->num_query_heads, p->num_kv_heads); + + ll = NULL; + err[0] = '\0'; + st = rocke_gfx942_attention_tiled_2d_lower_to_llvm( + &s2, "gfx942", ROCKE_LLVM_FLAVOR_AUTO, &ll, err, sizeof(err)); + snprintf(what, sizeof(what), "gfx942 2D %d/%d", p->num_query_heads, p->num_kv_heads); + failures += expect_refused(what, st, ll, err, p->reason); + + ll = NULL; + err[0] = '\0'; + st = rocke_gfx950_attention_tiled_2d_lower_to_llvm( + &s2, "gfx950", ROCKE_LLVM_FLAVOR_AUTO, &ll, err, sizeof(err)); + snprintf(what, sizeof(what), "gfx950 2D %d/%d", p->num_query_heads, p->num_kv_heads); + failures += expect_refused(what, st, ll, err, p->reason); + + ll = NULL; + err[0] = '\0'; + st = rocke_build_unified_attention_3d_tiled_gfx942_lower_to_llvm( + &s3, "gfx942", ROCKE_LLVM_FLAVOR_AUTO, &ll, err, sizeof(err)); + snprintf(what, sizeof(what), "gfx942 3D %d/%d", p->num_query_heads, p->num_kv_heads); + failures += expect_refused(what, st, ll, err, p->reason); + + ll = NULL; + err[0] = '\0'; + st = rocke_build_unified_attention_3d_tiled_gfx950_lower_to_llvm( + &s3, "gfx950", ROCKE_LLVM_FLAVOR_AUTO, &ll, err, sizeof(err)); + snprintf(what, sizeof(what), "gfx950 3D %d/%d", p->num_query_heads, p->num_kv_heads); + failures += expect_refused(what, st, ll, err, p->reason); + + ll = NULL; + err[0] = '\0'; + st = rocke_gfx950_attention_tiled_2d_fastkv_regp_lower_to_llvm( + &sf, "gfx950", ROCKE_LLVM_FLAVOR_AUTO, &ll, err, sizeof(err)); + snprintf( + what, sizeof(what), "gfx950 fastKV regp %d/%d", p->num_query_heads, p->num_kv_heads); + failures += expect_refused(what, st, ll, err, p->reason); + } + return failures; +} + +static int check_fastkv_arch(void) +{ + int failures = 0; + char err[512]; + char* ll = NULL; + const rocke_attention_tiled_2d_spec_t s = spec_fastkv(64, 8); + + err[0] = '\0'; + const rocke_status_t st = rocke_gfx950_attention_tiled_2d_fastkv_regp_lower_to_llvm( + &s, "gfx942", ROCKE_LLVM_FLAVOR_AUTO, &ll, err, sizeof(err)); + failures += expect_refused("gfx950 fastKV regp lower on gfx942", st, ll, err, NOT_GFX950); + + /* supports_fastkv_register_p_2d(arch="gfx942") over a shape it otherwise admits. */ + rocke_gfx950_supports_fastkv_register_p_2d_args_t a; + memset(&a, 0, sizeof(a)); + a.head_size = 64; + a.block_size = 32; + a.dtype = "bf16"; + a.num_queries_per_kv = 8; + a.num_query_heads = 64; + a.num_kv_heads = 8; + a.arch = "gfx942"; + err[0] = '\0'; + const bool ok = rocke_gfx950_supports_fastkv_register_p_2d(&a, err, sizeof(err)); + failures += expect_refused("gfx950 fastKV regp predicate on gfx942", + ok ? ROCKE_OK : ROCKE_ERR_VALUE, + NULL, + err, + NOT_GFX950); + return failures; +} + +int main(void) +{ + const int failures = check_head_counts() + check_fastkv_arch(); + if(failures != 0) + { + fprintf(stderr, "%d TILED ATTENTION ENTRY GUARD CHECK(S) FAILED\n", failures); + return 1; + } + printf("ALL TILED ATTENTION ENTRY GUARDS REFUSED WITH THE PYTHON REASON\n"); + return 0; +} diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/descriptors/rocKE/gfx950_attention_unified_2d/README.md b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/descriptors/rocKE/gfx950_attention_unified_2d/README.md index f9d7280c376c..a10ae6fc94cc 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/descriptors/rocKE/gfx950_attention_unified_2d/README.md +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/descriptors/rocKE/gfx950_attention_unified_2d/README.md @@ -26,8 +26,9 @@ plan's workspace (`packs/Gfx950AttentionUnified2DGeometry.hpp`). The full space's wide 32x32 geometry (4 warps x 32 rows, KV tile 64) won no graph and is not shipped. -- Batch capacity 511: `num_seqs` 511 bakes 9 binary-search iterations, which resolve at most - 2^9 - 1 sequences. The engine declines a graph with a larger batch. +- Batch: any. The kernel takes the batch as the `num_seqs` kernarg and derives its `cu_q` + binary-search trip count from it at runtime, so no batch is baked. The KMD still declares + `num_seqs` (511); the engine does not read it. - Decode (seqlen_q 1) whose GQA group divides 16 is left to `hipkernel:Gfx950AttentionUnified3D`. - Fixed in every variant: no sliding window, no sinks, no ALiBi, no softcap, no fp8, 32-bit K/V addressing (K/V larger than 0x7FFF0000 bytes is declined). diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified2DGeometry.hpp b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified2DGeometry.hpp index a8405b5968d2..590f6541882e 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified2DGeometry.hpp +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified2DGeometry.hpp @@ -14,17 +14,17 @@ /** * @file Gfx950AttentionUnified2DGeometry.hpp * @brief What gfx950 unified-attention 2D restates from its Python builder: the geometry - * rules, and the launch grid, batch capacity and paged-KV metadata of - * UnifiedAttentionGeometry.hpp bound to the gfx950 traits. + * rules, and the launch grid and paged-KV metadata of UnifiedAttentionGeometry.hpp + * bound to the gfx950 traits. * * A rocKE kernel is launched from Python; an ingestor engine relaunches the same binary * from C++. Nothing compares the two halves, and a mismatch does not fault: the kernel * computes something else. Everything here is a pure function of a candidate's KMD fields * and the graph's dimensions, so it is testable with no HIP context. * - * The arch-neutral contract (identity block table over a contiguous BSHD K/V, the batch - * capacity a baked `num_seqs` gives, the grid) lives in UnifiedAttentionGeometry.hpp; this - * header adds only what gfx950's builder fixes: its traits and its geometry legality. + * The arch-neutral contract (identity block table over a contiguous BSHD K/V, the grid) + * lives in UnifiedAttentionGeometry.hpp; this header adds only what gfx950's builder fixes: + * its traits and its geometry legality. */ namespace hip_kernel_provider::kernel_ingestor_engine { @@ -137,18 +137,6 @@ inline bool isSupportedGfx950Unified2DGeometry(const Gfx950Unified2DGeometryFiel return !g.useTransposedQk32x32 || g.useMfma32x32; } -/// `unifiedAttentionBinarySearchIters`, named for this pack. -inline int64_t gfx950Unified2DBinarySearchIters(int64_t numSeqs) -{ - return unifiedAttentionBinarySearchIters(numSeqs); -} - -/// `unifiedAttentionBatchCapacity`: 511 -> 9 iterations -> 511 sequences. -inline int64_t gfx950Unified2DBatchCapacity(int64_t numSeqs) -{ - return unifiedAttentionBatchCapacity(numSeqs); -} - using Gfx950Unified2DLaunchGeometry = UnifiedAttentionLaunchGeometry; /// `unifiedAttention2DLaunchGeometry` with the gfx950 traits: kv-major grid diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified2DNative.cpp b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified2DNative.cpp index 06f00d0eb3de..2062a3deec9c 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified2DNative.cpp +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified2DNative.cpp @@ -77,7 +77,9 @@ constexpr std::string_view DISPATCH_SYMBOL = "hipkernel.gfx950_attention_unified // KMD fields (14-field schema). Fields a shipped variant holds at one value across the whole // catalog (waves_per_eu, kv_storage_dtype, the sink/alibi/qq-bias/softcap switches) are absent: // a variant that moved one of them would have to add it here first, or the candidates collide -// on the catalog key and the loader drops one. +// on the catalog key and the loader drops one. `num_seqs` is declared but not read: the kernel +// takes the batch as a kernarg and derives its cu_q search trip count from it, so no batch is +// baked. constexpr std::string_view DTYPE_FIELD = "dtype"; constexpr std::string_view HEAD_SIZE_FIELD = "head_size"; constexpr std::string_view NUM_QUERY_HEADS_FIELD = "num_query_heads"; @@ -85,7 +87,6 @@ constexpr std::string_view NUM_KV_HEADS_FIELD = "num_kv_heads"; constexpr std::string_view BLOCK_SIZE_FIELD = "block_size"; constexpr std::string_view MASK_TYPE_FIELD = "mask_type"; constexpr std::string_view SLIDING_WINDOW_FIELD = "sliding_window"; -constexpr std::string_view NUM_SEQS_FIELD = "num_seqs"; constexpr std::string_view TILE_SIZE_FIELD = "tile_size"; constexpr std::string_view NUM_WARPS_FIELD = "num_warps"; constexpr std::string_view BLOCK_M_PER_WARP_FIELD = "block_m_per_warp"; @@ -241,7 +242,6 @@ struct AttentionUnified2DCandidate int64_t blockSize = 0; int64_t maskType = 0; int64_t slidingWindow = 0; - int64_t numSeqs = 0; int64_t tileSize = 0; int64_t numWarps = 0; int64_t blockMPerWarp = 0; @@ -266,15 +266,13 @@ std::optional candidateFor(const KernelDefinition& const auto blockSize = integerMetadata(kernel, BLOCK_SIZE_FIELD); const auto maskType = integerMetadata(kernel, MASK_TYPE_FIELD); const auto slidingWindow = integerMetadata(kernel, SLIDING_WINDOW_FIELD); - const auto numSeqs = integerMetadata(kernel, NUM_SEQS_FIELD); const auto tileSize = integerMetadata(kernel, TILE_SIZE_FIELD); const auto numWarps = integerMetadata(kernel, NUM_WARPS_FIELD); const auto blockMPerWarp = integerMetadata(kernel, BLOCK_M_PER_WARP_FIELD); const auto useMfma32x32 = integerMetadata(kernel, USE_MFMA_32X32_FIELD); const auto useTransposedQk32x32 = integerMetadata(kernel, USE_TRANSPOSED_QK_32X32_FIELD); if(!headSize || !numQueryHeads || !numKvHeads || !blockSize || !maskType || !slidingWindow - || !numSeqs || !tileSize || !numWarps || !blockMPerWarp || !useMfma32x32 - || !useTransposedQk32x32) + || !tileSize || !numWarps || !blockMPerWarp || !useMfma32x32 || !useTransposedQk32x32) { return std::nullopt; } @@ -293,9 +291,7 @@ std::optional candidateFor(const KernelDefinition& { return std::nullopt; } - // A capacity of 0 or less would make the kernel run its unspecialized 32-iteration - // search; the catalog always states one, so a record without it is malformed. - if(*numSeqs <= 0 || *useMfma32x32 < 0 || *useMfma32x32 > 1 || *useTransposedQk32x32 < 0 + if(*useMfma32x32 < 0 || *useMfma32x32 > 1 || *useTransposedQk32x32 < 0 || *useTransposedQk32x32 > 1) { return std::nullopt; @@ -318,7 +314,6 @@ std::optional candidateFor(const KernelDefinition& *blockSize, *maskType, *slidingWindow, - *numSeqs, *tileSize, *numWarps, *blockMPerWarp, @@ -335,10 +330,10 @@ std::optional candidateFor(const KernelDefinition& * what this checks against the graph: * * - dtype, head_size, num_query_heads, num_kv_heads, the mask kind and sliding window; - * - block_size, which must divide seqlen_kv (the identity cache is whole blocks); - * - the batch CAPACITY (`num_seqs`): the kernel's binary search resolves at most - * 2^iters - 1 sequences, so a larger batch reads another sequence's lengths without a - * fault. + * - block_size, which must divide seqlen_kv (the identity cache is whole blocks). + * + * The batch is not bounded here: the kernel derives its cu_q search trip count from the + * `num_seqs` kernarg, so one binary serves every batch the identity table can index. */ bool kernelMatches(const MatchContext& context, const BoundTokens& bound, @@ -396,13 +391,8 @@ bool kernelMatches(const MatchContext& context, // The identity cache must be whole blocks of THIS candidate's size, and small enough for // the table the kernel indexes in 32 bits. - if(!gfx950Unified2DWorkspaceLayout(problem.batch, problem.seqLenKv, candidate->blockSize) - .has_value()) - { - return false; - } - - return problem.batch <= gfx950Unified2DBatchCapacity(candidate->numSeqs); + return gfx950Unified2DWorkspaceLayout(problem.batch, problem.seqLenKv, candidate->blockSize) + .has_value(); } /// Scores for the cold ranking. The baseline geometry scores `BASELINE_SCORE`; every @@ -456,8 +446,9 @@ double scoreKernel(const MatchContext& /*context*/, // Dispatch // --------------------------------------------------------------------------- -/// The kernel signature for all variants in this engine: 10 pointers, 6 scalars and the two -/// strides (`_attn_signature(include_bt_stride=True, include_qq_bias_stride=True)`). +/// The kernel signature for all variants in this engine: 10 pointers, 6 scalars, the two +/// strides (`_attn_signature(include_bt_stride=True, include_qq_bias_stride=True)`) and the +/// runtime shape kernargs (`RUNTIME_KERNARG_FIELDS`: head counts and window). /// /// NAMES ARE LOAD-BEARING. requireSignatureMatch compares kind and size always, but names /// only when BOTH sides carry one. Kind and size alone cannot tell the ten pointers apart, nor @@ -493,7 +484,10 @@ std::vector unified2DKernelSignature() KernelArgument{"by_value", F32, 96, "softcap"}, KernelArgument{"by_value", I32, 100, "num_seqs"}, KernelArgument{"by_value", I32, 104, "block_table_stride"}, - KernelArgument{"by_value", I32, 108, "qq_bias_stride_0"}}; + KernelArgument{"by_value", I32, 108, "qq_bias_stride_0"}, + KernelArgument{"by_value", I32, 112, "num_query_heads"}, + KernelArgument{"by_value", I32, 116, "num_kv_heads"}, + KernelArgument{"by_value", I32, 120, "sliding_window"}}; } /// The compiled kernel plus everything launch() needs, owning nothing that points back into @@ -505,12 +499,14 @@ class PreparedGfx950AttentionUnified2D : public PreparedDispatch UnifiedAttentionBinding binding, UnifiedAttentionProblem problem, Gfx950Unified2DWorkspaceLayout layout, - std::vector workspaceImage) + std::vector workspaceImage, + int64_t slidingWindow) : _code(std::move(code)) , _binding(binding) , _problem(problem) , _layout(layout) , _workspaceImage(std::move(workspaceImage)) + , _slidingWindow(slidingWindow) { } @@ -539,12 +535,19 @@ class PreparedGfx950AttentionUnified2D : public PreparedDispatch return _workspaceImage; } + /// The variant's window, which the kernel reads as a kernarg. + int64_t slidingWindow() const + { + return _slidingWindow; + } + private: IngestorKernelCode _code; UnifiedAttentionBinding _binding; UnifiedAttentionProblem _problem; Gfx950Unified2DWorkspaceLayout _layout; std::vector _workspaceImage; + int64_t _slidingWindow = 0; }; /// The graph's problem, read from the bound Q and K uids. A null tensor means a bound uid @@ -586,15 +589,6 @@ Gfx950Unified2DWorkspaceLayout requireLayout(const AttentionUnified2DCandidate& const KernelDefinition& kernel) { const std::string name = toString(kernel.kernelId); - const auto capacity = gfx950Unified2DBatchCapacity(candidate.numSeqs); - if(problem.batch > capacity) - { - throw hipdnn_plugin_sdk::HipdnnPluginException( - HIPDNN_PLUGIN_STATUS_BAD_PARAM, - "gfx950 attention_unified_2d: kernel '" + name + "' serves at most " - + std::to_string(capacity) + " sequences and the graph has " - + std::to_string(problem.batch)); - } const auto layout = gfx950Unified2DWorkspaceLayout(problem.batch, problem.seqLenKv, candidate.blockSize); if(!layout.has_value()) @@ -683,7 +677,7 @@ class Gfx950AttentionUnified2DDispatchHandler : public IKernelDispatchHandler( - std::move(code), binding, problem, layout, std::move(*image)); + std::move(code), binding, problem, layout, std::move(*image), candidate.slidingWindow); } void launch(const Handle& handle, @@ -742,25 +736,29 @@ class Gfx950AttentionUnified2DDispatchHandler : public IKernelDispatchHandler(p.batch), - static_cast(layout.blocksPerSeq), - noQqBiasStride); + preparedUnified.kernelForStream(stream).launch( + stream, + o.ptr, + q.ptr, + k.ptr, + v.ptr, + unused, + blockTables, + seqLens, + unused, + unused, + queryStartLen, + binding.scale, + neutralScale, + neutralScale, + neutralScale, + noSoftcap, + static_cast(p.batch), + static_cast(layout.blocksPerSeq), + noQqBiasStride, + static_cast(p.numQueryHeads), + static_cast(p.numKvHeads), + static_cast(preparedUnified.slidingWindow())); } private: diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified3DNative.cpp b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified3DNative.cpp index 409fc73f9e77..13a0b3aefa6f 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified3DNative.cpp +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified3DNative.cpp @@ -448,7 +448,8 @@ double scoreKernel(const MatchContext& /*context*/, // --------------------------------------------------------------------------- /// The segment kernel's signature: 12 pointers (the three scratch outputs first), 4 f32 -/// scalars and three i32 (`_3d_signature`). +/// scalars, three i32 and the runtime shape kernargs (`_tiled_3d_signatures`: head counts and +/// window). /// /// NAMES ARE LOAD-BEARING. requireSignatureMatch compares kind and size always, but names /// only when BOTH sides carry one; kind and size alone cannot tell the twelve pointers @@ -479,21 +480,26 @@ std::vector unified3DSegmentSignature() KernelArgument{"by_value", F32, 108, "softcap"}, KernelArgument{"by_value", I32, 112, "num_seqs"}, KernelArgument{"by_value", I32, 116, "block_table_stride"}, - KernelArgument{"by_value", I32, 120, "qq_bias_stride_0"}}; + KernelArgument{"by_value", I32, 120, "qq_bias_stride_0"}, + KernelArgument{"by_value", I32, 124, "num_query_heads"}, + KernelArgument{"by_value", I32, 128, "num_kv_heads"}, + KernelArgument{"by_value", I32, 132, "sliding_window"}}; } -/// The reduce kernel's signature: the output then the three scratch arrays it folds, and -/// seq_lens (`_reduce_signature`). Names load-bearing as above; keep them identical to -/// build_unified_attention_reduce_tiled's `b.param` calls. +/// The reduce kernel's signature: the output then the three scratch arrays it folds, +/// seq_lens and the runtime query head count (`_tiled_3d_signatures`). Names load-bearing as +/// above; keep them identical to build_unified_attention_reduce_tiled's `b.param` calls. std::vector unified3DReduceSignature() { constexpr auto PTR = static_cast(sizeof(void*)); + constexpr auto I32 = static_cast(sizeof(int32_t)); return {KernelArgument{"global_buffer", PTR, 0, "output_ptr"}, KernelArgument{"global_buffer", PTR, 8, "segm_output_ptr"}, KernelArgument{"global_buffer", PTR, 16, "segm_max_ptr"}, KernelArgument{"global_buffer", PTR, 24, "segm_expsum_ptr"}, - KernelArgument{"global_buffer", PTR, 32, "seq_lens_ptr"}}; + KernelArgument{"global_buffer", PTR, 32, "seq_lens_ptr"}, + KernelArgument{"by_value", I32, 40, "num_query_heads"}}; } /// Both compiled kernels plus everything launch() needs, owning nothing that points back @@ -506,13 +512,15 @@ class PreparedGfx950AttentionUnified3D : public PreparedDispatch UnifiedAttentionBinding binding, UnifiedAttentionProblem problem, Gfx950Unified3DWorkspaceLayout layout, - std::vector workspaceImage) + std::vector workspaceImage, + int64_t slidingWindow) : _segment(std::move(segment)) , _reduce(std::move(reduce)) , _binding(binding) , _problem(problem) , _layout(layout) , _workspaceImage(std::move(workspaceImage)) + , _slidingWindow(slidingWindow) { } @@ -546,6 +554,12 @@ class PreparedGfx950AttentionUnified3D : public PreparedDispatch return _workspaceImage; } + /// The variant's window, which the kernel reads as a kernarg. + int64_t slidingWindow() const + { + return _slidingWindow; + } + private: IngestorKernelCode _segment; IngestorKernelCode _reduce; @@ -553,6 +567,7 @@ class PreparedGfx950AttentionUnified3D : public PreparedDispatch UnifiedAttentionProblem _problem; Gfx950Unified3DWorkspaceLayout _layout; std::vector _workspaceImage; + int64_t _slidingWindow = 0; }; /// The graph's problem, read from the bound Q and K uids. A null tensor means a bound uid @@ -718,8 +733,13 @@ class Gfx950AttentionUnified3DDispatchHandler : public IKernelDispatchHandler( - std::move(segment), std::move(reduce), binding, problem, layout, std::move(*image)); + return std::make_unique(std::move(segment), + std::move(reduce), + binding, + problem, + layout, + std::move(*image), + candidate.slidingWindow); } void launch(const Handle& handle, @@ -802,9 +822,17 @@ class Gfx950AttentionUnified3DDispatchHandler : public IKernelDispatchHandler(p.batch), static_cast(layout.identity.blocksPerSeq), - noQqBiasStride); - preparedUnified.reduceForStream(stream).launch( - stream, o.ptr, segmOutput, segmMax, segmExpsum, seqLens); + noQqBiasStride, + static_cast(p.numQueryHeads), + static_cast(p.numKvHeads), + static_cast(preparedUnified.slidingWindow())); + preparedUnified.reduceForStream(stream).launch(stream, + o.ptr, + segmOutput, + segmMax, + segmExpsum, + seqLens, + static_cast(p.numQueryHeads)); } private: diff --git a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/UnifiedAttentionGeometry.hpp b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/UnifiedAttentionGeometry.hpp index 55abeebd5681..dc79c07a5c4e 100644 --- a/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/UnifiedAttentionGeometry.hpp +++ b/dnn-providers/hip-kernel-provider/src/engines/kernel_ingestor_engine/packs/UnifiedAttentionGeometry.hpp @@ -16,8 +16,8 @@ /** * @file UnifiedAttentionGeometry.hpp * @brief The arch-neutral half of the unified-attention (vLLM `unified_attention`) packs: - * launch grid, batch capacity and the identity paged-KV metadata a contiguous K/V - * graph is served with, parameterized by the target's UnifiedAttentionArchTraits. + * launch grid and the identity paged-KV metadata a contiguous K/V graph is served + * with, parameterized by the target's UnifiedAttentionArchTraits. * * rocKE builds the unified-attention kernels per architecture, but the host contract around * them is shared: the kernel reads a paged K/V cache through `block_tables`, per-sequence @@ -95,41 +95,6 @@ inline bool isSupportedUnifiedAttentionBlockSize(const UnifiedAttentionArchTrait return detail::isOneOf(traits.blockSizes, blockSize); } -/** - * @brief Binary-search trip count a kernel was compiled with for a baked `num_seqs`. - * - * `binary_search_seq_idx_iters`: `max(1, ceil(log2(num_seqs + 1)))`, and 32 for a spec that - * declares no `num_seqs`. The same on every architecture. - */ -inline int64_t unifiedAttentionBinarySearchIters(int64_t numSeqs) -{ - constexpr int64_t UNSPECIALIZED_ITERS = 32; - if(numSeqs <= 0) - { - return UNSPECIALIZED_ITERS; - } - int64_t iters = 1; - // Smallest iters with 2^iters >= numSeqs + 1; bounded by the shift width anyway. - while(iters < UNSPECIALIZED_ITERS && (int64_t{1} << iters) < numSeqs + 1) - { - ++iters; - } - return iters; -} - -/** - * @brief The most sequences one launch of a kernel baked with @p numSeqs can address. - * - * The kernel bisects `[0, num_seqs_runtime]` in a fixed number of iterations (an unrolled - * scf.for), which resolves at most `2^iters - 1` sequences. A larger batch leaves the search - * unfinished: the last blocks read the wrong sequence's lengths and the launch still - * succeeds. 511 -> 9 iterations -> 511. - */ -inline int64_t unifiedAttentionBatchCapacity(int64_t numSeqs) -{ - return (int64_t{1} << unifiedAttentionBinarySearchIters(numSeqs)) - 1; -} - /// The grid and block a candidate must launch with. struct UnifiedAttentionLaunchGeometry { @@ -158,7 +123,7 @@ struct UnifiedAttentionLaunchGeometry * = (qblocks, num_kv_heads, 1) q-major (traits.qMajorGrid) * * and `launch_block`, `(waveLanes * num_warps, 1, 1)`. `total_q` is `batch * seqLenQ` and - * `num_seqs` the RUNTIME batch, not the baked capacity. + * `num_seqs` the runtime batch. * * Throws instead of returning a degenerate grid: an empty launch returns cleanly having * written nothing, and prepare() is the last place a named failure is cheap. diff --git a/dnn-providers/hip-kernel-provider/src/integration_tests/kernel_ingestor_engine/IntegrationGpuGfx950AttentionUnifiedKnobs.cpp b/dnn-providers/hip-kernel-provider/src/integration_tests/kernel_ingestor_engine/IntegrationGpuGfx950AttentionUnifiedKnobs.cpp index 5d390f64fe57..46ecef48abcd 100644 --- a/dnn-providers/hip-kernel-provider/src/integration_tests/kernel_ingestor_engine/IntegrationGpuGfx950AttentionUnifiedKnobs.cpp +++ b/dnn-providers/hip-kernel-provider/src/integration_tests/kernel_ingestor_engine/IntegrationGpuGfx950AttentionUnifiedKnobs.cpp @@ -249,8 +249,8 @@ std::string maskName(Mask mask) /// Every Unified2D kernel class -- geometry x dtype x mask -- forced once, on a prefill shape /// with B > 1 and Sq != Skv: the default geometry cycling through the head configurations -/// (every cell ships it), the others on a cell that ships them; then one cold case, which -/// the engine's ranking serves with the default geometry. +/// (every cell ships it), the others on a cell that ships them; then two cold cases, one at +/// B 512, which the engine's ranking serves with the default geometry. std::vector knobCases2D() { std::vector cases; @@ -290,14 +290,20 @@ std::vector knobCases2D() {DataType::BFLOAT16, 128, 8, 2, Mask::TOP_LEFT, 3, 512, 512}, {}, knobsOf(NARROW_4X16)}); + // B 512 is one past what the old baked 9-iteration cu_q search resolved; the kernel now + // derives the trip count from the num_seqs kernarg, so every batch is served. + cases.push_back({"Unified2D_Cold_BatchPastTheOldBakedSearch", + ENGINE_2D, + {DataType::BFLOAT16, 128, 8, 2, Mask::TOP_LEFT, 512, 16, 64}, + {}, + knobsOf(NARROW_4X16)}); return cases; } /// Every Unified3D kernel class -- num_segments x dtype x causal mask -- forced once on a /// decode shape (Sq 1): the default num_segments cycling through the head configurations, the /// others on a cell that ships them; a NO_MASK decode graph (served by the bottom-right -/// kernel); and a cold B = 1024 decode case, past any 2D batch capacity: the decode grid has -/// none. +/// kernel); and a cold B = 1024 decode case. std::vector knobCases3D() { std::vector cases; @@ -347,10 +353,6 @@ std::vector knobCases() std::vector declineCases() { return { - // 2D bakes a batch capacity of 511 into its binary search. - {"Unified2D_BatchBeyondTheBakedCapacity", - ENGINE_2D, - {DataType::BFLOAT16, 128, 8, 2, Mask::NO_MASK, 512, 16, 64}}, // The identity cache is cut into whole 16-token blocks. {"Unified2D_SeqKvNotAWholeNumberOfBlocks", ENGINE_2D, diff --git a/dnn-providers/hip-kernel-provider/src/tests/CMakeLists.txt b/dnn-providers/hip-kernel-provider/src/tests/CMakeLists.txt index ea88df505de3..ba7a81c952d3 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/CMakeLists.txt +++ b/dnn-providers/hip-kernel-provider/src/tests/CMakeLists.txt @@ -441,7 +441,7 @@ if(HIPDNN_ENABLE_KERNEL_INGESTOR) # suite below. foreach(_gfx950_engine IN ITEMS "dense|Gfx950AttentionDense|EveryShippedKernelsTileIsOneTheEngineLaunches" - "unified_2d|Gfx950AttentionUnified2D|EveryShippedKernelsGeometryIsOneTheEngineLaunches,EveryShippedKernelServesTheManifestsBatchCapacity" + "unified_2d|Gfx950AttentionUnified2D|EveryShippedKernelsGeometryIsOneTheEngineLaunches" "unified_3d|Gfx950AttentionUnified3D|EveryShippedKernelsGeometryIsOneTheEngineLaunches,EveryEntryLaunchesItsSegmentAndReduceKernels") string(REPLACE "|" ";" _gfx950_engine "${_gfx950_engine}") list(GET _gfx950_engine 0 _snake) diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified2DTestGraphs.hpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified2DTestGraphs.hpp index 01b72f23c9ac..85a59adc262a 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified2DTestGraphs.hpp +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified2DTestGraphs.hpp @@ -87,7 +87,8 @@ inline std::optional matchGraph(const GraphSpec& spec) /// The fields the engine's KMD declares, as a completed catalog record carries them. The /// defaults are the shipped baseline kernel for the default graph (bf16, D128, Hq 8 / Hkv 2, -/// top-left causal, block 16, T64, 4 warps x 16 rows, capacity 511). +/// top-left causal, block 16, T64, 4 warps x 16 rows, num_seqs 511). The engine does not read +/// num_seqs -- the batch is a kernarg -- but the KMD still declares it. struct KernelSpec { std::string dtype = "BF16"; @@ -172,7 +173,7 @@ inline double scoreOf(const GraphSpec& graphSpec, const KernelSpec& kernelSpec) return scorer(graph.context(), bound.value_or(BoundTokens{}), makeKernel(kernelSpec)); } -/// The 18-argument ABI the rocKE builder declares, under the names it gives them. +/// The 21-argument ABI the rocKE builder declares, under the names it gives them. inline std::vector pythonAbi() { constexpr uint32_t POINTER_BYTES = 8; @@ -198,7 +199,10 @@ inline std::vector pythonAbi() scalar("softcap"), scalar("num_seqs"), scalar("block_table_stride"), - scalar("qq_bias_stride_0")}; + scalar("qq_bias_stride_0"), + scalar("num_query_heads"), + scalar("num_kv_heads"), + scalar("sliding_window")}; } /// A KPACK descriptor for this engine naming an archive that is not there, recording diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified3DTestGraphs.hpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified3DTestGraphs.hpp index 47497605bf17..279a873fc2b3 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified3DTestGraphs.hpp +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/Gfx950AttentionUnified3DTestGraphs.hpp @@ -172,7 +172,7 @@ inline double scoreOf(const GraphSpec& graphSpec, const KernelSpec& kernelSpec) return scorer(graph.context(), bound.value_or(BoundTokens{}), makeKernel(kernelSpec)); } -/// The 19-argument segment-kernel ABI the rocKE builder declares, under its own names. +/// The 22-argument segment-kernel ABI the rocKE builder declares, under its own names. inline std::vector pythonSegmentAbi() { constexpr uint32_t POINTER_BYTES = 8; @@ -199,20 +199,25 @@ inline std::vector pythonSegmentAbi() scalar("softcap"), scalar("num_seqs"), scalar("block_table_stride"), - scalar("qq_bias_stride_0")}; + scalar("qq_bias_stride_0"), + scalar("num_query_heads"), + scalar("num_kv_heads"), + scalar("sliding_window")}; } -/// The 5-pointer reduce-kernel ABI. +/// The reduce-kernel ABI: 5 pointers and the query head count. inline std::vector pythonReduceAbi() { constexpr uint32_t POINTER_BYTES = 8; + constexpr uint32_t SCALAR_BYTES = 4; const auto buffer = [](const char* name) { return KernelArgument{"global_buffer", POINTER_BYTES, 0, name}; }; return {buffer("output_ptr"), buffer("segm_output_ptr"), buffer("segm_max_ptr"), buffer("segm_expsum_ptr"), - buffer("seq_lens_ptr")}; + buffer("seq_lens_ptr"), + KernelArgument{"by_value", SCALAR_BYTES, 0, "num_query_heads"}}; } /// A KPACK descriptor for this engine naming an archive that is not there, its `segment` diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DCatalogTiles.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DCatalogTiles.cpp index acdd98a544e8..6b7f2e5ed03f 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DCatalogTiles.cpp +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DCatalogTiles.cpp @@ -22,9 +22,8 @@ /** * @file TestGfx950AttentionUnified2DCatalogTiles.cpp - * @brief Census of the geometry and batch capacity the shipped - * hipkernel:Gfx950AttentionUnified2D catalog declares, judged by the checks the engine - * itself applies. + * @brief Census of the geometry the shipped hipkernel:Gfx950AttentionUnified2D catalog + * declares, judged by the checks the engine itself applies. * * The engine launches a loaded kernel only when its candidateFor() accepts the record's * geometry through isSupportedGfx950Unified2DGeometry(); a kernel it declines ships but is @@ -33,11 +32,6 @@ * unbuildable warp/row/tile combination, or a field missing from the record) fails here * instead of silently never running. * - * It also pins the batch capacity: the kernel's binary search resolves at most 2^iters - 1 - * sequences, and a catalog regenerated with a smaller `enumerate.fixed.batch` would ship - * binaries that quietly serve fewer sequences than the shipped ones. Raise the floor below - * deliberately, with the manifest. - * * TestGfx950AttentionUnified2DDecodeHandoff checks the decode boundary between the two * unified engines: Unified2D declines every decode graph whose GQA group divides * GFX950_UNIFIED_3D_BLOCK_M and leaves it to Unified3D, so every such cell the 2D catalog @@ -56,9 +50,6 @@ using namespace hip_kernel_provider::kernel_ingestor_engine; using namespace hip_kernel_provider::kernel_ingestor_engine::testing; namespace ingestor = hipdnn_plugin_sdk::ingestor; -/// The batch every shipped kernel must serve at least: the manifest's `enumerate.fixed.batch`. -constexpr int64_t MINIMUM_BATCH_CAPACITY = 511; - /// A kernel's integer metadata field, or nullopt when it is absent or another type: the cases /// candidateFor() declines. std::optional integerField(const ingestor::KernelDescriptor& kernel, const char* field) @@ -148,28 +139,6 @@ TEST(TestGfx950AttentionUnified2DCatalogTiles, EveryShippedKernelsGeometryIsOneT EXPECT_GT(checked, 0U) << "the loaded set holds no kernels to check"; } -TEST(TestGfx950AttentionUnified2DCatalogTiles, EveryShippedKernelServesTheManifestsBatchCapacity) -{ - const auto& set = loadedSet(GFX950_ATTENTION_UNIFIED_2D_ENGINE_NAME); - - std::size_t checked = 0; - for(const auto& pack : set.packs) - { - for(const auto& kernel : pack.kernels) - { - ++checked; - const auto numSeqs = integerField(kernel, "num_seqs"); - ASSERT_TRUE(numSeqs.has_value()) - << "kernel '" << kernel.name << "' does not carry an integer num_seqs"; - EXPECT_GT(*numSeqs, 0) << "kernel '" << kernel.name << "'"; - EXPECT_GE(gfx950Unified2DBatchCapacity(*numSeqs), MINIMUM_BATCH_CAPACITY) - << "kernel '" << kernel.name << "' bakes num_seqs " << *numSeqs - << ", which resolves fewer than " << MINIMUM_BATCH_CAPACITY << " sequences"; - } - } - EXPECT_GT(checked, 0U) << "the loaded set holds no kernels to check"; -} - TEST(TestGfx950AttentionUnified2DDecodeHandoff, EveryDecodeCellHandedToUnified3DHasAUnified3DKernel) { std::set unified3dCells; diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DDispatch.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DDispatch.cpp index d6f5096ced42..49b3ea1b5cda 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DDispatch.cpp +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DDispatch.cpp @@ -98,7 +98,6 @@ std::string prepareFailureFor(const KernelSpec& spec, const GraphSpec& graphSpec constexpr const char* MISSING_TOKEN_MARKER = "missing bound token"; constexpr const char* OUTPUT_LAYOUT_MARKER = "not dense BSHD"; constexpr const char* GEOMETRY_MARKER = "declares no supported geometry"; -constexpr const char* CAPACITY_MARKER = "serves at most"; constexpr const char* BLOCK_MARKER = "does not divide seqlen_kv"; } // namespace @@ -250,19 +249,16 @@ TEST(TestGfx950AttentionUnified2DDispatch, BindsExactlyTheTokensTheDispatchReads /// The positive control the refusals below depend on: a prepare() that threw for every input /// would satisfy all of them and dispatch nothing. A failure is still expected -- the archive -/// is absent -- but it must be the loader's, which means the bindings, the geometry, the -/// capacity and the identity tables were all accepted first. +/// is absent -- but it must be the loader's, which means the bindings, the geometry and the +/// identity tables were all accepted first. TEST(TestGfx950AttentionUnified2DDispatch, ReachesTheLoaderOnceEverythingBeforeItIsGood) { const std::string failure = prepareFailureFor(KernelSpec{}); ASSERT_FALSE(failure.empty()) << "the archive named by the fixture does not exist, so " "prepare() cannot have succeeded"; - for(const char* marker : {MISSING_TOKEN_MARKER, - OUTPUT_LAYOUT_MARKER, - GEOMETRY_MARKER, - CAPACITY_MARKER, - BLOCK_MARKER}) + for(const char* marker : + {MISSING_TOKEN_MARKER, OUTPUT_LAYOUT_MARKER, GEOMETRY_MARKER, BLOCK_MARKER}) { EXPECT_EQ(failure.find(marker), std::string::npos) << failure; } @@ -363,7 +359,6 @@ TEST(TestGfx950AttentionUnified2DDispatch, RefusesToPrepareACandidateWithoutASup with([](KernelSpec& s) { s.blockSize = 24; }); with([](KernelSpec& s) { s.useMfma32x32 = 1; }); // needs 32 rows per warp with([](KernelSpec& s) { s.useTransposedQk32x32 = 1; }); // needs the 32x32 path - with([](KernelSpec& s) { s.numSeqs = 0; }); with([](KernelSpec& s) { s.maskType = 3; }); with([](KernelSpec& s) { s.headSize = 96; }); with([](KernelSpec& s) { s.numQueryHeads = 9; }); @@ -390,6 +385,10 @@ TEST(TestGfx950AttentionUnified2DDispatch, RefusesToPrepareACandidateMissingAFie { continue; // read by the matcher, not by prepare() } + if(field == "num_seqs") + { + continue; // declared by the KMD, not read: the batch is a kernarg + } SCOPED_TRACE(field); auto kernel = makeKpackKernel(KernelSpec{}, pythonAbi()); kernel.metadata.erase(field); @@ -404,25 +403,21 @@ TEST(TestGfx950AttentionUnified2DDispatch, RefusesToPrepareACandidateMissingAFie EXPECT_NE(failure.find(GEOMETRY_MARKER), std::string::npos) << failure; } -/// One past the capacity the kernel was built for. The search would leave the last blocks -/// reading another sequence's lengths without a fault, so prepare() refuses by name. -TEST(TestGfx950AttentionUnified2DDispatch, RefusesToPrepareABatchAboveTheCandidatesCapacity) +/// The kernel derives its cu_q search trip count from the `num_seqs` kernarg, so a batch past +/// what the old baked 9-iteration search resolved (2^9 - 1) reaches the loader like any other. +TEST(TestGfx950AttentionUnified2DDispatch, PreparesABatchPastTheOldBakedSearch) { GraphSpec spec; - spec.batch = 4; + spec.batch = 512; spec.seqLenQ = 16; spec.seqLenKv = 16; - KernelSpec capacityThree; - capacityThree.numSeqs = 3; // 2 iterations -> 3 sequences - const std::string tooMany = prepareFailureFor(capacityThree, spec); - ASSERT_FALSE(tooMany.empty()); - EXPECT_NE(tooMany.find(CAPACITY_MARKER), std::string::npos) << tooMany; - - spec.batch = 3; - const std::string exactlyFull = prepareFailureFor(capacityThree, spec); - ASSERT_FALSE(exactlyFull.empty()); - EXPECT_EQ(exactlyFull.find(CAPACITY_MARKER), std::string::npos) << exactlyFull; + const std::string failure = prepareFailureFor(KernelSpec{}, spec); + ASSERT_FALSE(failure.empty()) << "the archive named by the fixture does not exist, so " + "prepare() cannot have succeeded"; + EXPECT_EQ(failure.find(GEOMETRY_MARKER), std::string::npos) << failure; + EXPECT_EQ(failure.find(BLOCK_MARKER), std::string::npos) << failure; + EXPECT_NE(failure.find("does not exist"), std::string::npos) << failure; } /// A block size that does not divide the graph's Skv is not a whole-block cache. diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DGeometry.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DGeometry.cpp index 0a8df84f6886..088e3efcf37c 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DGeometry.cpp +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DGeometry.cpp @@ -21,7 +21,7 @@ /** * @file TestGfx950AttentionUnified2DGeometry.cpp * @brief Pins what hipkernel:Gfx950AttentionUnified2D restates from its Python builder: the - * geometry rules, the launch grid, the batch capacity and the paged-KV metadata. + * geometry rules, the launch grid and the paged-KV metadata. * * Every expectation is a literal worked out from the Python (attention_tiled_2d.py, * `AttentionTuningSpec.launch_grid`), not computed the way the header computes it: a helper @@ -172,53 +172,6 @@ TEST(TestGfx950AttentionUnified2DGeometry, TheThirtyTwoByThirtyTwoPathsKeepTheir EXPECT_TRUE(isSupportedGfx950Unified2DGeometry(transposed)); } -// ============================================================================= -// Batch capacity -// ============================================================================= - -/// The kernel bisects [0, num_seqs] in `iters` unrolled steps, which resolves 2^iters - 1 -/// sequences. Table values are ceil(log2(num_seqs + 1)) worked out by hand. -TEST(TestGfx950AttentionUnified2DGeometry, BinarySearchItersIsCeilLogTwoOfNumSeqsPlusOne) -{ - const std::vector> table{{1, 1}, - {2, 2}, - {3, 2}, - {4, 3}, - {7, 3}, - {8, 4}, - {15, 4}, - {16, 5}, - {255, 8}, - {256, 9}, - {511, 9}, - {512, 10}, - {1023, 10}, - {1024, 11}}; - for(const auto& [numSeqs, iters] : table) - { - EXPECT_EQ(gfx950Unified2DBinarySearchIters(numSeqs), iters) << "num_seqs " << numSeqs; - } -} - -/// A spec that declares no num_seqs gets the kernel's conservative 32 iterations. -TEST(TestGfx950AttentionUnified2DGeometry, AnUndeclaredNumSeqsIsTheThirtyTwoIterationFallback) -{ - EXPECT_EQ(gfx950Unified2DBinarySearchIters(0), 32); - EXPECT_EQ(gfx950Unified2DBinarySearchIters(-7), 32); -} - -/// Capacity is 2^iters - 1, so the baked 511 serves exactly 511 and not 512. The boundary is -/// the whole point: one past it reads another sequence's seq_lens with no fault. -TEST(TestGfx950AttentionUnified2DGeometry, BatchCapacityIsTheLargestBatchTheSearchResolves) -{ - EXPECT_EQ(gfx950Unified2DBatchCapacity(511), 511); - EXPECT_EQ(gfx950Unified2DBatchCapacity(512), 1023); - EXPECT_EQ(gfx950Unified2DBatchCapacity(1), 1); - EXPECT_EQ(gfx950Unified2DBatchCapacity(2), 3); - EXPECT_EQ(gfx950Unified2DBatchCapacity(3), 3); - EXPECT_EQ(gfx950Unified2DBatchCapacity(4), 7); -} - // ============================================================================= // Launch geometry // ============================================================================= diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DMatchers.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DMatchers.cpp index 100513660ac4..08372dbb62dd 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DMatchers.cpp +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DMatchers.cpp @@ -23,7 +23,7 @@ * survivors rank. * * One case per rule, in severity order: the silent-wrong-answer rules first (layout, shape, - * mask, batch capacity), then faults (addressing limits), then declined features. A graph + * mask), then faults (addressing limits), then declined features. A graph * match that returns nullopt empties this engine's WHOLE catalog, so every case that expects * a decline is paired with a positive control that shows the same graph one field away is * served. @@ -550,11 +550,10 @@ TEST(TestGfx950AttentionUnified2DMatchers, KernelMatchRequiresTheCandidatesBlock EXPECT_FALSE(matchesKernel(sixteenOnly, block64)); } -/// The kernel bisects the sequences in a fixed number of unrolled steps, which resolves at -/// most 2^iters - 1 of them. Past that the last blocks read another sequence's lengths and the -/// launch still succeeds, so the boundary is exact: capacity 3 (2 iterations) serves a batch -/// of 3 and not 4, and the shipped 511 (9 iterations) serves 511 and not 512. -TEST(TestGfx950AttentionUnified2DMatchers, KernelMatchBoundsTheBatchByTheBakedCapacity) +/// The kernel derives its cu_q search trip count from the `num_seqs` kernarg, so no batch is +/// baked: a record declaring the shipped `num_seqs` 511 (or a smaller one, or none) serves a +/// batch past 2^9 - 1, which the old baked 9-iteration search could not resolve. +TEST(TestGfx950AttentionUnified2DMatchers, KernelMatchServesABatchPastTheOldBakedSearch) { const auto graphOfBatch = [](int64_t batch) { GraphSpec spec; @@ -563,20 +562,19 @@ TEST(TestGfx950AttentionUnified2DMatchers, KernelMatchBoundsTheBatchByTheBakedCa spec.seqLenKv = 16; return spec; }; - const auto kernelOfCapacity = [](int64_t numSeqs) { - KernelSpec spec; - spec.numSeqs = numSeqs; - return spec; - }; + KernelSpec numSeqsThree; + numSeqsThree.numSeqs = 3; + + for(const int64_t batch : {3, 4, 511, 512, 4096}) + { + EXPECT_TRUE(matchesKernel(graphOfBatch(batch), KernelSpec{})) << "batch " << batch; + EXPECT_TRUE(matchesKernel(graphOfBatch(batch), numSeqsThree)) << "batch " << batch; + } - EXPECT_TRUE(matchesKernel(graphOfBatch(3), kernelOfCapacity(3))); - EXPECT_FALSE(matchesKernel(graphOfBatch(4), kernelOfCapacity(3))); - EXPECT_TRUE(matchesKernel(graphOfBatch(511), kernelOfCapacity(511))); - EXPECT_FALSE(matchesKernel(graphOfBatch(512), kernelOfCapacity(511))); - // A capacity that is not 2^k - 1 still rounds UP to the iterations it needs: num_seqs 4 - // takes 3 iterations, so it serves 7. - EXPECT_TRUE(matchesKernel(graphOfBatch(7), kernelOfCapacity(4))); - EXPECT_FALSE(matchesKernel(graphOfBatch(8), kernelOfCapacity(4))); + auto withoutNumSeqs = makeKernel(KernelSpec{}); + withoutNumSeqs.metadata.erase(std::string("num_seqs")); + const Graph batch512(graphOfBatch(512)); + EXPECT_TRUE(matchesKernel(batch512, withoutNumSeqs)); } /// The metadata carries no shape: one binary serves any Sq and Skv for its configuration. @@ -613,8 +611,6 @@ TEST(TestGfx950AttentionUnified2DMatchers, KernelMatchDeclinesACandidateWithAnUn with([](KernelSpec& s) { s.blockSize = 24; }); with([](KernelSpec& s) { s.useMfma32x32 = 1; }); with([](KernelSpec& s) { s.useTransposedQk32x32 = 1; }); - with([](KernelSpec& s) { s.numSeqs = 0; }); - with([](KernelSpec& s) { s.numSeqs = -3; }); with([](KernelSpec& s) { s.maskType = 7; }); for(std::size_t i = 0; i < unbuildable.size(); ++i) @@ -630,6 +626,10 @@ TEST(TestGfx950AttentionUnified2DMatchers, KernelMatchDeclinesAnIncompleteOrMist const Graph graph; for(const auto& entry : metadataOf(KernelSpec{})) { + if(entry.first == "num_seqs") + { + continue; // declared by the KMD, not read: the batch is a kernarg + } SCOPED_TRACE(entry.first); auto kernel = makeKernel(KernelSpec{}); kernel.metadata.erase(entry.first); diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DSignature.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DSignature.cpp index 48051f56c8a1..aa59ee202dbc 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DSignature.cpp +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified2DSignature.cpp @@ -21,7 +21,7 @@ * A wrong argument count, order, kind or width is not a launch error. The driver copies * whatever the host hands it into the kernarg segment and the kernel reads it as the types it * was compiled for, so the failure mode is misread memory: wrong numerics with a success - * status. This kernel has eighteen arguments, ten of them pointers the engine mostly passes + * status. This kernel has twenty-one arguments, ten of them pointers the engine mostly passes * null, so a shifted slot is easy to write and invisible to a smoke test. * * requireSignatureMatch compares `kind` and `size`, and `name` only when both sides carry one. @@ -45,7 +45,7 @@ constexpr const char* BY_VALUE_KIND = "by_value"; constexpr uint32_t POINTER_BYTES = 8; constexpr uint32_t SCALAR_BYTES = 4; -constexpr std::size_t ARGUMENT_COUNT = 18; +constexpr std::size_t ARGUMENT_COUNT = 21; constexpr std::size_t FIRST_SCALAR_INDEX = 10; /// The fragment only the signature refusal carries. The rest of that message renders both @@ -111,18 +111,18 @@ TEST(TestGfx950AttentionUnified2DSignature, AcceptsTheAbiThePythonDeclares) TEST(TestGfx950AttentionUnified2DSignature, RefusesADroppedTrailingArgument) { - // qq_bias_stride_0 gone: 17 arguments against 18, so the kernel would read one slot past + // sliding_window gone: 20 arguments against 21, so the kernel would read one slot past // what the host wrote. auto signature = pythonAbi(); signature.pop_back(); - expectRefused(signature, "a seventeen-argument list is not this ABI"); + expectRefused(signature, "a twenty-argument list is not this ABI"); } TEST(TestGfx950AttentionUnified2DSignature, RefusesAnExtraTrailingArgument) { auto signature = pythonAbi(); signature.push_back(KernelArgument{BY_VALUE_KIND, SCALAR_BYTES, 0, ""}); - expectRefused(signature, "a nineteen-argument list is not this ABI"); + expectRefused(signature, "a twenty-two-argument list is not this ABI"); } /// The form without the two trailing strides, which `_attn_signature` emits when its @@ -130,8 +130,10 @@ TEST(TestGfx950AttentionUnified2DSignature, RefusesAnExtraTrailingArgument) /// so a build without them is one it must not launch. TEST(TestGfx950AttentionUnified2DSignature, RefusesTheFormWithoutTheTrailingStrides) { + // block_table_stride and qq_bias_stride_0 sit between num_seqs and the runtime shape + // kernargs. auto signature = pythonAbi(); - signature.resize(ARGUMENT_COUNT - 2); + signature.erase(signature.begin() + 16, signature.begin() + 18); expectRefused(signature, "the stride-less form is a different ABI"); } @@ -155,11 +157,11 @@ TEST(TestGfx950AttentionUnified2DSignature, RefusesAPointerInAScalarSlot) TEST(TestGfx950AttentionUnified2DSignature, RefusesBuffersAndScalarsTransposed) { - // The eight scalars ahead of the ten pointers: the same eighteen entries and the same + // The eleven scalars ahead of the ten pointers: the same twenty-one entries and the same // counts of each kind, and the comparison is positional. const KernelArgument buffer{BUFFER_KIND, POINTER_BYTES, 0, ""}; const KernelArgument scalar{BY_VALUE_KIND, SCALAR_BYTES, 0, ""}; - std::vector transposed(8, scalar); + std::vector transposed(11, scalar); transposed.insert(transposed.end(), 10, buffer); expectRefused(transposed, "the ten pointers come first"); } @@ -220,10 +222,10 @@ TEST(TestGfx950AttentionUnified2DSignature, RefusesEachExchangeOfTwoLikeSlots) ++exchanged; } } - // Ten pointers and eight scalars, every one of each group the same kind and size: - // C(10, 2) + C(8, 2) pairs. A signature that grew or lost a slot changes this count. + // Ten pointers and eleven scalars, every one of each group the same kind and size: + // C(10, 2) + C(11, 2) pairs. A signature that grew or lost a slot changes this count. constexpr std::size_t POINTER_PAIRS = 45; - constexpr std::size_t SCALAR_PAIRS = 28; + constexpr std::size_t SCALAR_PAIRS = 55; EXPECT_EQ(exchanged, POINTER_PAIRS + SCALAR_PAIRS); } diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified3DCatalogTiles.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified3DCatalogTiles.cpp index c5b2c41ab46d..98193ff466d4 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified3DCatalogTiles.cpp +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified3DCatalogTiles.cpp @@ -40,8 +40,8 @@ using namespace hip_kernel_provider::kernel_ingestor_engine; using namespace hip_kernel_provider::kernel_ingestor_engine::testing; namespace ingestor = hipdnn_plugin_sdk::ingestor; -constexpr std::size_t SEGMENT_ARGUMENTS = 19; -constexpr std::size_t REDUCE_ARGUMENTS = 5; +constexpr std::size_t SEGMENT_ARGUMENTS = 22; +constexpr std::size_t REDUCE_ARGUMENTS = 6; std::optional integerField(const ingestor::KernelDescriptor& kernel, const char* field) { @@ -124,7 +124,7 @@ TEST(TestGfx950AttentionUnified3DCatalogTiles, EveryEntryLaunchesItsSegmentAndRe ASSERT_EQ(reduce.signature.size(), REDUCE_ARGUMENTS); EXPECT_EQ(segment.signature.front().name, "segm_output_ptr"); EXPECT_EQ(reduce.signature.front().name, "output_ptr"); - EXPECT_EQ(reduce.signature.back().name, "seq_lens_ptr"); + EXPECT_EQ(reduce.signature.back().name, "num_query_heads"); } } EXPECT_GT(checked, 0U) << "the loaded set holds no kernels to check"; diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified3DSignature.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified3DSignature.cpp index 6d20596934a0..63ba1b67a4c6 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified3DSignature.cpp +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestGfx950AttentionUnified3DSignature.cpp @@ -34,8 +34,8 @@ namespace using namespace unified3d; -constexpr std::size_t SEGMENT_ARGUMENTS = 19; -constexpr std::size_t REDUCE_ARGUMENTS = 5; +constexpr std::size_t SEGMENT_ARGUMENTS = 22; +constexpr std::size_t REDUCE_ARGUMENTS = 6; constexpr const char* MISMATCH_MARKER = "is packaged with arguments"; std::string prepareFailureFor(const std::vector& segment, @@ -86,14 +86,14 @@ TEST(TestGfx950AttentionUnified3DSignature, RefusesASegmentListOfTheWrongLength) { auto shorter = pythonSegmentAbi(); shorter.pop_back(); - expectRefused(shorter, pythonReduceAbi(), "eighteen arguments is not the segment ABI"); + expectRefused(shorter, pythonReduceAbi(), "twenty-one arguments is not the segment ABI"); auto longer = pythonSegmentAbi(); longer.push_back(KernelArgument{"by_value", 4, 0, ""}); - expectRefused(longer, pythonReduceAbi(), "twenty arguments is not the segment ABI"); + expectRefused(longer, pythonReduceAbi(), "twenty-three arguments is not the segment ABI"); } -/// Unified2D's 18-argument list leads with output_ptr; the segment kernel leads with the +/// Unified2D's 21-argument list leads with output_ptr; the segment kernel leads with the /// three scratch pointers and has no out_scale. A descriptor naming a 2D kernel must not be /// launched as a segment kernel. TEST(TestGfx950AttentionUnified3DSignature, RefusesTheUnified2DListAsASegmentKernel) @@ -118,7 +118,10 @@ TEST(TestGfx950AttentionUnified3DSignature, RefusesTheUnified2DListAsASegmentKer scalar("softcap"), scalar("num_seqs"), scalar("block_table_stride"), - scalar("qq_bias_stride_0")}; + scalar("qq_bias_stride_0"), + scalar("num_query_heads"), + scalar("num_kv_heads"), + scalar("sliding_window")}; expectRefused(unified2d, pythonReduceAbi(), "the 2D kernel is not a segment kernel"); } @@ -147,7 +150,7 @@ TEST(TestGfx950AttentionUnified3DSignature, RefusesAReduceListOfTheWrongLength) { auto shorter = pythonReduceAbi(); shorter.pop_back(); - expectRefused(pythonSegmentAbi(), shorter, "four arguments is not the reduce ABI"); + expectRefused(pythonSegmentAbi(), shorter, "five arguments is not the reduce ABI"); } TEST(TestGfx950AttentionUnified3DSignature, RefusesTheReduceOutputMovedOffItsSlot) diff --git a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestUnifiedAttentionGeometry.cpp b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestUnifiedAttentionGeometry.cpp index d1f0f22984a1..be371db8969a 100644 --- a/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestUnifiedAttentionGeometry.cpp +++ b/dnn-providers/hip-kernel-provider/src/tests/engines/kernel_ingestor_engine/packs/TestUnifiedAttentionGeometry.cpp @@ -77,7 +77,6 @@ TEST(TestUnifiedAttentionGeometry, Gfx950WrappersAreTheNeutralFunctions) EXPECT_EQ(gfx950Unified2DLaunchGeometry(2, 32, 16, 4, 300, 3, "k"), unifiedAttention2DLaunchGeometry( GFX950_UNIFIED_ATTENTION_TRAITS, 2, 32, 16, 4, 300, 3, "k")); - EXPECT_EQ(gfx950Unified2DBatchCapacity(511), unifiedAttentionBatchCapacity(511)); EXPECT_EQ(gfx950Unified2DWorkspaceLayout(3, 512, 16), unifiedAttentionWorkspaceLayout(3, 512, 16)); EXPECT_EQ(gfx950Unified2DWorkspaceImage(3, 7, 512, 16),