From a2287d2153b21f982c8f9f5437b642c5cc075f69 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 27 Jul 2026 23:30:22 +0000 Subject: [PATCH 1/8] fix(tensilelite): count-preserving per-XCD StreamK work-queue counter self-reset (WGMXCC + SKXCC) The per-XCD StreamK work-queue atomic counter is designed to self-reset to zero every launch purely by atomic wraparound: each queue q must receive exactly tiles_q + W_q increments, where W_q = distribute(skGrid, q) assumes the queue index densely covers [0, skGrid). Deriving the queue index from the remapped StreamKIdx breaks that invariant, because neither the wgmXCC CU-count remap (WorkGroupMappingXCC == -1 auto path) nor the StreamKXCCMapping chiplet remap (SKXCC with WorkGroupMappingXCC > 1) is a % numQueues-count-preserving permutation when the grid does not block evenly. The skewed per-queue home-workgroup count leaves the counter drifting off zero across launches. Fix: derive the queue index from the raw pre-remap launch workgroup rank, which is a dense bijection onto [0, skGrid) (== physical XCD rank) and restores the counting invariant on both the SK4 and SK5 dynamic paths. The raw rank is snapshotted once in the prologue, before wgmXCC and the SKXCC preLoop XCCMapping remap rewrite WorkGroup0, into the persistent StreamKTileIdx SGPR -- an already-allocated slot that is provably dead in the [prologue, queue-read] window -- and read back masked % numQueues at the queue index computation. This adds zero additional persistent SGPRs, so it also fits tuned high-register SKXCC kernels that would overflow the SGPR file with a dedicated queue SGPR. Only once-per-workgroup setup instructions are added; no steady-state cost. Count-preserving regimes (fixed non-SKXCC WGMXCC == 1, WorkGroupIdFromTTM targets, single-queue arches) keep the cheap StreamKIdx % numQueues path. --- .../tensilelite/Tensile/Components/StreamK.py | 84 +++- .../tensilelite/Tensile/KernelWriter.py | 58 +++ .../Tensile/KernelWriterAssembly.py | 24 + .../Tests/unit/test_streamk_queue_index.py | 452 ++++++++++++++++++ 4 files changed, 609 insertions(+), 9 deletions(-) create mode 100644 projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_queue_index.py diff --git a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py index f4cff0d525e0..52e701f30f5d 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py +++ b/projects/hipblaslt/tensilelite/Tensile/Components/StreamK.py @@ -424,6 +424,79 @@ def _wsFlagsBaseOffset(self, writer, kernel): numQueues, _, _, cacheLineLog2 = self._wsQueueConstants(writer, kernel) return numQueues << cacheLineLog2 + @staticmethod + def usesRawQueueRank(writer, kernel): + """True when the per-XCD queue index is taken from the raw pre-remap + launch rank snapshotted into the reused, in-window-dead persistent + ``StreamKTileIdx`` carrier (zero extra SGPR -- see the prologue snapshot + in KernelWriterAssembly and KernelWriter.skUsesRawQueueRank). + + The auto-reset wrap bound (tiles_q + W_q [+ W_p]) assumes each queue's + home-workgroup count equals ``distribute(skGrid, q)`` -- i.e. that the + set of workgroup ids mapped to queue q is ``{i in [0,skGrid) : i %% + numQueues == q}``. That holds only if the value feeding ``% numQueues`` + densely covers ``[0, skGrid)``. ``StreamKIdx`` is the *remapped* id + (wgmXCC CU-count remap and/or the StreamKXCCMapping chiplet remap), and + neither remap is a ``% numQueues``-count-preserving permutation when the + grid does not block evenly, so ``StreamKIdx %% numQueues`` skews the + per-queue count away from W_q and the counter no longer wraps back to 0 + each launch. Using the raw launch rank (a dense bijection onto + ``[0, skGrid)`` == physical XCD rank) restores the invariant. + + Two disjoint remap regimes need the raw rank: + * WorkGroupMappingXCC == -1 (dynamic auto-WGM) -- host picks + WGMXCC = NUM_XCD > 1 and the wgmXCC remap skews the count. + * StreamKXCCMapping != 0 with WorkGroupMappingXCC > 1 (SKXCC) -- the + SKXCC chiplet remap (plus fixed WGMXCC > 1) skews the count. SKXCC + with WGMXCC == 1 is already count-preserving and stays on the cheap + ``StreamKIdx %% numQueues`` else-branch; WGMXCC == -1 is mutually + exclusive with SKXCC so the disjuncts never overlap. + + Fixed non-SKXCC WGMXCC == 1 needs no fix (StreamKIdx is already the raw + rank). On WorkGroupIdFromTTM targets (gfx12) StreamKIdx is re-read from + the raw hardware id (ttmp9); single-queue arches (NumXCD <= 1) are + trivially balanced. Kept in sync with KernelWriter.skUsesRawQueueRank.""" + return (writer.states.archCaps["NumXCD"] > 1 + and not writer.states.archCaps["WorkGroupIdFromTTM"] + and (kernel["WorkGroupMappingXCC"] == -1 + or (kernel["StreamKXCCMapping"] != 0 + and kernel["WorkGroupMappingXCC"] > 1))) + + def _emitQueueIndex(self, writer, kernel, sQueueIdx, wsLog2Queues) -> Module: + """Compute the per-XCD work-queue index into ``sQueueIdx``. + + Zero-overhead accounting fix: the queue must come from the raw + round-robin launch rank so its ``% numQueues`` count equals the + ``distribute(skGrid, q)`` the auto-reset bound assumes (see + ``usesRawQueueRank``). On gfx9 that raw rank is snapshotted once, before + wgmXCC / the SKXCC XCCMapping remap rewrites WorkGroup0, into the reused, + in-window-dead persistent ``StreamKTileIdx`` carrier (KernelWriterAssembly + prologue -- zero extra SGPR); here it is read back and reduced + ``% numQueues``. Otherwise (WGMXCC no-op, or gfx12) ``StreamKIdx`` + already holds the raw id, so fall back to ``StreamKIdx %% numQueues``. + """ + module = Module("StreamK queue index") + if self.usesRawQueueRank(writer, kernel): + # The queue index is the RAW pre-wgmXCC launch WG rank modulo + # numQueues. This raw rank densely covers [0, skGrid), so the number + # of home workgroups mapped to queue q equals distribute(skGrid, q) = + # W_q -- exactly the count the auto-reset wrap bound (tiles_q + W_q) + # assumes -- and the atomic counter self-resets to 0 every launch. + # (StreamKIdx is the wgmXCC CU-count-remapped id, whose % numQueues is + # NOT count-preserving and skews the per-queue count.) Uniform for SK4 + # and SK5 -- the snapshot lives in the reused, in-window-dead + # persistent StreamKTileIdx carrier (zero extra SGPR; see + # KernelWriterAssembly prologue and usesRawQueueRank). + _, numQueuesMask, _, _ = self._wsQueueConstants(writer, kernel) + module.add(SAndB32(dst=sgpr(sQueueIdx), src0=sgpr("StreamKTileIdx"), src1=hex(numQueuesMask), + comment="queue = rawWG %% numQueues (dense round-robin => home-WG count == distribute(skGrid,q))")) + else: + module.add(SLShiftRightB32(dst=sgpr(sQueueIdx), src=sgpr("StreamKIdx"), shiftHex=wsLog2Queues)) + module.add(SLShiftLeftB32(dst=sgpr(sQueueIdx), src=sgpr(sQueueIdx), shiftHex=wsLog2Queues)) + module.add(SSubU32(dst=sgpr(sQueueIdx), src0=sgpr("StreamKIdx"), src1=sgpr(sQueueIdx), + comment="Default queue index")) + return module + def _wsStructuralCount(self, mod, mask, log2Queues, sDst, sTotal, sQueue, sTmp, comment): """Emit sDst = (sTotal >> log2Queues) + [sQueue < (sTotal & mask)]. @@ -3100,9 +3173,7 @@ def graWorkGroup(self, writer, kernel, tPA, tPB): # Default queue index sQueueIdx = writer.sgprPool.checkOut(1, "QueueIdx") - module.add(SLShiftRightB32(dst=sgpr(sQueueIdx), src=sgpr("StreamKIdx"), shiftHex=wsLog2Queues)) - module.add(SLShiftLeftB32(dst=sgpr(sQueueIdx), src=sgpr(sQueueIdx), shiftHex=wsLog2Queues)) - module.add(SSubU32(dst=sgpr(sQueueIdx), src0=sgpr("StreamKIdx"), src1=sgpr(sQueueIdx), comment="Default queue index")) + module.add(self._emitQueueIndex(writer, kernel, sQueueIdx, wsLog2Queues)) # Queue address sAddress = writer.sgprPool.checkOutAligned(2, 2, "Address") @@ -3755,12 +3826,7 @@ def emitDynamicGRA(mod): # Default queue index sQueueIdx = writer.sgprPool.checkOut(1, "QueueIdx") - mod.add(SLShiftRightB32(dst=sgpr(sQueueIdx), src=sgpr("StreamKIdx"), - shiftHex=wsLog2Queues)) - mod.add(SLShiftLeftB32(dst=sgpr(sQueueIdx), src=sgpr(sQueueIdx), - shiftHex=wsLog2Queues)) - mod.add(SSubU32(dst=sgpr(sQueueIdx), src0=sgpr("StreamKIdx"), - src1=sgpr(sQueueIdx), comment="Default queue index")) + mod.add(self._emitQueueIndex(writer, kernel, sQueueIdx, wsLog2Queues)) # Queue address sAddress = writer.sgprPool.checkOutAligned(2, 2, "Address") diff --git a/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py b/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py index 589bed8e84aa..df2f698e42ae 100644 --- a/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py +++ b/projects/hipblaslt/tensilelite/Tensile/KernelWriter.py @@ -5321,6 +5321,50 @@ def releaseStreamKConstSgpr(self, nameOrIdx): if isinstance(nameOrIdx, int): self.sgprPool.checkIn(nameOrIdx) + def skUsesRawQueueRank(self, kernel): + """True when the per-XCD work-queue index must come from the RAW pre-remap + launch WG rank, captured once into the reused persistent ``StreamKTileIdx`` + carrier (KernelWriterAssembly prologue) and read back in + StreamK.graWorkGroup. + + The auto-reset wrap bound (tiles_q + W_q) assumes each queue's home-workgroup + count equals distribute(skGrid, q), i.e. that the value feeding % numQueues + densely covers [0, skGrid). Once WorkGroup0 has been remapped, its + % numQueues is NOT count-preserving when the grid does not block evenly, so + it skews the per-queue count and the counter drifts off 0. We therefore + snapshot the raw launch id (== physical XCD rank) BEFORE any remap. + + ZERO additional persistent SGPRs are spent: the raw rank is stashed in the + already-allocated ``StreamKTileIdx`` slot, which is provably dead in the + [prologue, queue-read) window (see StreamK.usesRawQueueRank). A dedicated + ``StreamKQueue`` SGPR was rejected because these SK4/SK5 kernels sit at the + register ceiling and it overflowed the SGPR file on the tuned high-register + SKXCC kernels (the unaligned-pool parity shift can cost up to 4 SGPRs). + + Two disjoint remap regimes need the raw rank: + + * WorkGroupMappingXCC == -1 -- the dynamic auto-WGM path, where the host + picks WGMXCC = NUM_XCD > 1 at runtime and the wgmXCC CU-count remap skews + the per-queue count. + * StreamKXCCMapping != 0 with WorkGroupMappingXCC > 1 -- the SKXCC path, + where the StreamKXCCMapping chiplet remap (plus the fixed WGMXCC > 1 + remap) rewrites WorkGroup0 non-count-preservingly. (SKXCC with WGMXCC == + 1 is already count-preserving -- it stays on the cheap StreamKIdx % + numQueues else-branch -- and WGMXCC == -1 is mutually exclusive with + SKXCC, so the two disjuncts never overlap.) + + Fixed non-SKXCC WGMXCC solutions (== 1 no-op, or a tuned power-of-two > 1 + without SKXCC) are excluded: == 1 needs no fix (StreamKIdx already equals the + raw rank). Single-XCD arches are trivially balanced; gfx12 + (WorkGroupIdFromTTM) re-reads the raw id from ttmp9. Kept in sync with + StreamK.usesRawQueueRank.""" + return (kernel["StreamK"] in (4, 5) + and self.states.archCaps["NumXCD"] > 1 + and not self.states.archCaps["WorkGroupIdFromTTM"] + and (kernel["WorkGroupMappingXCC"] == -1 + or (kernel["StreamKXCCMapping"] != 0 + and kernel["WorkGroupMappingXCC"] > 1))) + ############################################################################## # Move StreamK Constants to VGPRs ############################################################################## @@ -9481,6 +9525,14 @@ def vgprAllocationImplSubtile(): requiredUnalignedSgprVar = [] requiredAligned4SgprVar = [] + # Per-XCD work-queue index comes from the RAW pre-remap launch WG rank + # (see skUsesRawQueueRank). It is snapshotted once, before wgmXCC / the + # SKXCC XCCMapping rewrite WorkGroup0, into the ALREADY-allocated persistent + # StreamKTileIdx SGPR (provably dead in the [prologue, queue-read) window) + # and read back in StreamK.graWorkGroup. Reusing that slot as the raw-rank + # carrier costs ZERO additional persistent SGPRs -- a dedicated StreamKQueue + # SGPR overflowed the SGPR file on tuned high-register SKXCC kernels -- so no + # dedicated queue SGPR is declared here. if kernel["StreamK"] == 4: requiredUnalignedSgprVar += [ "StreamKIdx", @@ -9515,6 +9567,12 @@ def vgprAllocationImplSubtile(): "StreamKLocalEnd", "StreamKHybridMode", ] + # SK5 keeps StreamKHybridMode holding ONLY the mode bit (its SCmpEQU32==0 + # dispatch must stay a plain compare). The per-XCD queue index reuses the + # already-allocated persistent StreamKTileIdx slot as the raw-rank carrier + # (dead in the [prologue, queue-read) window: for SK5 it aliases + # StreamKIter, whose only in-window writes live on the mutually-exclusive + # SK3-static path), so no dedicated queue SGPR is declared. # Work stealing: per-WG sticky-empty flag (see SK4 note above). Only the # SK4 sub-path uses it, but it must persist for the whole persistent loop. if kernel["StreamKWorkStealing"]: diff --git a/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py b/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py index 4034cb1df50a..e6b10365783a 100644 --- a/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py +++ b/projects/hipblaslt/tensilelite/Tensile/KernelWriterAssembly.py @@ -2794,6 +2794,30 @@ def calculateWG(): module.add(moduleArgs) module.add(moduleRegInit) + # Zero-overhead StreamK work-queue accounting: snapshot the RAW launch WG + # rank BEFORE wgmXCC (and, for SKXCC, before the preLoop XCCMapping remap) + # rewrites WorkGroup0. The per-XCD queue counter self-resets purely by + # atomic wrap only if every queue receives exactly tiles_q + W_q + # increments per launch, where W_q = distribute(skGrid, q) assumes the + # queue index densely covers [0, skGrid). Both the wgmXCC CU-count remap + # (WGMXCC == -1 auto path) and the StreamKXCCMapping chiplet remap + # (SKXCC + WGMXCC > 1) rewrite WorkGroup0 with a permutation that is NOT + # % numQueues-count-preserving when the grid does not block evenly, so the + # remapped StreamKIdx % numQueues skews the per-queue home-workgroup count + # and the counter drifts off 0. We snapshot the raw id (== physical XCD + # rank) into the persistent StreamKTileIdx SGPR -- an ALREADY-allocated + # slot that is provably dead in the [prologue, queue-read) window on both + # the SK4 and SK5 dynamic paths (its first real write is after the queue + # read in graWorkGroup; for SK5 it aliases StreamKIter, whose only + # in-window writes live on the mutually-exclusive SK3-static path). Reusing + # it costs ZERO additional persistent SGPRs (unlike a dedicated StreamKQueue + # SGPR, which overflows the SGPR file on tuned high-register SKXCC kernels). + # The queue index reads it, masked % numQueues, in StreamK.graWorkGroup. + # Once-per-workgroup setup only -- no steady-state instructions added. + if self.skUsesRawQueueRank(kernel): + module.add(SMovB32(dst=sgpr("StreamKTileIdx"), src=sgpr("WorkGroup0"), + comment="StreamK: snapshot raw pre-remap launch WG id -> dead-in-window StreamKTileIdx carrier (queue = rawWG %% numQueues)")) + # Reorder WGIDs module.add(wgmXCC(self, kernel, tmpSgprNumWorkGroups)) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_queue_index.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_queue_index.py new file mode 100644 index 000000000000..c501617d5c6b --- /dev/null +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_queue_index.py @@ -0,0 +1,452 @@ +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +"""Unit tests for the count-preserving per-XCD StreamK work-queue index. + +These tests pin the per-XCD work-queue self-reset codegen fix: the SK4 +(``StreamKDynamic``) and SK5 (``StreamKHybrid``) dynamic work-queue per-XCD +atomic counters self-reset to 0 each launch only if every queue receives exactly +``tiles_q + W_q`` increments, which requires the value feeding ``% numQueues`` to +densely cover ``[0, skGrid)``. The queue index therefore must come from the RAW +pre-remap launch WG rank (== physical XCD rank), NOT from the remapped +``StreamKIdx`` (whose ``% numQueues`` is not count-preserving when the grid does +not block evenly). + +Zero-SGPR carrier reuse: the raw rank is snapshotted into the ALREADY-allocated +persistent ``StreamKTileIdx`` slot -- provably dead in the [prologue, queue-read) +window -- instead of a dedicated ``StreamKQueue`` SGPR, which overflowed the SGPR +file on tuned high-register SKXCC kernels. The fix now covers TWO disjoint remap +regimes: + * WorkGroupMappingXCC == -1 (dynamic auto-WGM), and + * StreamKXCCMapping != 0 with WorkGroupMappingXCC > 1 (SKXCC). + +Like the sibling StreamK codegen tests, they import rocisa instructions and +inspect emitted modules rather than matching source text, and reason about the +*real* KernelWriter / KernelWriterAssembly source via the AST so the ordering and +gating assertions track the actual code. + +Invariants pinned (each fails against the pre-fix code -- see per-test notes): + * Both dynamic auto-WGM and SKXCC (WGMXCC>1) queue indices are + ``StreamKTileIdx & (numQueues-1)`` -- a single mask of the raw-rank carrier, + never the post-remap StreamKIdx shifts. + * Both SK4 and SK5 route their queue index through the shared ``_emitQueueIndex``. + * The raw-rank snapshot ``s_mov_b32 StreamKTileIdx, WorkGroup0`` is emitted + BEFORE the wgmXCC workgroup remap. + * The reused carrier costs ZERO additional persistent SGPRs: ``StreamKQueue`` + is never declared, and the snapshot targets the already-allocated + ``StreamKTileIdx`` slot. +""" + +import ast +import inspect +import textwrap +import types + +import pytest + +# Prime the component registry before StreamK imports (avoids circular import). +from Tensile.KernelWriterAssembly import KernelWriterAssembly # noqa: F401 + +from rocisa.code import Module +from rocisa.instruction import ( + SAndB32, + SLShiftLeftB32, + SLShiftRightB32, + SMovB32, + SSubU32, +) + +from Tensile.KernelWriter import KernelWriter +from Tensile.Components.StreamK import ( + StreamK, + StreamKDynamic, + StreamKHybrid, + streamKVariantClass, +) + + +pytestmark = pytest.mark.unit + + +# --------------------------------------------------------------------------- +# Fakes: just enough of a "writer" for the standalone helper methods. +# +# _emitQueueIndex / usesRawQueueRank only touch writer.sgprPool (checkOut) and +# writer.states.archCaps, so a tiny monotonic pool plus a SimpleNamespace of the +# per-arch caps is all that is required -- no KernelWriter, no GPU. +# --------------------------------------------------------------------------- +class _FakeSgprPool: + def __init__(self, start: int = 100): + self._next = start + + def checkOut(self, n: int, name: str = "", *args, **kwargs) -> int: + reg = self._next + self._next += n + return reg + + def checkIn(self, *args, **kwargs): + return None + + +class _FakeWriter: + """Minimal writer: gfx942/gfx950-like caps, tunable for scoping tests.""" + + def __init__(self, numXCD: int = 8, cacheLineBytes: int = 128, + workGroupIdFromTTM: bool = False): + self.sgprPool = _FakeSgprPool() + self.states = types.SimpleNamespace( + archCaps={ + "NumXCD": numXCD, + "CacheLineBytes": cacheLineBytes, + "WorkGroupIdFromTTM": workGroupIdFromTTM, + }) + + +# gfx942/gfx950 both map to 8 queues (power of two) in the per-arch lookup. +def _kernel(streamk: int = 4, wgmXCC: int = -1, skxcc: int = 0, + isa=(9, 4, 0)) -> dict: + return {"ISA": isa, "StreamK": streamk, "WorkGroupMappingXCC": wgmXCC, + "StreamKXCCMapping": skxcc} + + +def _flat(module: Module) -> list: + return list(module.flatitems()) + + +def _param_texts(inst) -> list: + return [str(p) for p in inst.getParams()] + + +def _refs_sgpr(inst, name: str) -> bool: + """True if *inst* references the named SGPR (e.g. ``s[sgprStreamKTileIdx]``).""" + token = "sgpr" + name + return any(token in p for p in _param_texts(inst)) + + +def _imm_in(inst, value: int) -> bool: + """True if *inst* carries *value* as an immediate operand (dec or hex).""" + for p in inst.getParams(): + try: + if int(str(p), 0) == value: + return True + except (TypeError, ValueError): + continue + return False + + +def _emit_queue_index(streamk: int, wgmXCC: int, skxcc: int = 0, + workGroupIdFromTTM: bool = False, numXCD: int = 8) -> list: + """Render the shared ``_emitQueueIndex`` for a StreamK variant.""" + inst = streamKVariantClass(streamk)() + writer = _FakeWriter(numXCD=numXCD, workGroupIdFromTTM=workGroupIdFromTTM) + kernel = _kernel(streamk=streamk, wgmXCC=wgmXCC, skxcc=skxcc) + sQueueIdx = writer.sgprPool.checkOut(1, "QueueIdx") + wsLog2Queues = 3 # log2(8) + module = inst._emitQueueIndex(writer, kernel, sQueueIdx, wsLog2Queues) + return _flat(module) + + +# The carrier slot that holds the snapshotted raw rank (reused, in-window-dead). +_CARRIER = "StreamKTileIdx" + + +# =========================================================================== +# 1. Raw-rank paths: the queue index is the reused carrier (StreamKTileIdx) +# masked % numQueues -- NOT the post-remap StreamKIdx shift derivation. +# Applies uniformly to SK4 (StreamKDynamic) and SK5 (StreamKHybrid), and to +# BOTH raw-rank regimes: dynamic auto-WGM (WGMXCC == -1) and SKXCC (WGMXCC>1). +# =========================================================================== +class TestRawRankQueueIndex: + # (wgmXCC, skxcc) regimes that must derive the queue index from the raw rank. + RAW_REGIMES = [ + pytest.param(-1, 0, id="auto-wgm"), + pytest.param(2, 4, id="skxcc-wgmxcc2"), + pytest.param(8, 8, id="skxcc-wgmxcc8"), + ] + + @pytest.mark.parametrize("streamk", [4, 5]) + @pytest.mark.parametrize("wgmXCC,skxcc", RAW_REGIMES) + def test_queue_index_masks_raw_rank_carrier(self, streamk, wgmXCC, skxcc): + # Pre-fix emitted shr/shl/sub of StreamKIdx and NO SAndB32 -- so both the + # "single SAndB32" and "references carrier" assertions fail on it. The + # SKXCC regimes additionally fail against #10008 (which scoped the fix to + # WGMXCC == -1 only), which is exactly what this extension covers. + items = _emit_queue_index(streamk, wgmXCC=wgmXCC, skxcc=skxcc) + ands = [i for i in items if isinstance(i, SAndB32)] + assert len(ands) == 1, "raw-rank queue index must be a single mask op" + assert _refs_sgpr(ands[0], _CARRIER), ( + "the queue index must be derived from the raw-rank %s carrier" % _CARRIER + ) + # queue = rawWG & (numQueues-1); gfx942/gfx950 => 8 queues => mask 0x7. + assert _imm_in(ands[0], 0x7), "expected the (numQueues-1) = 0x7 mask" + + @pytest.mark.parametrize("streamk", [4, 5]) + @pytest.mark.parametrize("wgmXCC,skxcc", RAW_REGIMES) + def test_queue_index_does_not_use_post_remap_streamkidx(self, streamk, wgmXCC, skxcc): + # The whole point of the fix: on a raw-rank path the queue index must NOT + # come from the remapped StreamKIdx (StreamKIdx % numQueues). Pre-fix code + # emitted exactly the shift/shift/sub of StreamKIdx, so this fails on it. + items = _emit_queue_index(streamk, wgmXCC=wgmXCC, skxcc=skxcc) + assert not any(_refs_sgpr(i, "StreamKIdx") for i in items), ( + "the raw-rank queue index must not reference the post-remap StreamKIdx" + ) + assert not any( + isinstance(i, (SLShiftRightB32, SLShiftLeftB32, SSubU32)) for i in items + ), "the raw-rank path must not emit the StreamKIdx shift/shift/sub derivation" + + @pytest.mark.parametrize("streamk", [4, 5]) + @pytest.mark.parametrize("wgmXCC,skxcc", RAW_REGIMES) + def test_queue_index_never_uses_dead_streamkqueue(self, streamk, wgmXCC, skxcc): + # StreamKQueue is gone: the raw rank lives in the reused StreamKTileIdx. + items = _emit_queue_index(streamk, wgmXCC=wgmXCC, skxcc=skxcc) + assert not any(_refs_sgpr(i, "StreamKQueue") for i in items), ( + "StreamKQueue was removed; the carrier is the reused StreamKTileIdx" + ) + + +# =========================================================================== +# 2. Count-preserving else-branch: the fix is deliberately NOT applied when the +# remap is already count-preserving. The queue index stays the StreamKIdx +# shift derivation and never reads the carrier as a raw rank. Covers: +# * fixed non-SKXCC WGMXCC (== 1, or a tuned > 1 without SKXCC), +# * SKXCC with WGMXCC == 1 (already count-preserving), +# * gfx12 WorkGroupIdFromTTM (StreamKIdx already the raw id). +# =========================================================================== +class TestCountPreservingFallback: + @pytest.mark.parametrize("streamk", [4, 5]) + @pytest.mark.parametrize( + "wgmXCC,skxcc", + [ + pytest.param(1, 0, id="wgmxcc1"), + pytest.param(2, 0, id="fixed-wgmxcc2-no-skxcc"), + pytest.param(1, 4, id="skxcc-wgmxcc1"), + ], + ) + def test_count_preserving_uses_streamkidx(self, streamk, wgmXCC, skxcc): + items = _emit_queue_index(streamk, wgmXCC=wgmXCC, skxcc=skxcc) + assert any(_refs_sgpr(i, "StreamKIdx") for i in items), ( + "count-preserving paths keep the StreamKIdx-derived queue index" + ) + assert any(isinstance(i, SSubU32) for i in items), ( + "count-preserving paths keep the shift/shift/sub derivation" + ) + assert not any(isinstance(i, SAndB32) for i in items), ( + "count-preserving paths must not mask a raw-rank carrier" + ) + + @pytest.mark.parametrize("streamk", [4, 5]) + def test_gfx12_workgroupidfromttm_uses_streamkidx(self, streamk): + # On WorkGroupIdFromTTM targets StreamKIdx is already the raw id, so the + # raw-rank snapshot is unnecessary and the carrier is not read as a rank. + items = _emit_queue_index(streamk, wgmXCC=-1, workGroupIdFromTTM=True) + assert any(_refs_sgpr(i, "StreamKIdx") for i in items) + assert not any(isinstance(i, SAndB32) for i in items) + + +# =========================================================================== +# 3. Scoping predicate: usesRawQueueRank / skUsesRawQueueRank are True only on +# the raw-rank regimes (NumXCD > 1, not WorkGroupIdFromTTM, StreamK in (4,5)) +# for either WGMXCC == -1 or (StreamKXCCMapping != 0 and WGMXCC > 1). +# =========================================================================== +class _FakeKW: + """Minimal stand-in for the KernelWriter self used by skUsesRawQueueRank.""" + + def __init__(self, numXCD=8, workGroupIdFromTTM=False): + self.states = types.SimpleNamespace( + archCaps={"NumXCD": numXCD, "WorkGroupIdFromTTM": workGroupIdFromTTM}) + + +class TestUsesRawQueueRankScoping: + def test_true_for_dynamic_auto_wgm(self): + assert StreamK.usesRawQueueRank(_FakeWriter(), _kernel(wgmXCC=-1)) is True + + @pytest.mark.parametrize("wgmXCC", [2, 4, 8]) + def test_true_for_skxcc_fixed_wgmxcc_gt1(self, wgmXCC): + assert StreamK.usesRawQueueRank( + _FakeWriter(), _kernel(wgmXCC=wgmXCC, skxcc=4)) is True + + @pytest.mark.parametrize("wgmXCC", [1, 2, 4]) + def test_false_for_fixed_wgmxcc_without_skxcc(self, wgmXCC): + assert StreamK.usesRawQueueRank( + _FakeWriter(), _kernel(wgmXCC=wgmXCC, skxcc=0)) is False + + def test_false_for_skxcc_wgmxcc1(self): + # SKXCC with WGMXCC == 1 is already count-preserving -> cheap else-branch. + assert StreamK.usesRawQueueRank( + _FakeWriter(), _kernel(wgmXCC=1, skxcc=4)) is False + + def test_false_for_single_xcd(self): + w = _FakeWriter(numXCD=1) + assert StreamK.usesRawQueueRank(w, _kernel(wgmXCC=-1)) is False + assert StreamK.usesRawQueueRank(w, _kernel(wgmXCC=8, skxcc=8)) is False + + def test_false_for_workgroupidfromttm(self): + w = _FakeWriter(workGroupIdFromTTM=True) + assert StreamK.usesRawQueueRank(w, _kernel(wgmXCC=-1)) is False + assert StreamK.usesRawQueueRank(w, _kernel(wgmXCC=8, skxcc=8)) is False + + @pytest.mark.parametrize("streamk", [4, 5]) + def test_kw_predicate_true_for_dynamic_streamk(self, streamk): + assert KernelWriter.skUsesRawQueueRank( + _FakeKW(), _kernel(streamk=streamk, wgmXCC=-1)) is True + + @pytest.mark.parametrize("streamk", [4, 5]) + def test_kw_predicate_true_for_skxcc_wgmxcc_gt1(self, streamk): + assert KernelWriter.skUsesRawQueueRank( + _FakeKW(), _kernel(streamk=streamk, wgmXCC=8, skxcc=8)) is True + + @pytest.mark.parametrize("streamk", [0, 1, 2, 3]) + def test_kw_predicate_false_for_non_dynamic_streamk(self, streamk): + # Only the SK4/SK5 dynamic-queue variants use the raw-rank carrier. + assert KernelWriter.skUsesRawQueueRank( + _FakeKW(), _kernel(streamk=streamk, wgmXCC=-1)) is False + assert KernelWriter.skUsesRawQueueRank( + _FakeKW(), _kernel(streamk=streamk, wgmXCC=8, skxcc=8)) is False + + @pytest.mark.parametrize("streamk", [4, 5]) + def test_kw_predicate_false_for_fixed_wgmxcc_without_skxcc(self, streamk): + assert KernelWriter.skUsesRawQueueRank( + _FakeKW(), _kernel(streamk=streamk, wgmXCC=1)) is False + assert KernelWriter.skUsesRawQueueRank( + _FakeKW(), _kernel(streamk=streamk, wgmXCC=2, skxcc=0)) is False + + @pytest.mark.parametrize("streamk", [4, 5]) + def test_kw_predicate_false_for_skxcc_wgmxcc1(self, streamk): + assert KernelWriter.skUsesRawQueueRank( + _FakeKW(), _kernel(streamk=streamk, wgmXCC=1, skxcc=4)) is False + + +# --------------------------------------------------------------------------- +# AST helpers: read the *real* KernelWriter / KernelWriterAssembly source. +# --------------------------------------------------------------------------- +def _source_of(func) -> str: + return inspect.getsource(func) + + +def _mentions_const(node, value: str) -> bool: + return any( + isinstance(sub, ast.Constant) and sub.value == value + for sub in ast.walk(node) + ) + + +def _is_sk_raw_rank_guard(test) -> bool: + return ( + isinstance(test, ast.Call) + and isinstance(test.func, ast.Attribute) + and test.func.attr == "skUsesRawQueueRank" + ) + + +# =========================================================================== +# 4. Both dynamic graWorkGroup paths route through the shared _emitQueueIndex. +# Pre-fix each inlined its own StreamKIdx shift/shift/sub, so the shared +# helper call is absent -> this fails against the pre-fix source. +# =========================================================================== +class TestSharedHelperRouting: + @pytest.mark.parametrize( + "func", [StreamKDynamic.graWorkGroup, StreamKHybrid.graWorkGroup] + ) + def test_grawg_uses_shared_queue_index_helper(self, func): + assert "_emitQueueIndex" in _source_of(func), ( + "both SK4 and SK5 must derive the queue index via _emitQueueIndex" + ) + + +# =========================================================================== +# 5. The raw-rank snapshot is emitted BEFORE wgmXCC rewrites WorkGroup0, and +# only under the skUsesRawQueueRank guard, into the reused StreamKTileIdx +# carrier. Verified against the real KernelWriterAssembly.defineAndResources +# source. +# =========================================================================== +class TestSnapshotBeforeWgmXcc: + _SNAP = 'SMovB32(dst=sgpr("%s"), src=sgpr("WorkGroup0")' % _CARRIER + + def _src(self) -> str: + return _source_of(KernelWriterAssembly.defineAndResources) + + def test_snapshot_mov_targets_carrier(self): + # Pre-fix there was no snapshot; #10008 targeted StreamKQueue. The carrier + # reuse retargets it to the already-allocated StreamKTileIdx slot. + src = self._src() + assert self._SNAP in src, ( + "the raw pre-remap launch id must be snapshotted into %s" % _CARRIER + ) + assert 'sgpr("StreamKQueue")' not in src, ( + "StreamKQueue is removed; the snapshot must reuse StreamKTileIdx" + ) + + def test_snapshot_precedes_wgmxcc_reorder(self): + src = self._src() + snap = src.index('SMovB32(dst=sgpr("%s")' % _CARRIER) + remap = src.index("module.add(wgmXCC(") + assert snap < remap, ( + "the carrier snapshot must be emitted BEFORE the wgmXCC remap " + "so it captures the RAW (pre-remap) workgroup id" + ) + + def test_snapshot_is_guarded_by_predicate(self): + # The snapshot must sit inside `if self.skUsesRawQueueRank(kernel):`. + tree = ast.parse(textwrap.dedent(self._src())) + guarded = False + for node in ast.walk(tree): + if isinstance(node, ast.If) and _is_sk_raw_rank_guard(node.test): + if _mentions_const(node, _CARRIER): + guarded = True + assert guarded, ( + "the carrier snapshot must be gated by skUsesRawQueueRank" + ) + + +# =========================================================================== +# 6. Zero-SGPR carrier reuse (the whole point of the SKXCC extension): NO +# dedicated StreamKQueue SGPR is ever declared, and the carrier the snapshot +# reuses (StreamKTileIdx) is an already-allocated persistent slot present for +# BOTH the SK4 and SK5 dynamic paths -- so extending the fix to SKXCC adds no +# net persistent SGPR and cannot regress the SGPR-overflow ceiling. +# =========================================================================== +class TestZeroSgprCarrierReuse: + def _init_src(self) -> str: + return _source_of(KernelWriter._initKernel) + + def _list_literals_with(self, value: str) -> int: + """Count list literals (e.g. requiredUnalignedSgprVar += [...]) that + include *value* as a string element, across the _initKernel source.""" + tree = ast.parse(textwrap.dedent(self._init_src())) + count = 0 + for node in ast.walk(tree): + if isinstance(node, ast.List) and any( + isinstance(e, ast.Constant) and e.value == value + for e in node.elts + ): + count += 1 + return count + + def test_streamkqueue_sgpr_is_gone(self): + # No StreamKQueue anywhere in the SGPR declaration source: neither an + # append nor a list-literal element. + src = self._init_src() + assert '"StreamKQueue"' not in src and "'StreamKQueue'" not in src, ( + "the dedicated StreamKQueue SGPR must be removed (zero-SGPR reuse)" + ) + + def test_carrier_is_already_allocated_for_both_variants(self): + # StreamKTileIdx is declared in both the SK4 and SK5 blocks (>= 2 list + # literals), so the raw-rank snapshot reuses an existing slot for free. + assert self._list_literals_with(_CARRIER) >= 2, ( + "%s must be an already-allocated persistent slot for SK4 and SK5" % _CARRIER + ) + + def test_no_streamkqueue_append_remains(self): + # Belt-and-suspenders AST check: no requiredUnalignedSgprVar.append( + # "StreamKQueue") survives anywhere. + tree = ast.parse(textwrap.dedent(self._init_src())) + for node in ast.walk(tree): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "append" + and node.args + and isinstance(node.args[0], ast.Constant) + and node.args[0].value == "StreamKQueue" + ): + pytest.fail("a StreamKQueue SGPR append still exists") From cdf82027e1134f53cb8e90e6c296b7b10dedbae9 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 27 Jul 2026 23:31:24 +0000 Subject: [PATCH 2/8] feat(tensilelite): add StreamK launch summary diagnostic report (TENSILE_DB 0x200000) Add an observational diagnostic that reports the StreamK launch DECISIONS made for a (solution, problem, hardware) triple as an indented labeled block printed to stderr from ContractionSolution::solve(). It is gated solely by TENSILE_DB bit 0x200000 (sitting alongside the 0x100000 StreamK-mode-selection bit) and is off by default, so it changes no launch behaviour. The report is built from a StreamKDecisions snapshot populated by computeStreamKDecisions(), which is the single source of truth that solve() itself consumes to fill StreamKSettings (reduction, finalGrid, tiles). Sourcing the report from the same values solve() launches with means the summary cannot drift from the real launch. Reported fields: - mode: SK3/SK4/SK5 plus the effective dynamic/static resolution; - reduction: tree vs parallel (data-parallel); - grids: selected -> final with changedBy attribution (none/fixedGrid/treeBounds/workspaceDP), plus the pre-fallback grid when it differs; - tiles/skTiles/skSplit/totalItems and whether partials are present; - workspace: allocated flag plus required/ideal/given bytes in human units, with "unbounded" shown for SIZE_MAX; - dp-only: with the source distinguished (forceDPOnly param vs streamKDP debug vs runtime workspace-DP fallback); - work-queue fields: numQueues (NUM_XCD) and dynamicPartialsSlots, which print "NA (work-queues not used)" on non-dynamic paths; - the fallback flags that fired. Covered by gtests in StreamKLaunchSummary_test.cpp. --- .../include/Tensile/ContractionSolution.hpp | 88 +++ .../tensilelite/include/Tensile/Debug.hpp | 9 +- .../tensilelite/src/ContractionSolution.cpp | 406 +++++++++-- projects/hipblaslt/tensilelite/src/Debug.cpp | 5 + .../tensilelite/tests/CMakeLists.txt | 1 + .../tests/StreamKLaunchSummary_test.cpp | 634 ++++++++++++++++++ 6 files changed, 1090 insertions(+), 53 deletions(-) create mode 100644 projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp diff --git a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp index 27c049c56fda..00348820d5ca 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -214,6 +215,74 @@ namespace TensileLite { size_t globalAccumulation = 0; }; + + // Observational snapshot of the StreamK launch-parameter DECISIONS made for + // one (solution, problem, hardware) triple. Populated by + // ContractionSolution::computeStreamKDecisions(), which is the single source + // of truth that solve() itself consumes to fill StreamKSettings -- so the + // diagnostic launch summary can never drift from the real launch. Purely + // observational: computing this changes no launch behaviour. See + // Debug::printStreamKLaunchSummary() (TENSILE_DB bit 0x200000). + struct StreamKDecisions + { + // --- Mode --- + int streamKMode = 0; // sizeMapping.streamK (0 = not StreamK, else 3/4/5) + bool effectiveDynamic = false; // SK5 resolved to the dynamic (SK4) sub-path + bool isDynamic = false; // SK4, or SK5 resolved dynamic (work-queue path) + + // --- Reduction --- + origami::reduction_t reduction = origami::reduction_t::tree; // final (post-fallback) + + // --- Grid / tiles / split --- + size_t tiles = 0; // problem.getNumTiles() + // Grid selected by getSKGridImpl's config/CU/override logic BEFORE any of + // the "reset to tiles" fallbacks (tree-fixup bounds / workspace-DP) fire. + // This is the grid StreamK actually WANTED for this launch. + size_t selectedGrid = 0; + // Grid returned by getSKGridImpl -- i.e. AFTER the tree-fixup-bounds + // fallback (which may reset it to tiles) but BEFORE the workspace-DP + // fallback in solve(). Kept as a diagnostic intermediate. + size_t skGridPreFallback = 0; + // FINAL effective grid actually used at launch, after ALL fallbacks + // (fixed-grid override, tree-fixup-bounds, workspace-insufficient DP). + // Wired directly from solve()'s StreamKSettings::grid so it can never + // drift from the real launch. + size_t finalGrid = 0; + // Backward-compatible alias of finalGrid (the grid solve() launches with). + size_t skGrid = 0; + size_t skTiles = 0; // number of stream-k (partial) tiles packed by makeArgs + size_t skSplit = 0; // k-split factor (>=1) + size_t totalItems = 0; // (tiles - skTiles) + skTiles*skSplit + + // --- DP-only --- + // These three flags DISTINGUISH the source of a data-parallel-only launch: + // forceDPOnly -> sizeMapping.streamKForceDPOnly compile-time param + // streamKDP -> TENSILE_STREAMK_DATA_PARALLEL debug override + // workspaceDPFallbackFired (below) -> runtime workspace-insufficient + bool dpOnly = false; // any of the three above fired + bool forceDPOnly = false; // sizeMapping.streamKForceDPOnly (param) + bool streamKDP = false; // TENSILE_STREAMK_DATA_PARALLEL debug override + + // --- Workspace / partials --- + bool partialsPresent = false; // skTiles > 0 + bool workspaceAllocated = false; // the partials/queue block is taken and fits + size_t requiredWorkspaceBytes = 0; // SK partials(+queue) bytes actually reserved (0 if none) + size_t idealWorkspaceBytes = 0; // partials(+queue) bytes the launch wanted (pre fit check) + size_t givenWorkspaceBytes = 0; // problem.workspaceSize() + // Informational skTiles*skSplit slot count for the dynamic path, computed + // LOCALLY here for reporting only. NOTE: on this (#10008) base the actual + // partials-workspace sizing/allocation is driven by tiles%grid (see + // computeStreamKDecisions / solve), NOT by this value; it is reported so + // the invariant gap ("dynamic should reserve iff dynamicSlots>0") is + // observable without pulling in #9415's allocation behaviour change. + size_t dynamicPartialsSlots = 0; + size_t numQueues = 0; // baked per-XCD work-queue count (NUM_XCD), 0 if unknown + + // --- Fallbacks that fired (each can change selectedGrid -> finalGrid) --- + bool workspaceDPFallbackFired = false; // idealWorkspace > given -> tree + grid=tiles + bool treeBoundsFallbackFired = false; // 24-bit tree-fixup bounds -> grid=tiles + bool fixedGridUsed = false; // AMDGPU skFixedGrid override applied + }; /** * Represents a single kernel or set of kernels that can perform a single * tensor contraction. @@ -395,6 +464,25 @@ namespace TensileLite Hardware const& hardware) const; size_t partialTileSize(size_t skGrid) const; + // Compute the StreamK launch-parameter DECISIONS for this solution on the + // given problem/hardware. This is the single source of truth for the + // reduction strategy, grid, and workspace/DP fallbacks that solve() + // consumes to populate StreamKSettings, and it is directly callable from + // unit tests. It reuses the existing helpers (streamK5EffectiveDynamic, + // getSKReduction, getSKGrid, partialTileSize) so the returned snapshot + // reflects the real launch, not a re-derivation. Purely observational -- + // calling it changes no launch behaviour. Returns an all-default snapshot + // (streamKMode==0) for non-StreamK solutions. + StreamKDecisions computeStreamKDecisions(Problem const& problem, + Hardware const& hardware) const; + + // Print a one-line StreamK launch summary of the given decisions. Called + // from solve() only when Debug::printStreamKLaunchSummary() is set; + // exposed for tests. + void printStreamKLaunchSummary(std::ostream& os, + Problem const& problem, + StreamKDecisions const& decisions) const; + static float computeGranularity(float x); Granularities computeGranularities(Hardware const& hardware, diff --git a/projects/hipblaslt/tensilelite/include/Tensile/Debug.hpp b/projects/hipblaslt/tensilelite/include/Tensile/Debug.hpp index a7ef7c194f1b..40e0f0df10b8 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/Debug.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/Debug.hpp @@ -61,7 +61,7 @@ namespace TensileLite bool printWinningKernelName() const; // Re-reads TENSILE_DB, TENSILE_DB2, and TENSILE_STREAMK5_FORCE_MODE from - // the environment and updates cached state. Only these three fields are + // the environment and updates cached state. Only these fields are // refreshed; all other env-driven settings remain at their initial values. // Intended for tests that call setenv() in-process after the singleton has // already been constructed. @@ -78,6 +78,13 @@ namespace TensileLite // runtime by streamK5EffectiveDynamic(). Gated by TENSILE_DB bit 0x100000. bool printStreamKModeSelection() const; + // Emits a StreamK "launch summary" of the decisions made in the StreamK + // launch-parameter path (mode, reduction, grid/tiles/split, workspace, + // partials, DP-only, fallbacks) for each StreamK solve(). Purely + // observational. Gated by TENSILE_DB bit 0x200000 = StreamK launch summary + // (sits alongside the 0x100000 StreamK-mode-selection bit above). + bool printStreamKLaunchSummary() const; + bool naivePropertySearch() const; bool skipKernelLaunch() const; diff --git a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp index 660b453a13c5..dcaf4992615c 100644 --- a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp +++ b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -3217,7 +3218,10 @@ namespace TensileLite Hardware const& hardware, size_t tiles, origami::reduction_t reductionStrat, - bool const* sk5EffectiveDynamic); + bool const* sk5EffectiveDynamic, + bool* outFixedGridUsed = nullptr, + bool* outTreeBoundsFallback = nullptr, + size_t* outSelectedGrid = nullptr); } std::vector @@ -3306,13 +3310,20 @@ namespace TensileLite rv.push_back(generateBetaOnlyCall(problem, inputs)); } - StreamKSettings sk; + StreamKSettings sk; + StreamKDecisions skDecisions; if(sizeMapping.streamK > 0) { auto tiles = problem.getNumTiles(sizeMapping, 1); - const bool effectiveDynamic = (sizeMapping.streamK == 5) - ? streamK5EffectiveDynamic(problem, hardware) - : false; + // Single source of truth: compute the StreamK launch decisions once + // (also consumed by the diagnostic launch summary and unit tests) and + // consume them here. computeStreamKDecisions() reproduces exactly the + // reduction / grid / workspace-DP-fallback logic that previously lived + // inline, reusing the same helpers, so launch behaviour is unchanged. + // It has no side effects, so computing it before the dynamic-queue + // guard below cannot change observable behaviour (the guard still + // throws before any kernel-arg packing / kernel launch). + skDecisions = computeStreamKDecisions(problem, hardware); // Defensive: dynamic-queue / work-stealing StreamK solutions are // excluded from selection on devices whose runtime XCD count is not // a power of two or does not equal the baked per-XCD queue count @@ -3324,9 +3335,7 @@ namespace TensileLite // reject EXPLICITLY rather than silently running the fixed-mask // kernel with a mismatched queue count (which would corrupt // results). - const bool dynamicQueuePath - = (sizeMapping.streamK == 4) - || (sizeMapping.streamK == 5 && effectiveDynamic); + const bool dynamicQueuePath = skDecisions.isDynamic; if(dynamicQueuePath && streamKDynamicQueueUnsupported(hardware)) { warnStreamKDynamicQueueUnsupportedOnce(hardware); @@ -3347,55 +3356,28 @@ namespace TensileLite "per-XCD queue count; this kernel is unsupported here. " "Select a non-work-stealing solution instead."); } - if(sizeMapping.streamK == 4) - sk.reduction = origami::reduction_t::tree; - else if(sizeMapping.streamK == 5) - sk.reduction = effectiveDynamic ? origami::reduction_t::tree - : getSKReduction(problem, hardware); - else - sk.reduction = getSKReduction(problem, hardware); + // Consume the reduction / grid decided above. These are exactly the + // values the (pre-#9415) inline block computed -- computeStreamKDecisions + // mirrors that logic (including the workspace-insufficient DP fallback), + // reusing the same helpers, so launch behaviour is unchanged. + sk.reduction = skDecisions.reduction; + sk.grid = skDecisions.skGrid; sk.streamKTileSchedulingMode = problem.getParams().streamKTileSchedulingMode(); - sk.smCountTarget = problem.getParams().smCountTarget(); - sk.grid = getSKGridImpl(*this, - problem, - hardware, - tiles, - sk.reduction, - sizeMapping.streamK == 5 ? &effectiveDynamic : nullptr); - const bool streamKDP = Debug::Instance().useStreamKDataParrallel(); - const bool forceDPOnly = sizeMapping.streamKForceDPOnly != 0; - if(sk.grid > 0 - && (sk.reduction == origami::reduction_t::parallel - || (tiles % sk.grid != 0 && !streamKDP && !forceDPOnly))) - { - size_t idealWorkspace = partialTileSize(sk.grid); - // SK4 and SK5-dynamic need the per-XCD work-queue region; SK5-static - // sizes like standalone SK3. The region is sized as (per-queue - // stride) * (baked per-XCD queue count), both sourced from origami: - // the stride is the L2 cache-line size (get_default_cache_line_bytes, - // 128 B for gfx942/gfx950) so each counter owns its line (no false - // sharing), and the queue count is the per-arch XCD count. The - // acceptance guard requires runtime NUM_XCD == baked, so this equals - // cacheLineBytes * NUM_XCD for every device that reaches here. - if(dynamicQueuePath) - idealWorkspace - += streamKPerQueueStrideBytes(hardware) * streamKBakedQueueCount(hardware); - // If given workspace is less than ideal, we can fall back to DP mode - // Performance will likely be lower, but the kernel can run if workspace is unavailable. - // (The non-power-of-two XCD case is handled earlier by explicit - // rejection, not by a silent fall back to tree reduction.) - if(idealWorkspace > problem.workspaceSize()) - { - sk.reduction = origami::reduction_t::tree; - sk.grid = tiles; - } - } + sk.smCountTarget = problem.getParams().smCountTarget(); if(sk.reduction == origami::reduction_t::parallel && sk.grid / tiles < 2) { throw std::runtime_error("hipblasLT Error: Cannot use Parallel reduction with " "StreamK kernel with splitting factor < 2\n"); } + + // Wire the FINAL launch grid from the value solve() actually launches + // with (post all fallbacks), so the diagnostic summary can never drift + // from the real launch. + skDecisions.finalGrid = sk.grid; + skDecisions.skGrid = sk.grid; + if(Debug::Instance().printStreamKLaunchSummary()) + printStreamKLaunchSummary(std::cerr, problem, skDecisions); } GSUSettings gsuSettings; @@ -4158,8 +4140,15 @@ namespace TensileLite Hardware const& hardware, size_t tiles, origami::reduction_t reductionStrat, - bool const* sk5EffectiveDynamic) - { + bool const* sk5EffectiveDynamic, + bool* outFixedGridUsed, + bool* outTreeBoundsFallback, + size_t* outSelectedGrid) + { + if(outFixedGridUsed) + *outFixedGridUsed = false; + if(outTreeBoundsFallback) + *outTreeBoundsFallback = false; size_t skGrid = tiles; // Fallback const bool streamKDP = Debug::Instance().useStreamKDataParrallel(); if(streamKDP) @@ -4183,6 +4172,8 @@ namespace TensileLite if(pAMDGPU->skFixedGrid > 0) { skGrid = pAMDGPU->skFixedGrid; + if(outFixedGridUsed) + *outFixedGridUsed = true; } else if(pAMDGPU->skDynamicGrid > 0) { @@ -4303,6 +4294,11 @@ namespace TensileLite skGrid = cuCount; } + // Grid selected by the config/CU/override logic, captured before the + // "reset to tiles" tree-fixup-bounds fallback below. + if(outSelectedGrid) + *outSelectedGrid = skGrid; + // Tree-fixup uses scalarUInt24DivideAndRemainder (dividend < 2^24, divisor < 2^16). // If we exceed those bounds, fall back to DP. if(reductionStrat == origami::reduction_t::tree) @@ -4314,6 +4310,8 @@ namespace TensileLite || (tiles * itersPerTile) >= 16777216) { skGrid = tiles; + if(outTreeBoundsFallback) + *outTreeBoundsFallback = true; } } @@ -4342,6 +4340,310 @@ namespace TensileLite return size; } + // Single source of truth for the StreamK launch decisions. This mirrors -- + // and is consumed by -- the sizing block in solve(): the reduction, grid, and + // workspace/DP fallbacks below are exactly what solve() applies to the + // StreamKSettings it launches with, and the skTiles/skSplit/totalItems fields + // mirror the kernel-arg packing in makeArgs(). It reuses the existing helpers + // rather than re-deriving, so the summary cannot drift from the real launch. + // + // NOTE (this branch = #10008, WITHOUT #9415): the partials-workspace guard and + // sizing below intentionally reproduce the PRE-#9415 behaviour -- reserve iff + // (reduction==parallel || tiles%grid!=0), sized as partialTileSize(grid) (+ the + // per-XCD work-queue region for the dynamic path). It does NOT use #9415's + // dynamicPartialsSlots-driven sizing. The dynamicPartialsSlots FIELD is still + // populated (skTiles*skSplit, computed locally) purely for reporting so the + // "dynamic should reserve iff dynamicSlots>0" invariant gap is observable. + StreamKDecisions + ContractionSolution::computeStreamKDecisions(Problem const& problem, + Hardware const& hardware) const + { + StreamKDecisions d; + d.streamKMode = sizeMapping.streamK; + if(sizeMapping.streamK <= 0) + return d; + + const size_t tiles = problem.getNumTiles(sizeMapping, 1); + d.tiles = tiles; + + const bool effectiveDynamic + = (sizeMapping.streamK == 5) ? streamK5EffectiveDynamic(problem, hardware) : false; + d.effectiveDynamic = effectiveDynamic; + + // Reduction strategy -- mirrors solve(). + origami::reduction_t reduction; + if(sizeMapping.streamK == 4) + reduction = origami::reduction_t::tree; + else if(sizeMapping.streamK == 5) + reduction = effectiveDynamic ? origami::reduction_t::tree + : getSKReduction(problem, hardware); + else + reduction = getSKReduction(problem, hardware); + + // Grid -- reuses getSKGridImpl (same call solve() makes), and captures the + // fixed-grid / tree-bounds fallbacks plus the pre-tree-bounds "selected" + // grid via the optional out-params. + bool fixedGridUsed = false; + bool treeBounds = false; + size_t selectedGrid = 0; + const size_t gridInitial = getSKGridImpl(*this, + problem, + hardware, + tiles, + reduction, + sizeMapping.streamK == 5 ? &effectiveDynamic + : nullptr, + &fixedGridUsed, + &treeBounds, + &selectedGrid); + d.selectedGrid = selectedGrid; + d.skGridPreFallback = gridInitial; + d.fixedGridUsed = fixedGridUsed; + d.treeBoundsFallbackFired = treeBounds; + + size_t grid = gridInitial; + const bool streamKDP = Debug::Instance().useStreamKDataParrallel(); + const bool forceDPOnly = sizeMapping.streamKForceDPOnly != 0; + d.streamKDP = streamKDP; + d.forceDPOnly = forceDPOnly; + + const bool isDynamic + = (sizeMapping.streamK == 4) || (sizeMapping.streamK == 5 && effectiveDynamic); + d.isDynamic = isDynamic; + + d.numQueues = streamKBakedQueueCount(hardware); + d.givenWorkspaceBytes = problem.workspaceSize(); + + // Workspace / DP fallback -- mirrors the PRE-#9415 guard in solve(): + // reserve iff (reduction==parallel || tiles%grid!=0), sized by grid (not by + // dynamicSlots). No #9415 behaviour is pulled in here. + size_t idealWorkspace = 0; + bool needPartials = false; + if(grid > 0 + && (reduction == origami::reduction_t::parallel + || (tiles % grid != 0 && !streamKDP && !forceDPOnly))) + { + needPartials = true; + idealWorkspace = partialTileSize(grid); + if(isDynamic) + idealWorkspace + += streamKPerQueueStrideBytes(hardware) * streamKBakedQueueCount(hardware); + if(idealWorkspace > problem.workspaceSize()) + { + reduction = origami::reduction_t::tree; + grid = tiles; + d.workspaceDPFallbackFired = true; + } + } + + d.reduction = reduction; + d.skGrid = grid; + d.finalGrid = grid; + d.idealWorkspaceBytes = idealWorkspace; + d.requiredWorkspaceBytes = (needPartials && !d.workspaceDPFallbackFired) ? idealWorkspace : 0; + d.workspaceAllocated = d.requiredWorkspaceBytes > 0; + d.dpOnly = streamKDP || forceDPOnly || d.workspaceDPFallbackFired; + + // skTiles / skSplit / totalItems -- mirror the kernel-arg packing in + // makeArgs, using the FINAL (post-fallback) grid and reduction. + const size_t itersPerTile = std::max(size_t{1}, problem.getItersPerTile(sizeMapping)); + if(isDynamic) + { + AMDGPU const* pAMDGPU = dynamic_cast(&hardware); + int overrideT = pAMDGPU ? pAMDGPU->skTiles : -1; + int overrideS = pAMDGPU ? pAMDGPU->skSplit : -1; + uint32_t skTiles = 0; + uint32_t skSplit = 2; + if(overrideT > -1) + skTiles = static_cast(overrideT); + if(overrideS > -1) + skSplit = static_cast(overrideS); + uint32_t skItersPerWI = CeilDivide(static_cast(itersPerTile), skSplit); + skSplit = CeilDivide(static_cast(itersPerTile), skItersPerWI); + d.skTiles = skTiles; + d.skSplit = skSplit; + d.totalItems = (tiles - skTiles) + static_cast(skTiles) * skSplit; + } + else if(reduction == origami::reduction_t::parallel && tiles > 0) + { + uint32_t skSplit = static_cast(grid / tiles); + d.skSplit = skSplit; + d.skTiles = skSplit; // parallel path packs skTiles = skSplit + d.totalItems = tiles; + } + else + { + AMDGPU const* pAMDGPU = dynamic_cast(&hardware); + int fullTiles = pAMDGPU ? pAMDGPU->skFullTiles : 0; + bool bigEnough = tiles > grid; + uint32_t skTiles = forceDPOnly ? 0u : static_cast(grid); + if(!forceDPOnly && grid != 0 && tiles % grid != 0) + { + skTiles = bigEnough ? static_cast(grid * fullTiles + tiles % grid) + : static_cast(tiles); + skTiles = std::min(skTiles, static_cast(tiles)); + } + d.skTiles = skTiles; + d.skSplit = 1; + d.totalItems = tiles; + } + + // Informational only (see field doc): skTiles*skSplit slot count for the + // dynamic path, computed LOCALLY here. It does NOT feed the allocation + // guard above, so no #9415 sizing behaviour is introduced. + d.dynamicPartialsSlots = isDynamic ? static_cast(d.skTiles) * d.skSplit : 0; + + d.partialsPresent = d.skTiles > 0; + return d; + } + + void ContractionSolution::printStreamKLaunchSummary(std::ostream& os, + Problem const& problem, + StreamKDecisions const& d) const + { + (void)problem; // reserved for future per-problem detail; keeps the API stable + auto reductionStr = [](origami::reduction_t r) { + return r == origami::reduction_t::parallel ? "parallel(DP)" : "tree"; + }; + const char* modeStr = "?"; + switch(d.streamKMode) + { + case 0: modeStr = "none"; break; + case 3: modeStr = "SK3(static)"; break; + case 4: modeStr = "SK4(dynamic)"; break; + case 5: + modeStr = d.effectiveDynamic ? "SK5->dynamic(SK4)" : "SK5->static(SK3)"; + break; + default: modeStr = "SK?"; break; + } + + // Which fallback (if any) turned the initially-selected grid into the + // final launch grid. Reported alongside selectedGrid vs finalGrid. + const char* gridChangedBy = "none"; + if(d.workspaceDPFallbackFired) + gridChangedBy = "workspaceDP"; + else if(d.treeBoundsFallbackFired) + gridChangedBy = "treeBounds"; + else if(d.fixedGridUsed) + gridChangedBy = "fixedGrid"; + + // Which mechanism (if any) makes this launch data-parallel-only. + const char* dpOnlySource = "none"; + if(d.forceDPOnly) + dpOnlySource = "forceDPOnly(param)"; + else if(d.streamKDP) + dpOnlySource = "streamKDP(debug)"; + else if(d.workspaceDPFallbackFired) + dpOnlySource = "workspaceDP(runtime)"; + + // Human-readable byte annotation, e.g. "1245184 (1.19 MiB)". SIZE_MAX is + // reported as "unbounded" (the sentinel bench passes for "no cap"). + auto humanUnit = [](size_t bytes) -> std::string { + static const char* units[] = {"B", "KiB", "MiB", "GiB", "TiB"}; + double v = static_cast(bytes); + int u = 0; + while(v >= 1024.0 && u < 4) + { + v /= 1024.0; + ++u; + } + std::ostringstream o; + if(u == 0) + o << bytes << " B"; + else + { + o.setf(std::ios::fixed); + o.precision(2); + o << v << " " << units[u]; + } + return o.str(); + }; + auto fmtBytes = [&](size_t bytes) -> std::string { + std::ostringstream o; + o << bytes << " (" << humanUnit(bytes) << ")"; + return o.str(); + }; + auto fmtGiven = [&](size_t bytes) -> std::string { + if(bytes == std::numeric_limits::max()) + return "unbounded"; + return fmtBytes(bytes); + }; + + // Deeply-indented, multi-line labeled block. The leading token + // "TensileLite::StreamK LAUNCH SUMMARY" is preserved verbatim (with the + // kernel name on the same first line) so existing log scrapers / test + // substring matches keep working. Each section header sits on its own + // line and its fields are indented beneath it as aligned "key = value" + // pairs, one field per line. This is a pure LAYOUT change: no field, + // value, label, or the NA-when-!isDynamic behaviour is altered, and no + // launch behaviour is affected. + // + // 'field' prints one indented, '='-aligned pair. 'width' is the per-section + // key column width chosen so every '=' in that section lines up. + auto field = [&os](int width, const char* key, std::string const& value) { + os << " " << std::left << std::setw(width) << key << " = " << value << "\n"; + }; + auto yn = [](bool b) -> std::string { return b ? "yes" : "no"; }; + + os << "TensileLite::StreamK LAUNCH SUMMARY kernel='" << this->kernelName << "'\n"; + + os << " mode:\n"; + field(9, "mode", modeStr); + field(9, "streamK", std::to_string(d.streamKMode)); + field(9, "reduction", reductionStr(d.reduction)); + field(9, "isDynamic", yn(d.isDynamic)); + + os << " grid:\n"; + field(11, "selected", std::to_string(d.selectedGrid)); + field(11, "final", std::to_string(d.finalGrid)); + field(11, "changedBy", gridChangedBy); + // preFallback is only interesting when the tree-bounds fixup already moved + // the grid away from what was selected (before the workspace-DP fallback). + if(d.skGridPreFallback != d.selectedGrid) + field(11, "preFallback", std::to_string(d.skGridPreFallback)); + + os << " tiles:\n"; + field(10, "tiles", std::to_string(d.tiles)); + field(10, "skTiles", std::to_string(d.skTiles)); + field(10, "skSplit", std::to_string(d.skSplit)); + field(10, "totalItems", std::to_string(d.totalItems)); + field(10, "partials", yn(d.partialsPresent)); + + os << " workspace:\n"; + field(9, "allocated", yn(d.workspaceAllocated)); + field(9, "required", fmtBytes(d.requiredWorkspaceBytes)); + field(9, "ideal", fmtBytes(d.idealWorkspaceBytes)); + field(9, "given", fmtGiven(d.givenWorkspaceBytes)); + + os << " dp-only:\n"; + field(11, "dpOnly", yn(d.dpOnly)); + field(11, "source", dpOnlySource); + field(11, "forceDPOnly", yn(d.forceDPOnly)); + field(11, "streamKDP", yn(d.streamKDP)); + + // Work-queue fields describe the per-XCD dynamic work-queue synchronizer, + // which only exists on the dynamic path (SK4, or SK5 resolved dynamic). + // On non-dynamic paths (SK3 / SK5-static / parallel-DP / dp-only) these + // values are meaningless, so they are printed as NA rather than a + // misleading number. Gated on the SAME isDynamic predicate solve() uses; + // the struct still holds the raw counts. + os << " work-queue:\n"; + if(d.isDynamic) + { + field(20, "numQueues(NUM_XCD)", std::to_string(d.numQueues)); + field(20, "dynamicPartialsSlots", std::to_string(d.dynamicPartialsSlots)); + } + else + { + os << " NA (work-queues not used)\n"; + } + + os << " fallbacks:\n"; + field(19, "fixedGrid", yn(d.fixedGridUsed)); + field(19, "workspaceDPFallback", yn(d.workspaceDPFallbackFired)); + field(19, "treeBoundsFallback", yn(d.treeBoundsFallbackFired)); + } + float ContractionSolution::computeGranularity(float x) { return x / ceil(x); diff --git a/projects/hipblaslt/tensilelite/src/Debug.cpp b/projects/hipblaslt/tensilelite/src/Debug.cpp index f9870cdc99a0..a7f705aa8d7b 100644 --- a/projects/hipblaslt/tensilelite/src/Debug.cpp +++ b/projects/hipblaslt/tensilelite/src/Debug.cpp @@ -115,6 +115,11 @@ namespace TensileLite return m_value & 0x100000; } + bool Debug::printStreamKLaunchSummary() const + { + return m_value & 0x200000; + } + bool Debug::naivePropertySearch() const { return m_naivePropertySearch; diff --git a/projects/hipblaslt/tensilelite/tests/CMakeLists.txt b/projects/hipblaslt/tensilelite/tests/CMakeLists.txt index 00458f198c12..da21046bd56f 100644 --- a/projects/hipblaslt/tensilelite/tests/CMakeLists.txt +++ b/projects/hipblaslt/tensilelite/tests/CMakeLists.txt @@ -40,6 +40,7 @@ target_sources(tensilelite-tests "${CMAKE_CURRENT_SOURCE_DIR}/ResultComparison_test.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/ReferenceMXFastPath_test.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/SolutionValidation_test.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/StreamKLaunchSummary_test.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TensorDescriptor_test.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/TestData.cpp" ) diff --git a/projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp b/projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp new file mode 100644 index 000000000000..d6c622074259 --- /dev/null +++ b/projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp @@ -0,0 +1,634 @@ +// Copyright (C) Advanced Micro Devices, Inc., or its affiliates. +// SPDX-License-Identifier: MIT + +// Unit tests for the StreamK "launch summary" decision snapshot produced by +// ContractionSolution::computeStreamKDecisions(). This is the single source of +// truth solve() consumes to fill StreamKSettings, so asserting on it here is +// asserting on the real launch DECISIONS (mode / reduction / grid / tiles / +// split / workspace / partials / DP-only / fallbacks) made in the StreamK +// launch-parameter path -- without needing a GPU. Host-only: mock AMDGPU and +// hip::HipAMDGPU devices, no device library required. +// +// This file lives on the #10008-only base (StreamK per-XCD work-queue counter +// self-reset fix), WITHOUT #9415 (dynamic partials-workspace sizing). The +// partials-workspace behaviour asserted below is therefore the PRE-#9415 +// behaviour: the dynamic (SK4 / SK5-dynamic) path reserves the partials region +// based on tiles%grid divisibility, NOT on the skTiles*skSplit slot count. The +// invariant-gap tests near the bottom pin that CURRENT behaviour AND document +// the DESIRED invariant (reserve iff dynamicSlots>0) that #9415 implements. + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "FallbackTestUtils.hpp" + +using namespace TensileLite; +using namespace TensileLite::testing; + +namespace +{ + constexpr size_t kGfx950AnalyticalCuCount = 256; + + // gfx950 analytical hardware advertising NUM_XCD=8 (matches the baked + // per-XCD work-queue count), so the SK4 / SK5-dynamic work-stealing path is + // supported. Mirrors makeGfx950AnalyticalHardware in CuCount_test.cpp. + origami::hardware_t makeGfx950AnalyticalHardware() + { + using arch_t = origami::hardware_t::architecture_t; + return origami::hardware_t(arch_t::gfx950, + kGfx950AnalyticalCuCount, + 163840, + 262144, + 8, // NUM_XCD + 1.0, + 1.0, + 1.0, + 4000000, + 1.2, + 1, + std::make_tuple(0.0, 0.008, 0.0)); + } + + hip::HipAMDGPU makeHipDeviceWithAnalytical(origami::hardware_t const& hw) + { + hip::HipAMDGPU device; + device.processor = AMDGPU::Processor::gfx950; + device.computeUnitCount = static_cast(hw.N_CU); + device.deviceName = "test-gfx950-analytical"; + device.analyticalHardware = std::make_shared(hw); + return device; + } + + void initStreamKSolution(ContractionSolution& solution, int streamK) + { + solution.sizeMapping.streamK = streamK; + solution.sizeMapping.streamKAtomic = 0; + solution.sizeMapping.streamKForceDPOnly = 0; + solution.sizeMapping.macroTile = TensileLite::dim3(128, 128, 1); + solution.sizeMapping.depthU = 64; + solution.sizeMapping.matrixInstruction = {16, 16, 32, 1}; + solution.sizeMapping.CUOccupancy = 1; + solution.sizeMapping.workspaceSizePerElemC = 4; + } + + ContractionProblemGemm makeGemmProblem(size_t m, size_t n, size_t k) + { + auto problem = ContractionProblemGemm::GEMM(false, false, m, n, k, m, n, m, 1.0, false, 1); + problem.setComputeInputTypeA(rocisa::DataType::Float); + problem.setComputeInputTypeB(rocisa::DataType::Float); + return problem; + } + + struct AnalyticalEnv + { + AnalyticalEnv() + : hw(makeGfx950AnalyticalHardware()) + , device(makeHipDeviceWithAnalytical(hw)) + { + } + origami::hardware_t hw; + hip::HipAMDGPU device; + }; + + // The summary now uses a deeply-indented "key = value" layout whose column + // widths are chosen per-section for alignment. Collapsing runs of spaces to a + // single space lets the assertions below match on the stable tokens + // ("changedBy = ...", "source = ...", etc.) without pinning exact whitespace. + // Newlines are preserved so section boundaries still matter. + std::string collapseSpaces(std::string const& s) + { + std::string out; + out.reserve(s.size()); + bool prevSpace = false; + for(char c : s) + { + if(c == ' ') + { + if(!prevSpace) + out.push_back(' '); + prevSpace = true; + } + else + { + out.push_back(c); + prevSpace = false; + } + } + return out; + } +} // namespace + +// --------------------------------------------------------------------------- +// No-drift contract: the snapshot fields equal what the individual production +// helpers report, so the summary reflects the REAL decisions (not a re-derivation +// that could drift). This is the property that makes the summary trustworthy. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, SnapshotMatchesHelpersForDynamicPartialTiles) +{ + AnalyticalEnv env; + ContractionSolution solution; + initStreamKSolution(solution, 4); // SK4 = unconditionally dynamic (tree, work-queue) + + // 4096x4224 -> tiles = 32*33 = 1056; grid = min(1056, 256) = 256; 1056 % 256 + // != 0 -> partial tiles -> partials workspace required. + auto problem = makeGemmProblem(4096, 4224, 64); + problem.setWorkspaceSize(std::numeric_limits::max()); + + auto d = solution.computeStreamKDecisions(problem, env.device); + + const size_t tiles = problem.getNumTiles(solution.sizeMapping, 1); + const auto red = solution.getSKReduction(problem, env.device); + const size_t grid = solution.getSKGrid(problem, env.device, tiles, red); + + EXPECT_EQ(d.streamKMode, 4); + EXPECT_TRUE(d.isDynamic); + EXPECT_EQ(d.reduction, origami::reduction_t::tree); + EXPECT_EQ(d.tiles, tiles); + // getSKGrid() reproduces the pre-fallback grid the snapshot records. + EXPECT_EQ(d.skGridPreFallback, grid); + ASSERT_NE(tiles % grid, 0u) << "test needs partial tiles"; + + // No fallback fires here, so selected == pre-fallback == final launch grid. + EXPECT_EQ(d.selectedGrid, d.skGridPreFallback); + EXPECT_EQ(d.finalGrid, d.skGridPreFallback); + EXPECT_EQ(d.skGrid, d.finalGrid); + EXPECT_FALSE(d.workspaceDPFallbackFired); + EXPECT_FALSE(d.treeBoundsFallbackFired); + EXPECT_FALSE(d.fixedGridUsed); + + // The authoritative workspace query must agree with the snapshot byte-for-byte. + EXPECT_EQ(d.requiredWorkspaceBytes, solution.requiredWorkspaceSize(problem, env.device)); + EXPECT_GT(d.requiredWorkspaceBytes, 0u); + EXPECT_TRUE(d.workspaceAllocated); + EXPECT_FALSE(d.dpOnly); + EXPECT_EQ(d.numQueues, 8u) << "gfx950 bakes 8 per-XCD work queues (NUM_XCD)"; +} + +// --------------------------------------------------------------------------- +// SK5 resolves to the dynamic (SK4) sub-path when the API mode is ON, with tree +// reduction; SK5 OFF stays static (SK3) with no work-queue. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, Sk5OnResolvesDynamicTree) +{ + AnalyticalEnv env; + ContractionSolution solution; + initStreamKSolution(solution, 5); + + auto problem = makeGemmProblem(4096, 4224, 64); + problem.setWorkspaceSize(std::numeric_limits::max()); + problem.setParams().setStreamKTileSchedulingMode(1); // ON + + auto d = solution.computeStreamKDecisions(problem, env.device); + + EXPECT_EQ(d.streamKMode, 5); + EXPECT_TRUE(d.effectiveDynamic); + EXPECT_TRUE(d.isDynamic); + EXPECT_EQ(d.reduction, origami::reduction_t::tree); + EXPECT_EQ(d.numQueues, 8u); + // With plenty of workspace the partials block is taken (no DP fallback). + EXPECT_TRUE(d.workspaceAllocated); + EXPECT_FALSE(d.dpOnly); +} + +TEST(StreamKLaunchSummaryTest, Sk5OffResolvesStaticSk3) +{ + AnalyticalEnv env; + ContractionSolution solution; + initStreamKSolution(solution, 5); + + auto problem = makeGemmProblem(4096, 4224, 64); + problem.setWorkspaceSize(std::numeric_limits::max()); + problem.setParams().setStreamKTileSchedulingMode(0); // OFF (static, smCountTarget=0) + + auto d = solution.computeStreamKDecisions(problem, env.device); + + EXPECT_EQ(d.streamKMode, 5); + EXPECT_FALSE(d.effectiveDynamic); + EXPECT_FALSE(d.isDynamic) << "SK5-OFF must take the static (SK3) sub-path"; + EXPECT_EQ(d.numQueues, 8u); // baked count still reported (informational) + // SK3-static: no per-XCD work-queue region in the workspace it reserves. + if(d.workspaceAllocated) + EXPECT_EQ(d.requiredWorkspaceBytes, solution.partialTileSize(d.skGrid)) + << "static SK3 workspace = partialTileSize(grid), no work-queue region"; +} + +// --------------------------------------------------------------------------- +// SK3 static with partial tiles: partials present, skTiles>0, workspace>0, +// not dynamic, not DP-only. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, Sk3StaticPartialTiles) +{ + ContractionSolution solution; + initStreamKSolution(solution, 3); + + // 4096x4224 -> tiles = 1056; grid = cuCount = 64; 1056 % 64 == 32 (!=0). + auto problem = makeGemmProblem(4096, 4224, 512); + problem.setWorkspaceSize(std::numeric_limits::max()); + + auto device = makeDevice(_MI350_CHIP_ID, _CPX_CU, "mi350cpx"); + device.skDynamicGrid = 0; + + auto d = solution.computeStreamKDecisions(problem, device); + + EXPECT_EQ(d.streamKMode, 3); + EXPECT_FALSE(d.isDynamic); + EXPECT_EQ(d.reduction, origami::reduction_t::tree); + ASSERT_NE(d.tiles % d.skGrid, 0u) << "test needs partial tiles"; + EXPECT_TRUE(d.partialsPresent); + EXPECT_GT(d.skTiles, 0u); + EXPECT_TRUE(d.workspaceAllocated); + EXPECT_EQ(d.requiredWorkspaceBytes, solution.requiredWorkspaceSize(problem, device)); + EXPECT_FALSE(d.dpOnly); + EXPECT_EQ(d.numQueues, 0u) << "static SK3 mock device has no analytical work-queue count"; +} + +// --------------------------------------------------------------------------- +// Force-DP-only (SK3): every tile stays data-parallel. skTiles==0, no partials, +// workspace==0, dpOnly reported and sourced from the compile-time PARAM. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, ForceDpOnlyIsDpOnlyNoWorkspace) +{ + ContractionSolution solution; + initStreamKSolution(solution, 3); + solution.sizeMapping.streamKForceDPOnly = 1; + + auto problem = makeGemmProblem(4096, 4224, 512); + problem.setWorkspaceSize(std::numeric_limits::max()); + + auto device = makeDevice(_MI350_CHIP_ID, _CPX_CU, "mi350cpx"); + device.skDynamicGrid = 0; + + auto d = solution.computeStreamKDecisions(problem, device); + + EXPECT_TRUE(d.forceDPOnly); + EXPECT_TRUE(d.dpOnly); + // dp-only distinction: this is the PARAM source, NOT the runtime fallback. + EXPECT_FALSE(d.workspaceDPFallbackFired) << "forceDPOnly is a param, not a runtime fallback"; + EXPECT_FALSE(d.streamKDP); + EXPECT_EQ(d.skTiles, 0u); + EXPECT_FALSE(d.partialsPresent); + EXPECT_EQ(d.requiredWorkspaceBytes, 0u); + EXPECT_FALSE(d.workspaceAllocated); + // Force-DP-only launches on exactly the hardware CU count (mirrors + // StreamKForceDPOnlyTest.UsesHardwareCuCount). The param does NOT reset the + // grid to tiles (that is the runtime fallback), so finalGrid == selectedGrid. + EXPECT_EQ(d.skGrid, static_cast(_CPX_CU)); + EXPECT_EQ(d.finalGrid, static_cast(_CPX_CU)); + EXPECT_EQ(d.finalGrid, d.selectedGrid); +} + +// --------------------------------------------------------------------------- +// Workspace-starved SK4 falls back to a DP grid (grid=tiles, tree reduction): +// workspaceDPFallbackFired and dpOnly set, nothing reserved. This is the RUNTIME +// dp-only source, and it is the fallback that turns selectedGrid into finalGrid. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, WorkspaceDpFallbackFires) +{ + AnalyticalEnv env; + ContractionSolution solution; + initStreamKSolution(solution, 4); + + auto problem = makeGemmProblem(4096, 4224, 64); + problem.setWorkspaceSize(0); // no workspace at all -> must fall back to DP + + auto d = solution.computeStreamKDecisions(problem, env.device); + + ASSERT_NE(d.tiles % d.skGridPreFallback, 0u) << "test needs partial tiles pre-fallback"; + EXPECT_GT(d.idealWorkspaceBytes, 0u) << "the launch wanted a partials region"; + EXPECT_TRUE(d.workspaceDPFallbackFired); + EXPECT_TRUE(d.dpOnly); + // dp-only distinction: RUNTIME fallback, not the compile-time param. + EXPECT_FALSE(d.forceDPOnly); + // selected vs final: the fallback resets the grid to tiles. + EXPECT_EQ(d.selectedGrid, d.skGridPreFallback) << "no tree-bounds fallback here"; + EXPECT_NE(d.selectedGrid, d.finalGrid) << "workspace-DP fallback changed the grid"; + EXPECT_EQ(d.finalGrid, d.tiles) << "DP fallback sets grid = tiles"; + EXPECT_EQ(d.skGrid, d.tiles); + EXPECT_EQ(d.reduction, origami::reduction_t::tree); + EXPECT_EQ(d.requiredWorkspaceBytes, 0u); + EXPECT_FALSE(d.workspaceAllocated); + // requiredWorkspaceSize agrees: nothing reserved when the workspace is too small. + EXPECT_EQ(solution.requiredWorkspaceSize(problem, env.device), 0u); +} + +// --------------------------------------------------------------------------- +// Selected vs final grid + which-fallback attribution, exercised directly: +// the workspace-DP fallback is the mechanism that makes finalGrid != selectedGrid. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, SelectedVsFinalGridAttribution) +{ + AnalyticalEnv env; + ContractionSolution solution; + initStreamKSolution(solution, 4); + + // Enough tiles for a real StreamK grid, but zero workspace forces the DP + // fallback that overwrites the selected grid with tiles. + auto problem = makeGemmProblem(4096, 4224, 64); + problem.setWorkspaceSize(0); + + auto d = solution.computeStreamKDecisions(problem, env.device); + + // "selected" = what StreamK wanted (CU/config), "final" = what launched. + EXPECT_GT(d.selectedGrid, 0u); + EXPECT_EQ(d.finalGrid, d.tiles); + EXPECT_NE(d.selectedGrid, d.finalGrid); + // Exactly one of the grid-changing fallbacks is attributed, and it is the + // workspace-DP one for this scenario. + EXPECT_TRUE(d.workspaceDPFallbackFired); + EXPECT_FALSE(d.fixedGridUsed); +} + +// --------------------------------------------------------------------------- +// DP-only source disambiguation: the snapshot distinguishes the compile-time +// PARAM (forceDPOnly) from the RUNTIME workspace-insufficient fallback. Both +// yield dpOnly, but only the runtime path sets workspaceDPFallbackFired and +// resets finalGrid to tiles. +// (The third source, the TENSILE_STREAMK_DATA_PARALLEL debug flag, is not +// toggleable in-process here: Debug caches it at construction and +// reloadDebugBitsForTest() intentionally does not refresh it. Its plumbing is +// still asserted-absent below via d.streamKDP.) +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, DpOnlySourceDistinguishesParamVsRuntime) +{ + AnalyticalEnv env; + + // (1) PARAM source: forceDPOnly on an SK4 solution with ample workspace. + ContractionSolution paramSol; + initStreamKSolution(paramSol, 4); + paramSol.sizeMapping.streamKForceDPOnly = 1; + auto paramProblem = makeGemmProblem(4096, 4224, 64); + paramProblem.setWorkspaceSize(std::numeric_limits::max()); + auto pd = paramSol.computeStreamKDecisions(paramProblem, env.device); + + EXPECT_TRUE(pd.dpOnly); + EXPECT_TRUE(pd.forceDPOnly); + EXPECT_FALSE(pd.workspaceDPFallbackFired); + EXPECT_FALSE(pd.streamKDP); + // forceDPOnly does NOT reset the grid to tiles. + EXPECT_EQ(pd.finalGrid, pd.selectedGrid); + EXPECT_NE(pd.finalGrid, pd.tiles); + + // (2) RUNTIME source: no workspace forces the DP fallback (grid=tiles). + ContractionSolution runSol; + initStreamKSolution(runSol, 4); + auto runProblem = makeGemmProblem(4096, 4224, 64); + runProblem.setWorkspaceSize(0); + auto rd = runSol.computeStreamKDecisions(runProblem, env.device); + + EXPECT_TRUE(rd.dpOnly); + EXPECT_FALSE(rd.forceDPOnly); + EXPECT_TRUE(rd.workspaceDPFallbackFired); + EXPECT_EQ(rd.finalGrid, rd.tiles); +} + +// --------------------------------------------------------------------------- +// INVARIANT GAP (#9415 territory) -- pinned on the #10008-only base. +// +// Desired invariant (implemented by PR #9415): a DYNAMIC (SK4 / SK5-dynamic) +// launch should reserve the partials workspace IFF dynamicPartialsSlots +// (skTiles*skSplit) > 0, because that path indexes the workspace by tile-split +// slot, not by grid position. +// +// CURRENT behaviour on this #10008 base: the dynamic path reserves the partials +// workspace based on tiles%grid divisibility (the SK3 rule), independent of +// dynamicPartialsSlots. The three tests below bracket that gap. +// --------------------------------------------------------------------------- + +// Case A: dynamicSlots == 0 (no split stream-k tiles) AND tiles % grid != 0. +// CURRENT: workspace IS reserved (because tiles%grid!=0) even though the dynamic +// packing produced no partial tiles. DESIRED: it should NOT reserve here. +TEST(StreamKLaunchSummaryTest, DynamicNoSlotsButIndivisible_ReservesWorkspace_GAP) +{ + ContractionSolution solution; + initStreamKSolution(solution, 4); // dynamic + + // 4096x4224 -> tiles = 1056; grid = cuCount = 64; 1056 % 64 == 32 (!=0). + // No skTiles override -> the dynamic packing yields skTiles == 0. + auto problem = makeGemmProblem(4096, 4224, 512); + problem.setWorkspaceSize(std::numeric_limits::max()); + + auto device = makeDevice(_MI350_CHIP_ID, _CPX_CU, "mi350cpx"); + device.skDynamicGrid = 0; + + auto d = solution.computeStreamKDecisions(problem, device); + + ASSERT_TRUE(d.isDynamic); + ASSERT_NE(d.tiles % d.skGrid, 0u) << "case needs tiles % grid != 0"; + EXPECT_EQ(d.skTiles, 0u) << "no override -> dynamic packing produces no split tiles"; + EXPECT_EQ(d.dynamicPartialsSlots, 0u) << "skTiles*skSplit == 0"; + + // CURRENT (#10008) behaviour: reserves anyway because tiles%grid != 0. + EXPECT_TRUE(d.workspaceAllocated); + EXPECT_GT(d.requiredWorkspaceBytes, 0u); + EXPECT_EQ(d.requiredWorkspaceBytes, solution.requiredWorkspaceSize(problem, device)); + // DESIRED invariant (PR #9415): with dynamicPartialsSlots==0 the dynamic path + // needs no partials region, so ideally this would be: + // EXPECT_FALSE(d.workspaceAllocated); + // EXPECT_EQ(d.requiredWorkspaceBytes, 0u); + // Pinned here as the observed gap until #9415 lands on this base. +} + +// Case B (complement): dynamicSlots == 0 AND tiles % grid == 0. +// CURRENT and DESIRED agree: no workspace reserved. +TEST(StreamKLaunchSummaryTest, DynamicNoSlotsAndDivisible_NoWorkspace) +{ + ContractionSolution solution; + initStreamKSolution(solution, 4); // dynamic + + // 4096x4096 -> tiles = 1024; grid = cuCount = 64; 1024 % 64 == 0. + auto problem = makeGemmProblem(4096, 4096, 512); + problem.setWorkspaceSize(std::numeric_limits::max()); + + auto device = makeDevice(_MI350_CHIP_ID, _CPX_CU, "mi350cpx"); + device.skDynamicGrid = 0; + + auto d = solution.computeStreamKDecisions(problem, device); + + ASSERT_TRUE(d.isDynamic); + ASSERT_EQ(d.tiles % d.skGrid, 0u) << "case needs tiles % grid == 0"; + EXPECT_EQ(d.dynamicPartialsSlots, 0u); + EXPECT_FALSE(d.workspaceAllocated); + EXPECT_EQ(d.requiredWorkspaceBytes, 0u); + EXPECT_EQ(d.requiredWorkspaceBytes, solution.requiredWorkspaceSize(problem, device)); +} + +// Case C: dynamicSlots > 0 (skTiles override) BUT tiles % grid == 0. +// CURRENT (#10008): NO workspace reserved -- this is the OOB-write bug #9415 +// fixes, because the kernel indexes up to skTiles*skSplit slots. DESIRED +// (PR #9415): reserve partialTileSize(max(dynamicSlots, grid)) + queue region. +TEST(StreamKLaunchSummaryTest, DynamicSlotsPositiveButDivisible_NoWorkspace_GAP) +{ + ContractionSolution solution; + initStreamKSolution(solution, 4); // dynamic + + // 4096x4096 -> tiles = 1024; grid = cuCount = 64; 1024 % 64 == 0, so ONLY the + // skTiles override creates partial (split) tiles. + auto problem = makeGemmProblem(4096, 4096, 512); + problem.setWorkspaceSize(std::numeric_limits::max()); + + auto device = makeDevice(_MI350_CHIP_ID, _CPX_CU, "mi350cpx"); + device.skDynamicGrid = 0; + device.skTiles = 256; // override: number of split stream-k tiles + device.skSplit = 4; // override: k-split factor per tile + + auto d = solution.computeStreamKDecisions(problem, device); + + ASSERT_TRUE(d.isDynamic); + ASSERT_EQ(d.tiles % d.skGrid, 0u) << "case needs tiles % grid == 0"; + EXPECT_EQ(d.skTiles, 256u); + EXPECT_GE(d.skSplit, 1u); + EXPECT_TRUE(d.partialsPresent) << "override produced split stream-k tiles"; + EXPECT_GT(d.dynamicPartialsSlots, 0u) << "skTiles*skSplit > 0"; + // totalItems = (tiles - skTiles) + skTiles*skSplit + EXPECT_EQ(d.totalItems, (d.tiles - d.skTiles) + d.skTiles * d.skSplit); + + // CURRENT (#10008) behaviour: NO workspace reserved despite dynamicSlots>0, + // because tiles%grid==0 gates the SK3-style reservation. requiredWorkspaceSize + // agrees (also 0). + EXPECT_FALSE(d.workspaceAllocated); + EXPECT_EQ(d.requiredWorkspaceBytes, 0u); + EXPECT_EQ(d.requiredWorkspaceBytes, solution.requiredWorkspaceSize(problem, device)); + // DESIRED invariant (PR #9415): dynamicPartialsSlots>0 must reserve, i.e. + // EXPECT_TRUE(d.workspaceAllocated); + // EXPECT_GT(d.requiredWorkspaceBytes, 0u); + // Pinned here as the observed gap until #9415 lands on this base. +} + +// --------------------------------------------------------------------------- +// Non-StreamK solutions produce an inert (mode==0) snapshot. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, NonStreamKProducesEmptySnapshot) +{ + ContractionSolution solution; // streamK defaults to 0 + auto problem = makeGemmProblem(512, 512, 512); + auto device = makeDevice(_MI350_CHIP_ID, _SPX_CU, "mi350spx"); + + auto d = solution.computeStreamKDecisions(problem, device); + EXPECT_EQ(d.streamKMode, 0); + EXPECT_FALSE(d.isDynamic); + EXPECT_FALSE(d.dpOnly); + EXPECT_EQ(d.skGrid, 0u); + EXPECT_EQ(d.finalGrid, 0u); + EXPECT_EQ(d.selectedGrid, 0u); +} + +// --------------------------------------------------------------------------- +// The printed summary is well-formed and reports the key fields, including the +// new selected-vs-final grid attribution. Exercises the formatting path. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, PrintSummaryEmitsFields) +{ + AnalyticalEnv env; + ContractionSolution solution; + solution.kernelName = "test_streamk_kernel"; + initStreamKSolution(solution, 4); + + auto problem = makeGemmProblem(4096, 4224, 64); + problem.setWorkspaceSize(std::numeric_limits::max()); + + auto d = solution.computeStreamKDecisions(problem, env.device); + std::ostringstream os; + solution.printStreamKLaunchSummary(os, problem, d); + const std::string line = collapseSpaces(os.str()); + + // Deeply-indented multi-line labeled block. The leading token is preserved + // verbatim, and the labeled fields carry the same information as before, now + // as aligned "key = value" pairs (whitespace-collapsed for matching). + EXPECT_NE(line.find("LAUNCH SUMMARY"), std::string::npos); + EXPECT_NE(line.find("test_streamk_kernel"), std::string::npos); + EXPECT_NE(line.find("reduction = tree"), std::string::npos); + // SK4 is unconditionally dynamic -> mode line reports it and the work-queue + // line carries the real per-XCD counts (not NA). + EXPECT_NE(line.find("isDynamic = yes"), std::string::npos); + EXPECT_NE(line.find("selected = "), std::string::npos); + EXPECT_NE(line.find("final = "), std::string::npos); + EXPECT_NE(line.find("changedBy = "), std::string::npos); + EXPECT_NE(line.find("source = "), std::string::npos); + EXPECT_NE(line.find("numQueues(NUM_XCD) = 8"), std::string::npos); + // Section headers are present on their own lines (multi-line block). + EXPECT_NE(line.find("mode:"), std::string::npos); + EXPECT_NE(line.find("grid:"), std::string::npos); + EXPECT_NE(line.find("work-queue:"), std::string::npos); +} + +// --------------------------------------------------------------------------- +// The printed summary attributes the grid change to the workspace-DP fallback +// when it fires (selected vs final are both reported and differ). +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, PrintSummaryReportsFallbackGridChange) +{ + AnalyticalEnv env; + ContractionSolution solution; + solution.kernelName = "test_streamk_fallback"; + initStreamKSolution(solution, 4); + + auto problem = makeGemmProblem(4096, 4224, 64); + problem.setWorkspaceSize(0); // force DP fallback + + auto d = solution.computeStreamKDecisions(problem, env.device); + std::ostringstream os; + solution.printStreamKLaunchSummary(os, problem, d); + const std::string line = collapseSpaces(os.str()); + + EXPECT_NE(line.find("changedBy = workspaceDP"), std::string::npos); + EXPECT_NE(line.find("source = workspaceDP(runtime)"), std::string::npos); + EXPECT_NE(line.find("workspaceDPFallback = yes"), std::string::npos); +} + +// --------------------------------------------------------------------------- +// Work-queue fields are per-XCD dynamic-path only. On the SK5-static (SK3) +// sub-path (isDynamic == false) the summary must print the work-queue line as +// "NA (work-queues not used)" instead of a misleading numQueues value, while +// still reporting all the StreamK-wide fields (mode/grid/tiles/workspace). +// This is display-only: the struct still carries d.numQueues. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, PrintSummaryNaWorkQueueWhenNotDynamic) +{ + AnalyticalEnv env; + ContractionSolution solution; + solution.kernelName = "test_streamk_static"; + initStreamKSolution(solution, 5); + + auto problem = makeGemmProblem(4096, 4224, 64); + problem.setWorkspaceSize(std::numeric_limits::max()); + problem.setParams().setStreamKTileSchedulingMode(0); // OFF -> static (SK3), not dynamic + + auto d = solution.computeStreamKDecisions(problem, env.device); + ASSERT_FALSE(d.isDynamic) << "SK5-OFF must resolve to the static (non-work-queue) path"; + // The struct still holds the baked count; only the DISPLAY is NA'd. + EXPECT_EQ(d.numQueues, 8u); + + std::ostringstream os; + solution.printStreamKLaunchSummary(os, problem, d); + const std::string line = collapseSpaces(os.str()); + + // Non-dynamic -> work-queue fields are NA, and the misleading numeric + // per-XCD field is NOT printed. The header now sits on its own line, with the + // NA note indented beneath it. + EXPECT_NE(line.find("isDynamic = no"), std::string::npos); + EXPECT_NE(line.find("work-queue:"), std::string::npos); + EXPECT_NE(line.find("NA (work-queues not used)"), std::string::npos); + EXPECT_EQ(line.find("numQueues(NUM_XCD)"), std::string::npos) + << "static path must not print a per-XCD work-queue count"; + EXPECT_EQ(line.find("dynamicPartialsSlots"), std::string::npos) + << "dynamicPartialsSlots is a dynamic-path-only field"; + // StreamK-wide fields are still reported (not NA'd). + EXPECT_NE(line.find("reduction = "), std::string::npos); + EXPECT_NE(line.find("selected = "), std::string::npos); + EXPECT_NE(line.find("tiles:"), std::string::npos); + EXPECT_NE(line.find("workspace:"), std::string::npos); + EXPECT_NE(line.find("fallbacks:"), std::string::npos); +} From 1990f4feefa92078846831ed133932c4dfed2adf Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Tue, 28 Jul 2026 15:10:12 +0000 Subject: [PATCH 3/8] docs(tensilelite): make StreamK comments self-contained --- .../Tests/unit/test_streamk_queue_index.py | 31 +++++----- .../include/Tensile/ContractionSolution.hpp | 9 ++- .../tensilelite/src/ContractionSolution.cpp | 31 +++++----- .../tests/StreamKLaunchSummary_test.cpp | 59 +++++++------------ 4 files changed, 55 insertions(+), 75 deletions(-) diff --git a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_queue_index.py b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_queue_index.py index c501617d5c6b..70b69b8b89f5 100644 --- a/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_queue_index.py +++ b/projects/hipblaslt/tensilelite/Tensile/Tests/unit/test_streamk_queue_index.py @@ -14,7 +14,7 @@ Zero-SGPR carrier reuse: the raw rank is snapshotted into the ALREADY-allocated persistent ``StreamKTileIdx`` slot -- provably dead in the [prologue, queue-read) window -- instead of a dedicated ``StreamKQueue`` SGPR, which overflowed the SGPR -file on tuned high-register SKXCC kernels. The fix now covers TWO disjoint remap +file on tuned high-register SKXCC kernels. The fix covers TWO disjoint remap regimes: * WorkGroupMappingXCC == -1 (dynamic auto-WGM), and * StreamKXCCMapping != 0 with WorkGroupMappingXCC > 1 (SKXCC). @@ -24,7 +24,7 @@ *real* KernelWriter / KernelWriterAssembly source via the AST so the ordering and gating assertions track the actual code. -Invariants pinned (each fails against the pre-fix code -- see per-test notes): +Invariants pinned (see per-test notes): * Both dynamic auto-WGM and SKXCC (WGMXCC>1) queue indices are ``StreamKTileIdx & (numQueues-1)`` -- a single mask of the raw-rank carrier, never the post-remap StreamKIdx shifts. @@ -166,10 +166,9 @@ class TestRawRankQueueIndex: @pytest.mark.parametrize("streamk", [4, 5]) @pytest.mark.parametrize("wgmXCC,skxcc", RAW_REGIMES) def test_queue_index_masks_raw_rank_carrier(self, streamk, wgmXCC, skxcc): - # Pre-fix emitted shr/shl/sub of StreamKIdx and NO SAndB32 -- so both the - # "single SAndB32" and "references carrier" assertions fail on it. The - # SKXCC regimes additionally fail against #10008 (which scoped the fix to - # WGMXCC == -1 only), which is exactly what this extension covers. + # For every raw-rank regime (dynamic auto-WGM and SKXCC with WGMXCC > 1), + # the queue index must be a single mask of the raw-rank carrier, not the + # StreamKIdx shr/shl/sub derivation. items = _emit_queue_index(streamk, wgmXCC=wgmXCC, skxcc=skxcc) ands = [i for i in items if isinstance(i, SAndB32)] assert len(ands) == 1, "raw-rank queue index must be a single mask op" @@ -182,9 +181,9 @@ def test_queue_index_masks_raw_rank_carrier(self, streamk, wgmXCC, skxcc): @pytest.mark.parametrize("streamk", [4, 5]) @pytest.mark.parametrize("wgmXCC,skxcc", RAW_REGIMES) def test_queue_index_does_not_use_post_remap_streamkidx(self, streamk, wgmXCC, skxcc): - # The whole point of the fix: on a raw-rank path the queue index must NOT - # come from the remapped StreamKIdx (StreamKIdx % numQueues). Pre-fix code - # emitted exactly the shift/shift/sub of StreamKIdx, so this fails on it. + # On a raw-rank path the queue index must NOT come from the remapped + # StreamKIdx (StreamKIdx % numQueues), so neither StreamKIdx nor its + # shift/shift/sub derivation may appear. items = _emit_queue_index(streamk, wgmXCC=wgmXCC, skxcc=skxcc) assert not any(_refs_sgpr(i, "StreamKIdx") for i in items), ( "the raw-rank queue index must not reference the post-remap StreamKIdx" @@ -196,10 +195,11 @@ def test_queue_index_does_not_use_post_remap_streamkidx(self, streamk, wgmXCC, s @pytest.mark.parametrize("streamk", [4, 5]) @pytest.mark.parametrize("wgmXCC,skxcc", RAW_REGIMES) def test_queue_index_never_uses_dead_streamkqueue(self, streamk, wgmXCC, skxcc): - # StreamKQueue is gone: the raw rank lives in the reused StreamKTileIdx. + # The queue index never references a dedicated StreamKQueue SGPR; the raw + # rank lives in the reused StreamKTileIdx carrier. items = _emit_queue_index(streamk, wgmXCC=wgmXCC, skxcc=skxcc) assert not any(_refs_sgpr(i, "StreamKQueue") for i in items), ( - "StreamKQueue was removed; the carrier is the reused StreamKTileIdx" + "no dedicated StreamKQueue SGPR; the carrier is the reused StreamKTileIdx" ) @@ -338,9 +338,8 @@ def _is_sk_raw_rank_guard(test) -> bool: # =========================================================================== -# 4. Both dynamic graWorkGroup paths route through the shared _emitQueueIndex. -# Pre-fix each inlined its own StreamKIdx shift/shift/sub, so the shared -# helper call is absent -> this fails against the pre-fix source. +# 4. Both dynamic graWorkGroup paths route through the shared _emitQueueIndex, +# rather than each inlining its own StreamKIdx shift/shift/sub derivation. # =========================================================================== class TestSharedHelperRouting: @pytest.mark.parametrize( @@ -365,8 +364,8 @@ def _src(self) -> str: return _source_of(KernelWriterAssembly.defineAndResources) def test_snapshot_mov_targets_carrier(self): - # Pre-fix there was no snapshot; #10008 targeted StreamKQueue. The carrier - # reuse retargets it to the already-allocated StreamKTileIdx slot. + # The raw-rank snapshot targets the already-allocated StreamKTileIdx slot, + # not a dedicated StreamKQueue SGPR. src = self._src() assert self._SNAP in src, ( "the raw pre-remap launch id must be snapshotted into %s" % _CARRIER diff --git a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp index 00348820d5ca..6bcab9c5a907 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp @@ -270,11 +270,10 @@ namespace TensileLite size_t idealWorkspaceBytes = 0; // partials(+queue) bytes the launch wanted (pre fit check) size_t givenWorkspaceBytes = 0; // problem.workspaceSize() // Informational skTiles*skSplit slot count for the dynamic path, computed - // LOCALLY here for reporting only. NOTE: on this (#10008) base the actual - // partials-workspace sizing/allocation is driven by tiles%grid (see - // computeStreamKDecisions / solve), NOT by this value; it is reported so - // the invariant gap ("dynamic should reserve iff dynamicSlots>0") is - // observable without pulling in #9415's allocation behaviour change. + // LOCALLY here for reporting only. The actual partials-workspace + // sizing/allocation is driven by tiles%grid (see computeStreamKDecisions / + // solve), NOT by this value; it is reported so the relationship between + // the dynamic slot count and workspace reservation is observable. size_t dynamicPartialsSlots = 0; size_t numQueues = 0; // baked per-XCD work-queue count (NUM_XCD), 0 if unknown diff --git a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp index dcaf4992615c..d94cd56303a9 100644 --- a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp +++ b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp @@ -3317,9 +3317,9 @@ namespace TensileLite auto tiles = problem.getNumTiles(sizeMapping, 1); // Single source of truth: compute the StreamK launch decisions once // (also consumed by the diagnostic launch summary and unit tests) and - // consume them here. computeStreamKDecisions() reproduces exactly the - // reduction / grid / workspace-DP-fallback logic that previously lived - // inline, reusing the same helpers, so launch behaviour is unchanged. + // consume them here. computeStreamKDecisions() applies exactly the + // reduction / grid / workspace-DP-fallback logic used at launch, + // reusing the same helpers, so the decisions match what is launched. // It has no side effects, so computing it before the dynamic-queue // guard below cannot change observable behaviour (the guard still // throws before any kernel-arg packing / kernel launch). @@ -3356,10 +3356,9 @@ namespace TensileLite "per-XCD queue count; this kernel is unsupported here. " "Select a non-work-stealing solution instead."); } - // Consume the reduction / grid decided above. These are exactly the - // values the (pre-#9415) inline block computed -- computeStreamKDecisions - // mirrors that logic (including the workspace-insufficient DP fallback), - // reusing the same helpers, so launch behaviour is unchanged. + // Consume the reduction / grid decided above. computeStreamKDecisions + // applies the same logic (including the workspace-insufficient DP + // fallback) using the same helpers, so these match what is launched. sk.reduction = skDecisions.reduction; sk.grid = skDecisions.skGrid; sk.streamKTileSchedulingMode = problem.getParams().streamKTileSchedulingMode(); @@ -4347,13 +4346,11 @@ namespace TensileLite // mirror the kernel-arg packing in makeArgs(). It reuses the existing helpers // rather than re-deriving, so the summary cannot drift from the real launch. // - // NOTE (this branch = #10008, WITHOUT #9415): the partials-workspace guard and - // sizing below intentionally reproduce the PRE-#9415 behaviour -- reserve iff + // The partials-workspace guard and sizing below reserve iff // (reduction==parallel || tiles%grid!=0), sized as partialTileSize(grid) (+ the - // per-XCD work-queue region for the dynamic path). It does NOT use #9415's - // dynamicPartialsSlots-driven sizing. The dynamicPartialsSlots FIELD is still - // populated (skTiles*skSplit, computed locally) purely for reporting so the - // "dynamic should reserve iff dynamicSlots>0" invariant gap is observable. + // per-XCD work-queue region for the dynamic path); the reservation does not + // depend on dynamicPartialsSlots. The dynamicPartialsSlots field is still + // populated (skTiles*skSplit, computed locally) purely for reporting. StreamKDecisions ContractionSolution::computeStreamKDecisions(Problem const& problem, Hardware const& hardware) const @@ -4414,9 +4411,9 @@ namespace TensileLite d.numQueues = streamKBakedQueueCount(hardware); d.givenWorkspaceBytes = problem.workspaceSize(); - // Workspace / DP fallback -- mirrors the PRE-#9415 guard in solve(): - // reserve iff (reduction==parallel || tiles%grid!=0), sized by grid (not by - // dynamicSlots). No #9415 behaviour is pulled in here. + // Workspace / DP fallback -- mirrors the guard in solve(): reserve iff + // (reduction==parallel || tiles%grid!=0), sized by grid (not by + // dynamicSlots). size_t idealWorkspace = 0; bool needPartials = false; if(grid > 0 @@ -4490,7 +4487,7 @@ namespace TensileLite // Informational only (see field doc): skTiles*skSplit slot count for the // dynamic path, computed LOCALLY here. It does NOT feed the allocation - // guard above, so no #9415 sizing behaviour is introduced. + // guard above. d.dynamicPartialsSlots = isDynamic ? static_cast(d.skTiles) * d.skSplit : 0; d.partialsPresent = d.skTiles > 0; diff --git a/projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp b/projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp index d6c622074259..378f297a5e9c 100644 --- a/projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp +++ b/projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp @@ -9,13 +9,11 @@ // launch-parameter path -- without needing a GPU. Host-only: mock AMDGPU and // hip::HipAMDGPU devices, no device library required. // -// This file lives on the #10008-only base (StreamK per-XCD work-queue counter -// self-reset fix), WITHOUT #9415 (dynamic partials-workspace sizing). The -// partials-workspace behaviour asserted below is therefore the PRE-#9415 -// behaviour: the dynamic (SK4 / SK5-dynamic) path reserves the partials region -// based on tiles%grid divisibility, NOT on the skTiles*skSplit slot count. The -// invariant-gap tests near the bottom pin that CURRENT behaviour AND document -// the DESIRED invariant (reserve iff dynamicSlots>0) that #9415 implements. +// The partials-workspace behaviour asserted below is that the dynamic (SK4 / +// SK5-dynamic) path reserves the partials region based on tiles%grid +// divisibility, NOT on the skTiles*skSplit slot count. The tests near the bottom +// pin that relationship between dynamicPartialsSlots, tiles%grid divisibility, +// and whether a partials workspace is reserved. #include #include @@ -395,22 +393,19 @@ TEST(StreamKLaunchSummaryTest, DpOnlySourceDistinguishesParamVsRuntime) } // --------------------------------------------------------------------------- -// INVARIANT GAP (#9415 territory) -- pinned on the #10008-only base. +// Dynamic-path partials-workspace reservation rule. // -// Desired invariant (implemented by PR #9415): a DYNAMIC (SK4 / SK5-dynamic) -// launch should reserve the partials workspace IFF dynamicPartialsSlots -// (skTiles*skSplit) > 0, because that path indexes the workspace by tile-split -// slot, not by grid position. -// -// CURRENT behaviour on this #10008 base: the dynamic path reserves the partials -// workspace based on tiles%grid divisibility (the SK3 rule), independent of -// dynamicPartialsSlots. The three tests below bracket that gap. +// The dynamic (SK4 / SK5-dynamic) path reserves the partials workspace based on +// tiles%grid divisibility (the same rule the static SK3 path uses), independent +// of dynamicPartialsSlots (skTiles*skSplit). The three tests below pin that +// behaviour across the combinations of dynamicPartialsSlots (0 vs >0) and +// tiles%grid (==0 vs !=0). // --------------------------------------------------------------------------- // Case A: dynamicSlots == 0 (no split stream-k tiles) AND tiles % grid != 0. -// CURRENT: workspace IS reserved (because tiles%grid!=0) even though the dynamic -// packing produced no partial tiles. DESIRED: it should NOT reserve here. -TEST(StreamKLaunchSummaryTest, DynamicNoSlotsButIndivisible_ReservesWorkspace_GAP) +// Workspace IS reserved (because tiles%grid!=0) even though the dynamic packing +// produced no partial tiles. +TEST(StreamKLaunchSummaryTest, DynamicNoSlotsButIndivisible_ReservesWorkspace) { ContractionSolution solution; initStreamKSolution(solution, 4); // dynamic @@ -430,19 +425,15 @@ TEST(StreamKLaunchSummaryTest, DynamicNoSlotsButIndivisible_ReservesWorkspace_GA EXPECT_EQ(d.skTiles, 0u) << "no override -> dynamic packing produces no split tiles"; EXPECT_EQ(d.dynamicPartialsSlots, 0u) << "skTiles*skSplit == 0"; - // CURRENT (#10008) behaviour: reserves anyway because tiles%grid != 0. + // Workspace is reserved because tiles%grid != 0, independent of the zero + // dynamic slot count. EXPECT_TRUE(d.workspaceAllocated); EXPECT_GT(d.requiredWorkspaceBytes, 0u); EXPECT_EQ(d.requiredWorkspaceBytes, solution.requiredWorkspaceSize(problem, device)); - // DESIRED invariant (PR #9415): with dynamicPartialsSlots==0 the dynamic path - // needs no partials region, so ideally this would be: - // EXPECT_FALSE(d.workspaceAllocated); - // EXPECT_EQ(d.requiredWorkspaceBytes, 0u); - // Pinned here as the observed gap until #9415 lands on this base. } // Case B (complement): dynamicSlots == 0 AND tiles % grid == 0. -// CURRENT and DESIRED agree: no workspace reserved. +// No workspace reserved. TEST(StreamKLaunchSummaryTest, DynamicNoSlotsAndDivisible_NoWorkspace) { ContractionSolution solution; @@ -466,10 +457,9 @@ TEST(StreamKLaunchSummaryTest, DynamicNoSlotsAndDivisible_NoWorkspace) } // Case C: dynamicSlots > 0 (skTiles override) BUT tiles % grid == 0. -// CURRENT (#10008): NO workspace reserved -- this is the OOB-write bug #9415 -// fixes, because the kernel indexes up to skTiles*skSplit slots. DESIRED -// (PR #9415): reserve partialTileSize(max(dynamicSlots, grid)) + queue region. -TEST(StreamKLaunchSummaryTest, DynamicSlotsPositiveButDivisible_NoWorkspace_GAP) +// No workspace is reserved, because tiles%grid==0 gates the partials reservation +// independent of the positive dynamic slot count. +TEST(StreamKLaunchSummaryTest, DynamicSlotsPositiveButDivisible_NoWorkspace) { ContractionSolution solution; initStreamKSolution(solution, 4); // dynamic @@ -495,16 +485,11 @@ TEST(StreamKLaunchSummaryTest, DynamicSlotsPositiveButDivisible_NoWorkspace_GAP) // totalItems = (tiles - skTiles) + skTiles*skSplit EXPECT_EQ(d.totalItems, (d.tiles - d.skTiles) + d.skTiles * d.skSplit); - // CURRENT (#10008) behaviour: NO workspace reserved despite dynamicSlots>0, - // because tiles%grid==0 gates the SK3-style reservation. requiredWorkspaceSize - // agrees (also 0). + // No workspace reserved despite dynamicSlots>0, because tiles%grid==0 gates + // the partials reservation. requiredWorkspaceSize agrees (also 0). EXPECT_FALSE(d.workspaceAllocated); EXPECT_EQ(d.requiredWorkspaceBytes, 0u); EXPECT_EQ(d.requiredWorkspaceBytes, solution.requiredWorkspaceSize(problem, device)); - // DESIRED invariant (PR #9415): dynamicPartialsSlots>0 must reserve, i.e. - // EXPECT_TRUE(d.workspaceAllocated); - // EXPECT_GT(d.requiredWorkspaceBytes, 0u); - // Pinned here as the observed gap until #9415 lands on this base. } // --------------------------------------------------------------------------- From d545c9f402ba920990c06974eb0a78320607ffae Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Tue, 28 Jul 2026 15:51:08 +0000 Subject: [PATCH 4/8] docs(tensilelite): annotate StreamK launch summary field provenance Replace the general "single source of truth / cannot drift / observational" prose on StreamKDecisions with precise per-field provenance: each field now states whether it is available (read from the value the launch uses, or the same helper/param solve() uses) or recomputed (re-derived by mirroring the makeArgs/solve() sizing and packing logic). --- .../include/Tensile/ContractionSolution.hpp | 130 ++++++++++-------- 1 file changed, 76 insertions(+), 54 deletions(-) diff --git a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp index 6bcab9c5a907..463aa5f9f506 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp @@ -216,71 +216,91 @@ namespace TensileLite size_t globalAccumulation = 0; }; - // Observational snapshot of the StreamK launch-parameter DECISIONS made for - // one (solution, problem, hardware) triple. Populated by - // ContractionSolution::computeStreamKDecisions(), which is the single source - // of truth that solve() itself consumes to fill StreamKSettings -- so the - // diagnostic launch summary can never drift from the real launch. Purely - // observational: computing this changes no launch behaviour. See + // Snapshot of the StreamK launch-parameter DECISIONS for one + // (solution, problem, hardware) triple. Populated by + // ContractionSolution::computeStreamKDecisions() and consumed by solve(). + // Each field's provenance is annotated inline as one of: + // available = read from the value the launch uses, or from the same real + // helper/param solve() uses, so it reflects the launch exactly. + // recomputed = re-derived here by mirroring the makeArgs/solve() sizing and + // packing logic (a faithful re-derivation, not a stored launched + // value). + // Computing or printing this report does not change launch behaviour. See // Debug::printStreamKLaunchSummary() (TENSILE_DB bit 0x200000). struct StreamKDecisions { // --- Mode --- - int streamKMode = 0; // sizeMapping.streamK (0 = not StreamK, else 3/4/5) - bool effectiveDynamic = false; // SK5 resolved to the dynamic (SK4) sub-path - bool isDynamic = false; // SK4, or SK5 resolved dynamic (work-queue path) + // available: sizeMapping.streamK, the mode solve() uses (0 = not StreamK, else 3/4/5). + int streamKMode = 0; + // available: streamK5EffectiveDynamic(), the same helper solve()'s grid path uses + // (SK5 resolved to the dynamic SK4 sub-path). + bool effectiveDynamic = false; + // recomputed: derived from streamKMode + effectiveDynamic (SK4, or SK5 resolved dynamic). + bool isDynamic = false; // --- Reduction --- - origami::reduction_t reduction = origami::reduction_t::tree; // final (post-fallback) + // available: solve() wires this exact value into sk.reduction -- it is the final + // (post workspace-DP fallback) reduction the launch uses. + origami::reduction_t reduction = origami::reduction_t::tree; // --- Grid / tiles / split --- - size_t tiles = 0; // problem.getNumTiles() - // Grid selected by getSKGridImpl's config/CU/override logic BEFORE any of - // the "reset to tiles" fallbacks (tree-fixup bounds / workspace-DP) fire. - // This is the grid StreamK actually WANTED for this launch. + // available: problem.getNumTiles(sizeMapping, 1), the same call solve() makes. + size_t tiles = 0; + // available: getSKGridImpl out-param -- grid selected by its config/CU/override + // logic BEFORE the "reset to tiles" fallbacks (tree-fixup bounds / workspace-DP). size_t selectedGrid = 0; - // Grid returned by getSKGridImpl -- i.e. AFTER the tree-fixup-bounds - // fallback (which may reset it to tiles) but BEFORE the workspace-DP - // fallback in solve(). Kept as a diagnostic intermediate. + // available: getSKGridImpl return -- AFTER the tree-fixup-bounds fallback but + // BEFORE the workspace-DP fallback in solve(). Diagnostic intermediate. size_t skGridPreFallback = 0; - // FINAL effective grid actually used at launch, after ALL fallbacks - // (fixed-grid override, tree-fixup-bounds, workspace-insufficient DP). - // Wired directly from solve()'s StreamKSettings::grid so it can never - // drift from the real launch. + // available: wired from sk.grid, the FINAL grid solve() launches with (after all + // fallbacks: fixed-grid override, tree-fixup-bounds, workspace-insufficient DP). size_t finalGrid = 0; - // Backward-compatible alias of finalGrid (the grid solve() launches with). + // available: alias of finalGrid (the grid solve() launches with). size_t skGrid = 0; - size_t skTiles = 0; // number of stream-k (partial) tiles packed by makeArgs - size_t skSplit = 0; // k-split factor (>=1) - size_t totalItems = 0; // (tiles - skTiles) + skTiles*skSplit + // recomputed: mirrors the makeArgs tile packing (stream-k / partial tile count). + size_t skTiles = 0; + // recomputed: mirrors the makeArgs k-split packing (>=1). + size_t skSplit = 0; + // recomputed: mirrors the makeArgs packing, (tiles - skTiles) + skTiles*skSplit. + size_t totalItems = 0; // --- DP-only --- - // These three flags DISTINGUISH the source of a data-parallel-only launch: - // forceDPOnly -> sizeMapping.streamKForceDPOnly compile-time param - // streamKDP -> TENSILE_STREAMK_DATA_PARALLEL debug override - // workspaceDPFallbackFired (below) -> runtime workspace-insufficient - bool dpOnly = false; // any of the three above fired - bool forceDPOnly = false; // sizeMapping.streamKForceDPOnly (param) - bool streamKDP = false; // TENSILE_STREAMK_DATA_PARALLEL debug override + // The three flags below distinguish the source of a data-parallel-only launch: + // forceDPOnly -> sizeMapping.streamKForceDPOnly compile-time param + // streamKDP -> TENSILE_STREAMK_DATA_PARALLEL debug override + // workspaceDPFallbackFired -> runtime workspace-insufficient (below) + // recomputed: OR of the three DP triggers above. + bool dpOnly = false; + // available: sizeMapping.streamKForceDPOnly param. + bool forceDPOnly = false; + // available: Debug::useStreamKDataParrallel() (TENSILE_STREAMK_DATA_PARALLEL). + bool streamKDP = false; // --- Workspace / partials --- - bool partialsPresent = false; // skTiles > 0 - bool workspaceAllocated = false; // the partials/queue block is taken and fits - size_t requiredWorkspaceBytes = 0; // SK partials(+queue) bytes actually reserved (0 if none) - size_t idealWorkspaceBytes = 0; // partials(+queue) bytes the launch wanted (pre fit check) - size_t givenWorkspaceBytes = 0; // problem.workspaceSize() - // Informational skTiles*skSplit slot count for the dynamic path, computed - // LOCALLY here for reporting only. The actual partials-workspace - // sizing/allocation is driven by tiles%grid (see computeStreamKDecisions / - // solve), NOT by this value; it is reported so the relationship between - // the dynamic slot count and workspace reservation is observable. + // recomputed: skTiles > 0. + bool partialsPresent = false; + // recomputed: mirrors the sizing guard (requiredWorkspaceBytes > 0). + bool workspaceAllocated = false; + // recomputed: mirrors the sizing guard -- partials(+queue) bytes reserved (0 if none/fallback). + size_t requiredWorkspaceBytes = 0; + // recomputed: mirrors the sizing guard -- partials(+queue) bytes wanted (pre fit check). + size_t idealWorkspaceBytes = 0; + // available: problem.workspaceSize(). + size_t givenWorkspaceBytes = 0; + // recomputed: skTiles*skSplit slot count for the dynamic path (informational; does + // NOT feed the allocation guard, which is sized by grid). size_t dynamicPartialsSlots = 0; - size_t numQueues = 0; // baked per-XCD work-queue count (NUM_XCD), 0 if unknown + // available: streamKBakedQueueCount() -- baked per-XCD work-queue count (NUM_XCD), + // 0 if unknown. + size_t numQueues = 0; // --- Fallbacks that fired (each can change selectedGrid -> finalGrid) --- - bool workspaceDPFallbackFired = false; // idealWorkspace > given -> tree + grid=tiles - bool treeBoundsFallbackFired = false; // 24-bit tree-fixup bounds -> grid=tiles - bool fixedGridUsed = false; // AMDGPU skFixedGrid override applied + // recomputed: mirrors solve()'s workspace fallback (idealWorkspace > given -> tree + grid=tiles). + bool workspaceDPFallbackFired = false; + // available: getSKGridImpl out-param (24-bit tree-fixup bounds -> grid=tiles). + bool treeBoundsFallbackFired = false; + // available: getSKGridImpl out-param (AMDGPU skFixedGrid override applied). + bool fixedGridUsed = false; }; /** * Represents a single kernel or set of kernels that can perform a single @@ -464,14 +484,16 @@ namespace TensileLite size_t partialTileSize(size_t skGrid) const; // Compute the StreamK launch-parameter DECISIONS for this solution on the - // given problem/hardware. This is the single source of truth for the - // reduction strategy, grid, and workspace/DP fallbacks that solve() - // consumes to populate StreamKSettings, and it is directly callable from - // unit tests. It reuses the existing helpers (streamK5EffectiveDynamic, - // getSKReduction, getSKGrid, partialTileSize) so the returned snapshot - // reflects the real launch, not a re-derivation. Purely observational -- - // calling it changes no launch behaviour. Returns an all-default snapshot - // (streamKMode==0) for non-StreamK solutions. + // given problem/hardware. solve() consumes the reduction strategy, grid, + // and workspace/DP fallbacks from here to populate StreamKSettings; it is + // also directly callable from unit tests. Some fields are read from the + // values the launch uses (or the same helpers solve() uses: + // streamK5EffectiveDynamic, getSKReduction, getSKGrid, partialTileSize); + // others are re-derived here by mirroring the makeArgs/solve() sizing and + // packing logic. Each StreamKDecisions field documents its own provenance + // (available vs recomputed). Calling it changes no launch behaviour. + // Returns an all-default snapshot (streamKMode==0) for non-StreamK + // solutions. StreamKDecisions computeStreamKDecisions(Problem const& problem, Hardware const& hardware) const; From 9a7dadab176137ada50e5908e727befcfb894f36 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Thu, 27 Aug 2026 21:09:52 +0000 Subject: [PATCH 5/8] test(tensilelite): cover the StreamK cluster-multicast grid clamp The ForceDPOnly cluster-multicast clamp in getSKGridImpl() -- SK3 + streamKForceDPOnly + clusterDim.x*clusterDim.y > 1 resetting the grid to tiles -- had no unit coverage. Add four cases: * the clamp resets the grid to tiles and is reflected in the snapshot, with getSKGrid() cross-checked against the reported final grid; * the three-way conjunction is a real guard (SK4, non-force-DP SK3, and a z-only cluster each leave the grid alone); * the printed summary names the clamp, credits it for the selected-vs-final grid change, and surfaces preFallback; * changedBy credits the clamp over an skFixedGrid override that the clamp subsequently overwrote, so the report never names a grid the launch did not use. The clamp reads only sizeMapping, so all four run on the existing mock AMDGPU with no analytical hardware and no gfx1250 device. Also tighten a few existing assertions and comments: * replace a totalItems assertion that restated the implementation character-for-character with the concrete literals the fully determined fixture produces (1792); * check all four fallback flags in the workspace-DP attribution test now that clusterDPGridClamped exists; * turn an if(workspaceAllocated)-guarded workspace sizing check into ASSERT_TRUE plus an unconditional check so it cannot silently vanish; * pin d.tiles and d.skGridPreFallback to literals rather than only to the same helpers the implementation calls; * correct the ForceDPOnly comment, which claimed the param never resets the grid to tiles -- true only because the fixture leaves clusterDim at 1; * restate the dynamic-path workspace guard in full (parallel OR tiles%grid != 0) and note why the fourth combination is not tested; * rewrite the comments that read as a diff against an earlier revision, and rename NonStreamKProducesEmptySnapshot -> NonStreamKProducesInertSnapshot and Sk3StaticPartialTiles -> Sk3StaticPartialTilesReserveWorkspace. --- .../tests/StreamKLaunchSummary_test.cpp | 301 ++++++++++++++++-- 1 file changed, 275 insertions(+), 26 deletions(-) diff --git a/projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp b/projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp index 378f297a5e9c..5b1ca89af3f3 100644 --- a/projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp +++ b/projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp @@ -101,11 +101,11 @@ namespace hip::HipAMDGPU device; }; - // The summary now uses a deeply-indented "key = value" layout whose column - // widths are chosen per-section for alignment. Collapsing runs of spaces to a - // single space lets the assertions below match on the stable tokens - // ("changedBy = ...", "source = ...", etc.) without pinning exact whitespace. - // Newlines are preserved so section boundaries still matter. + // The summary is a deeply-indented "key = value" block whose column widths are + // chosen per-section for alignment. Collapsing runs of spaces to a single space + // lets the assertions below match on the stable tokens ("changedBy = ...", + // "source = ...", etc.) without pinning exact column widths. Newlines are left + // intact so a token can never be matched across a line break. std::string collapseSpaces(std::string const& s) { std::string out; @@ -155,8 +155,10 @@ TEST(StreamKLaunchSummaryTest, SnapshotMatchesHelpersForDynamicPartialTiles) EXPECT_TRUE(d.isDynamic); EXPECT_EQ(d.reduction, origami::reduction_t::tree); EXPECT_EQ(d.tiles, tiles); + EXPECT_EQ(d.tiles, 1056u) << "4096/128 * 4224/128 = 32 * 33"; // getSKGrid() reproduces the pre-fallback grid the snapshot records. EXPECT_EQ(d.skGridPreFallback, grid); + EXPECT_EQ(d.skGridPreFallback, 256u) << "min(tiles, cuCount * CUOccupancy) = min(1056, 256)"; ASSERT_NE(tiles % grid, 0u) << "test needs partial tiles"; // No fallback fires here, so selected == pre-fallback == final launch grid. @@ -218,16 +220,18 @@ TEST(StreamKLaunchSummaryTest, Sk5OffResolvesStaticSk3) EXPECT_FALSE(d.isDynamic) << "SK5-OFF must take the static (SK3) sub-path"; EXPECT_EQ(d.numQueues, 8u); // baked count still reported (informational) // SK3-static: no per-XCD work-queue region in the workspace it reserves. - if(d.workspaceAllocated) - EXPECT_EQ(d.requiredWorkspaceBytes, solution.partialTileSize(d.skGrid)) - << "static SK3 workspace = partialTileSize(grid), no work-queue region"; + ASSERT_TRUE(d.workspaceAllocated) + << "scenario must actually reserve a workspace, otherwise the sizing check below " + "would silently assert nothing"; + EXPECT_EQ(d.requiredWorkspaceBytes, solution.partialTileSize(d.skGrid)) + << "static SK3 workspace = partialTileSize(grid), no work-queue region"; } // --------------------------------------------------------------------------- // SK3 static with partial tiles: partials present, skTiles>0, workspace>0, // not dynamic, not DP-only. // --------------------------------------------------------------------------- -TEST(StreamKLaunchSummaryTest, Sk3StaticPartialTiles) +TEST(StreamKLaunchSummaryTest, Sk3StaticPartialTilesReserveWorkspace) { ContractionSolution solution; initStreamKSolution(solution, 3); @@ -281,8 +285,10 @@ TEST(StreamKLaunchSummaryTest, ForceDpOnlyIsDpOnlyNoWorkspace) EXPECT_EQ(d.requiredWorkspaceBytes, 0u); EXPECT_FALSE(d.workspaceAllocated); // Force-DP-only launches on exactly the hardware CU count (mirrors - // StreamKForceDPOnlyTest.UsesHardwareCuCount). The param does NOT reset the - // grid to tiles (that is the runtime fallback), so finalGrid == selectedGrid. + // StreamKForceDPOnlyTest.UsesHardwareCuCount). With clusterDim == 1 the param + // alone does not reset the grid to tiles -- only the cluster-multicast clamp + // (clusterDim.x*clusterDim.y > 1) and the runtime workspace-DP fallback do -- + // so finalGrid == selectedGrid here. EXPECT_EQ(d.skGrid, static_cast(_CPX_CU)); EXPECT_EQ(d.finalGrid, static_cast(_CPX_CU)); EXPECT_EQ(d.finalGrid, d.selectedGrid); @@ -343,10 +349,12 @@ TEST(StreamKLaunchSummaryTest, SelectedVsFinalGridAttribution) EXPECT_GT(d.selectedGrid, 0u); EXPECT_EQ(d.finalGrid, d.tiles); EXPECT_NE(d.selectedGrid, d.finalGrid); - // Exactly one of the grid-changing fallbacks is attributed, and it is the - // workspace-DP one for this scenario. + // The workspace-DP fallback is the flag responsible for the change here, and + // the fixed-grid override did not fire. EXPECT_TRUE(d.workspaceDPFallbackFired); EXPECT_FALSE(d.fixedGridUsed); + EXPECT_FALSE(d.treeBoundsFallbackFired); + EXPECT_FALSE(d.clusterDPGridClamped); } // --------------------------------------------------------------------------- @@ -395,11 +403,19 @@ TEST(StreamKLaunchSummaryTest, DpOnlySourceDistinguishesParamVsRuntime) // --------------------------------------------------------------------------- // Dynamic-path partials-workspace reservation rule. // -// The dynamic (SK4 / SK5-dynamic) path reserves the partials workspace based on -// tiles%grid divisibility (the same rule the static SK3 path uses), independent -// of dynamicPartialsSlots (skTiles*skSplit). The three tests below pin that -// behaviour across the combinations of dynamicPartialsSlots (0 vs >0) and -// tiles%grid (==0 vs !=0). +// The dynamic (SK4 / SK5-dynamic) path reserves the partials workspace under the +// same guard the static path uses -- reduction==parallel OR tiles%grid!=0 -- and +// the dynamic path is always tree reduction, so divisibility alone decides. The +// reservation is independent of dynamicPartialsSlots (skTiles*skSplit). The three +// tests below pin that by varying dynamicPartialsSlots (0 vs >0) against tiles%grid +// (==0 vs !=0); the fourth combination (slots>0 and tiles%grid!=0) adds nothing, +// since indivisibility alone already forces the reservation. +// +// All three use a plain mock AMDGPU with no analyticalHardware, so +// streamKBakedQueueCount() is 0 and solve() would reject this device/solution pair +// at its dynamic-queue guard before ever sizing a workspace. Calling +// computeStreamKDecisions() directly is what lets the sizing rule be tested in +// isolation from that rejection. // --------------------------------------------------------------------------- // Case A: dynamicSlots == 0 (no split stream-k tiles) AND tiles % grid != 0. @@ -483,7 +499,9 @@ TEST(StreamKLaunchSummaryTest, DynamicSlotsPositiveButDivisible_NoWorkspace) EXPECT_TRUE(d.partialsPresent) << "override produced split stream-k tiles"; EXPECT_GT(d.dynamicPartialsSlots, 0u) << "skTiles*skSplit > 0"; // totalItems = (tiles - skTiles) + skTiles*skSplit - EXPECT_EQ(d.totalItems, (d.tiles - d.skTiles) + d.skTiles * d.skSplit); + // = (1024 - 256) + 256*4 = 1792. + EXPECT_EQ(d.skSplit, 4u) << "itersPerTile=8, skSplit override 4 -> 4 work items per tile"; + EXPECT_EQ(d.totalItems, 1792u); // No workspace reserved despite dynamicSlots>0, because tiles%grid==0 gates // the partials reservation. requiredWorkspaceSize agrees (also 0). @@ -495,7 +513,7 @@ TEST(StreamKLaunchSummaryTest, DynamicSlotsPositiveButDivisible_NoWorkspace) // --------------------------------------------------------------------------- // Non-StreamK solutions produce an inert (mode==0) snapshot. // --------------------------------------------------------------------------- -TEST(StreamKLaunchSummaryTest, NonStreamKProducesEmptySnapshot) +TEST(StreamKLaunchSummaryTest, NonStreamKProducesInertSnapshot) { ContractionSolution solution; // streamK defaults to 0 auto problem = makeGemmProblem(512, 512, 512); @@ -512,7 +530,7 @@ TEST(StreamKLaunchSummaryTest, NonStreamKProducesEmptySnapshot) // --------------------------------------------------------------------------- // The printed summary is well-formed and reports the key fields, including the -// new selected-vs-final grid attribution. Exercises the formatting path. +// selected-vs-final grid attribution. Exercises the formatting path. // --------------------------------------------------------------------------- TEST(StreamKLaunchSummaryTest, PrintSummaryEmitsFields) { @@ -529,9 +547,9 @@ TEST(StreamKLaunchSummaryTest, PrintSummaryEmitsFields) solution.printStreamKLaunchSummary(os, problem, d); const std::string line = collapseSpaces(os.str()); - // Deeply-indented multi-line labeled block. The leading token is preserved - // verbatim, and the labeled fields carry the same information as before, now - // as aligned "key = value" pairs (whitespace-collapsed for matching). + // Deeply-indented multi-line labeled block. The leading "LAUNCH SUMMARY" token + // and the kernel name are emitted verbatim on the first line; the remaining + // fields are aligned "key = value" pairs (whitespace-collapsed for matching). EXPECT_NE(line.find("LAUNCH SUMMARY"), std::string::npos); EXPECT_NE(line.find("test_streamk_kernel"), std::string::npos); EXPECT_NE(line.find("reduction = tree"), std::string::npos); @@ -601,8 +619,8 @@ TEST(StreamKLaunchSummaryTest, PrintSummaryNaWorkQueueWhenNotDynamic) const std::string line = collapseSpaces(os.str()); // Non-dynamic -> work-queue fields are NA, and the misleading numeric - // per-XCD field is NOT printed. The header now sits on its own line, with the - // NA note indented beneath it. + // per-XCD field is NOT printed. The "work-queue:" header is on its own line + // with the NA note indented beneath it. EXPECT_NE(line.find("isDynamic = no"), std::string::npos); EXPECT_NE(line.find("work-queue:"), std::string::npos); EXPECT_NE(line.find("NA (work-queues not used)"), std::string::npos); @@ -617,3 +635,234 @@ TEST(StreamKLaunchSummaryTest, PrintSummaryNaWorkQueueWhenNotDynamic) EXPECT_NE(line.find("workspace:"), std::string::npos); EXPECT_NE(line.find("fallbacks:"), std::string::npos); } + +// --------------------------------------------------------------------------- +// StreamKForceDPOnly cluster-multicast grid clamp (gfx1250 ClusterDim path). +// +// This clamp is the third "reset the grid to tiles" clamp inside getSKGridImpl, +// running AFTER the tree-fixup-bounds fallback. It fires on the conjunction of +// three SOLUTION-side properties -- SK3, streamKForceDPOnly, and a cluster whose +// x*y extent exceeds 1 -- so the launch runs exactly one work-group per output +// tile (a multicast broadcast, not a K-split). It consults no hardware field, so +// it is reproducible on the same mock AMDGPU the other static-SK3 tests use; no +// gfx1250 device and no analytical hardware are required. streamKForceDPOnly +// also short-circuits getSKReduction() to tree, which keeps the scenario fully +// deterministic. +// +// The tests below pin the resulting REPORT behaviour: the clamp is visible in +// the snapshot, it is what makes selectedGrid differ from finalGrid, and it wins +// the changedBy attribution over the earlier clamps it supersedes. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, ClusterDpMulticastClampsGridToTiles) +{ + ContractionSolution solution; + initStreamKSolution(solution, 3); + solution.sizeMapping.streamKForceDPOnly = 1; + // ClusterDim.x * ClusterDim.y == 4 > 1 -> multicast cluster launch. + solution.sizeMapping.clusterDim = TensileLite::dim3(2, 2, 1); + + // 4096x4224 -> tiles = 32*33 = 1056; the CU-count grid the selection logic + // picks is 64, so the clamp is observable (64 != 1056). + auto problem = makeGemmProblem(4096, 4224, 512); + problem.setWorkspaceSize(std::numeric_limits::max()); + + auto device = makeDevice(_MI350_CHIP_ID, _CPX_CU, "mi350cpx"); + device.skDynamicGrid = 0; + + auto d = solution.computeStreamKDecisions(problem, device); + + ASSERT_EQ(d.streamKMode, 3); + ASSERT_TRUE(d.forceDPOnly); + + // The clamp fired, and it is the thing that moved the grid. + EXPECT_TRUE(d.clusterDPGridClamped); + EXPECT_EQ(d.selectedGrid, static_cast(_CPX_CU)) + << "selection still picks the CU-count grid before the clamp"; + EXPECT_NE(d.selectedGrid, d.finalGrid) << "cluster-DP clamp changed the grid"; + EXPECT_EQ(d.finalGrid, d.tiles) << "cluster multicast launches one work-group per tile"; + EXPECT_EQ(d.skGrid, d.finalGrid); + // The clamp lives inside getSKGridImpl, so it is already reflected in the + // pre-(workspace)-fallback grid. + EXPECT_EQ(d.skGridPreFallback, d.tiles); + + // ...and no OTHER grid-changing fallback is credited for it. + EXPECT_FALSE(d.fixedGridUsed); + EXPECT_FALSE(d.treeBoundsFallbackFired); + EXPECT_FALSE(d.workspaceDPFallbackFired) + << "forceDPOnly suppresses the partials reservation, so no workspace fallback"; + + // getSKGrid() -- the same helper solve() calls -- reproduces the clamped grid, + // so the report is not describing a grid the launch does not use. + EXPECT_EQ(solution.getSKGrid(problem, device, d.tiles, d.reduction), d.finalGrid); + + // Force-DP-only semantics are unchanged by the clamp: every tile stays whole. + EXPECT_TRUE(d.dpOnly); + EXPECT_EQ(d.skTiles, 0u); + EXPECT_FALSE(d.partialsPresent); + EXPECT_EQ(d.requiredWorkspaceBytes, 0u); + EXPECT_EQ(d.requiredWorkspaceBytes, solution.requiredWorkspaceSize(problem, device)); +} + +// --------------------------------------------------------------------------- +// The clamp is a conjunction, and the report must not claim it on any of the +// near-miss configurations. Each sub-case below flips exactly one of the three +// conditions and asserts the grid is left alone. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, ClusterDpGridClampRequiresSk3ForceDpOnlyAndXyCluster) +{ + auto problem = makeGemmProblem(4096, 4224, 512); + problem.setWorkspaceSize(std::numeric_limits::max()); + + auto device = makeDevice(_MI350_CHIP_ID, _CPX_CU, "mi350cpx"); + device.skDynamicGrid = 0; + + // (1) Dynamic mode (SK4) with the same DP-only + cluster settings: the + // multicast clamp is a static-SK3 construct, so the dynamic grid survives. + { + ContractionSolution solution; + initStreamKSolution(solution, 4); + solution.sizeMapping.streamKForceDPOnly = 1; + solution.sizeMapping.clusterDim = TensileLite::dim3(2, 2, 1); + + auto d = solution.computeStreamKDecisions(problem, device); + + EXPECT_FALSE(d.clusterDPGridClamped) << "SK4 is not the static multicast path"; + EXPECT_EQ(d.finalGrid, d.selectedGrid); + EXPECT_NE(d.finalGrid, d.tiles); + } + + // (2) SK3 with a multicast cluster but WITHOUT force-DP-only: the clamp is + // only sound when every tile is already data-parallel, so it must not fire. + { + ContractionSolution solution; + initStreamKSolution(solution, 3); // leaves streamKForceDPOnly == 0 + solution.sizeMapping.clusterDim = TensileLite::dim3(2, 2, 1); + + auto d = solution.computeStreamKDecisions(problem, device); + + ASSERT_FALSE(d.forceDPOnly); + EXPECT_FALSE(d.clusterDPGridClamped); + EXPECT_EQ(d.finalGrid, d.selectedGrid); + EXPECT_NE(d.finalGrid, d.tiles); + } + + // (3) SK3 + force-DP-only, but the cluster only extends in z. Multicast peers + // are laid out over the x/y tile grid, so a z-only cluster is not a multicast + // launch and the grid must be left alone. + { + ContractionSolution solution; + initStreamKSolution(solution, 3); + solution.sizeMapping.streamKForceDPOnly = 1; + solution.sizeMapping.clusterDim = TensileLite::dim3(1, 1, 8); + + auto d = solution.computeStreamKDecisions(problem, device); + + EXPECT_FALSE(d.clusterDPGridClamped) << "clusterDim.z does not make a multicast cluster"; + EXPECT_EQ(d.finalGrid, d.selectedGrid); + EXPECT_EQ(d.finalGrid, static_cast(_CPX_CU)); + } +} + +// --------------------------------------------------------------------------- +// The printed summary reports the cluster-multicast clamp: it is named in the +// fallbacks section, it is credited for the selected-vs-final grid change, and +// the preFallback grid is surfaced because the clamp moved the grid inside +// getSKGridImpl. The complementary run (no multicast cluster) prints the +// negative form and attributes nothing. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, PrintSummaryReportsClusterDpMulticastClamp) +{ + auto problem = makeGemmProblem(4096, 4224, 512); + problem.setWorkspaceSize(std::numeric_limits::max()); + + auto device = makeDevice(_MI350_CHIP_ID, _CPX_CU, "mi350cpx"); + device.skDynamicGrid = 0; + + // (1) Multicast cluster -> clamp fires and is reported. + ContractionSolution clustered; + clustered.kernelName = "test_streamk_cluster_dp"; + initStreamKSolution(clustered, 3); + clustered.sizeMapping.streamKForceDPOnly = 1; + clustered.sizeMapping.clusterDim = TensileLite::dim3(2, 2, 1); + + auto dc = clustered.computeStreamKDecisions(problem, device); + ASSERT_TRUE(dc.clusterDPGridClamped) << "scenario must actually trip the clamp"; + + std::ostringstream osc; + clustered.printStreamKLaunchSummary(osc, problem, dc); + const std::string clusteredLine = collapseSpaces(osc.str()); + + EXPECT_NE(clusteredLine.find("clusterDPMulticast = yes"), std::string::npos); + EXPECT_NE(clusteredLine.find("changedBy = clusterDPMulticast"), std::string::npos); + // The other fallbacks are reported as not-fired, so the clamp is unambiguous. + EXPECT_NE(clusteredLine.find("fixedGrid = no"), std::string::npos); + EXPECT_NE(clusteredLine.find("workspaceDPFallback = no"), std::string::npos); + EXPECT_NE(clusteredLine.find("treeBoundsFallback = no"), std::string::npos); + // selected != final, and the intra-getSKGridImpl clamp surfaces preFallback. + EXPECT_NE(clusteredLine.find("selected = " + std::to_string(dc.selectedGrid)), + std::string::npos); + EXPECT_NE(clusteredLine.find("final = " + std::to_string(dc.tiles)), std::string::npos); + EXPECT_NE(clusteredLine.find("preFallback = " + std::to_string(dc.tiles)), std::string::npos); + // dp-only is still sourced from the param, not from a runtime fallback. + EXPECT_NE(clusteredLine.find("source = forceDPOnly(param)"), std::string::npos); + + // (2) Same solution without a multicast cluster -> negative form, and the + // grid is not attributed to anything. + ContractionSolution plain; + plain.kernelName = "test_streamk_no_cluster"; + initStreamKSolution(plain, 3); + plain.sizeMapping.streamKForceDPOnly = 1; // clusterDim stays {1, 1, 1} + + auto dp = plain.computeStreamKDecisions(problem, device); + ASSERT_FALSE(dp.clusterDPGridClamped); + + std::ostringstream osp; + plain.printStreamKLaunchSummary(osp, problem, dp); + const std::string plainLine = collapseSpaces(osp.str()); + + EXPECT_NE(plainLine.find("clusterDPMulticast = no"), std::string::npos); + EXPECT_NE(plainLine.find("changedBy = none"), std::string::npos); + EXPECT_EQ(plainLine.find("changedBy = clusterDPMulticast"), std::string::npos); + // Nothing moved the grid, so preFallback is suppressed. + EXPECT_EQ(plainLine.find("preFallback"), std::string::npos); +} + +// --------------------------------------------------------------------------- +// changedBy attribution names the clamp that actually produced the launch grid, +// not merely the first one that fired. The skFixedGrid override picks 32 and the +// cluster-multicast clamp then overwrites it with tiles, so 32 is NOT the +// launched grid and crediting "fixedGrid" would be a lie. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, ClusterDpClampWinsAttributionOverFixedGrid) +{ + ContractionSolution solution; + solution.kernelName = "test_streamk_cluster_over_fixed"; + initStreamKSolution(solution, 3); + solution.sizeMapping.streamKForceDPOnly = 1; + solution.sizeMapping.clusterDim = TensileLite::dim3(2, 2, 1); + + auto problem = makeGemmProblem(4096, 4224, 512); + problem.setWorkspaceSize(std::numeric_limits::max()); + + auto device = makeDevice(_MI350_CHIP_ID, _CPX_CU, "mi350cpx"); + device.skDynamicGrid = 0; + device.skFixedGrid = 32; + + auto d = solution.computeStreamKDecisions(problem, device); + + ASSERT_TRUE(d.fixedGridUsed); + ASSERT_TRUE(d.clusterDPGridClamped); + EXPECT_EQ(d.selectedGrid, 32u) << "the override is what selection produced"; + EXPECT_EQ(d.finalGrid, d.tiles) << "but the cluster clamp produced the launch grid"; + + std::ostringstream os; + solution.printStreamKLaunchSummary(os, problem, d); + const std::string line = collapseSpaces(os.str()); + + EXPECT_NE(line.find("changedBy = clusterDPMulticast"), std::string::npos); + EXPECT_EQ(line.find("changedBy = fixedGrid"), std::string::npos) + << "the fixed grid was overwritten; it did not produce the launch grid"; + // Both are still reported as having fired in the fallbacks section. + EXPECT_NE(line.find("fixedGrid = yes"), std::string::npos); + EXPECT_NE(line.find("clusterDPMulticast = yes"), std::string::npos); +} From 6de26d3a3c1c146632fe9ed4c64fcae12255740d Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Thu, 27 Aug 2026 21:10:07 +0000 Subject: [PATCH 6/8] docs(tensilelite): fix StreamK launch-summary comment provenance The workspace / DP-fallback logic now lives in computeStreamKDecisions() rather than solve(), but a dozen comments still described themselves as mirroring solve(). There is nothing left in solve() to mirror: it copies the reduction, grid and isDynamic predicate straight out of the snapshot and does no further StreamK sizing. Point those comments at the real drift peer instead. requiredWorkspaceSize() is a second, independent implementation of the same reserve-or-not rule and is what the caller allocates from, yet it already differs: it always uses getSKReduction() and sizes parallel reduction with requiredWorkspaceSizeGsu() rather than partialTileSize(). Note explicitly that StreamKDecisions::requiredWorkspaceBytes is NOT that function's return value despite the near-identical name. Other corrections: * skTiles/skSplit/totalItems were documented with a formula that is false on the parallel and static paths -- the SK3 ABI has no TotalItems arg, and the parallel path packs the split factor into the skTiles slot. Document all three paths. * "It has no side effects" in solve() was wrong: computeStreamKDecisions() must precede the dynamic-queue guard because the guard consumes isDynamic, and the helpers it calls can write TENSILE_DB diagnostics. Record what is and is not reachable on a guard-rejected launch. * The SIZE_MAX -> "unbounded" sentinel is ContractionProblem's default workspace size (setWorkspaceSize() never called), not something the bench client passes. * The print-layout comment claimed a format was "preserved verbatim" for existing log scrapers; the function is new, so describe the output shape and which tokens are load-bearing instead. * Restore "Only these three fields are refreshed" in reloadDebugBitsForTest()'s doc -- it still refreshes exactly three. * Document that effectiveDynamic is SK5-only (false for SK4), that isDynamic is a launch input, why finalGrid and skGrid both exist, the dpOnlySource ordering, and the (void)problem parameter retention. --- .../include/Tensile/ContractionSolution.hpp | 89 ++++++++++---- .../tensilelite/include/Tensile/Debug.hpp | 2 +- .../tensilelite/src/ContractionSolution.cpp | 111 ++++++++++++------ 3 files changed, 142 insertions(+), 60 deletions(-) diff --git a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp index eb40d5950818..222361531ecc 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp @@ -300,10 +300,14 @@ namespace TensileLite // Each field's provenance is annotated inline as one of: // available = read from the value the launch uses, or from the same real // helper/param solve() uses, so it reflects the launch exactly. - // recomputed = re-derived here by mirroring the makeArgs/solve() sizing and - // packing logic (a faithful re-derivation, not a stored launched - // value). - // Computing or printing this report does not change launch behaviour. See + // recomputed = re-derived here rather than read back from a launched value: + // either by mirroring the kernel-arg packing in makeArgs() + // (skTiles/skSplit/totalItems) or by deriving from other fields. + // Mirrors can drift silently if makeArgs() changes; each such + // field names what it mirrors. + // This snapshot is not purely observational: reduction, skGrid and isDynamic are + // launch INPUTS -- solve() copies them into StreamKSettings and uses isDynamic as + // its dynamic-queue predicate. Only the PRINTING is optional; see // Debug::printStreamKLaunchSummary() (TENSILE_DB bit 0x200000). struct StreamKDecisions { @@ -311,9 +315,13 @@ namespace TensileLite // available: sizeMapping.streamK, the mode solve() uses (0 = not StreamK, else 3/4/5). int streamKMode = 0; // available: streamK5EffectiveDynamic(), the same helper solve()'s grid path uses - // (SK5 resolved to the dynamic SK4 sub-path). + // (SK5 resolved to the dynamic SK4 sub-path). Only meaningful for SK5: it stays + // false for SK4 even though SK4 is unconditionally dynamic, so ask isDynamic + // (below) -- not this -- whether a launch takes the dynamic path. bool effectiveDynamic = false; // recomputed: derived from streamKMode + effectiveDynamic (SK4, or SK5 resolved dynamic). + // Launch-relevant rather than merely reported: solve() consumes this as the + // dynamicQueuePath predicate guarding the work-stealing rejection. bool isDynamic = false; // --- Reduction --- @@ -335,13 +343,34 @@ namespace TensileLite // available: wired from sk.grid, the FINAL grid solve() launches with (after all // fallbacks: fixed-grid override, tree-fixup-bounds, workspace-insufficient DP). size_t finalGrid = 0; - // available: alias of finalGrid (the grid solve() launches with). + // available: the grid solve() initialises sk.grid from. finalGrid and skGrid + // always hold the same value -- computeStreamKDecisions() sets both to its final + // grid, and solve() writes the launched sk.grid back into both. Both exist + // because they are read for different reasons: skGrid is the launch INPUT + // solve() consumes, finalGrid is the reported outcome printed against + // selectedGrid. size_t skGrid = 0; - // recomputed: mirrors the makeArgs tile packing (stream-k / partial tile count). + // The next three mirror the kernel-arg packing in makeArgs() (search for + // "SKTiles"/"skTiles"), evaluated on the FINAL post-fallback grid and + // reduction. They are hand-written mirrors: any change to the packing in + // makeArgs() must be reflected in computeStreamKDecisions() or the report + // silently drifts from the launch. + // + // recomputed: dynamic path -> the SKTiles arg (debug override, else 0); + // parallel path -> the split factor, which is what makeArgs packs + // into the skTiles slot on that path; + // static SK3 -> streamKStaticSplit().skTiles, the stream-k + // (partial) tile count. size_t skTiles = 0; - // recomputed: mirrors the makeArgs k-split packing (>=1). + // recomputed: dynamic path -> the SKSplit arg after the CeilDivide round-trip; + // parallel path -> grid / tiles; + // static SK3 -> 1 (no k-split; SK3 packs SKItersPerWG instead). size_t skSplit = 0; - // recomputed: mirrors the makeArgs packing, (tiles - skTiles) + skTiles*skSplit. + // recomputed: work-item count. Dynamic path mirrors the packed TotalItems arg, + // (tiles - skTiles) + skTiles*skSplit. The SK3 ABI has no TotalItems + // arg, so on the parallel and static paths this is reported as + // 'tiles' (one work item per output tile) rather than mirroring a + // packed value. size_t totalItems = 0; // --- DP-only --- @@ -359,11 +388,22 @@ namespace TensileLite // --- Workspace / partials --- // recomputed: skTiles > 0. bool partialsPresent = false; - // recomputed: mirrors the sizing guard (requiredWorkspaceBytes > 0). + // recomputed: requiredWorkspaceBytes > 0. bool workspaceAllocated = false; - // recomputed: mirrors the sizing guard -- partials(+queue) bytes reserved (0 if none/fallback). + // recomputed: partials(+work-queue) bytes this launch reserves; 0 when no + // partials are needed or the workspace-DP fallback fired. + // + // WARNING: this is NOT ContractionSolution::requiredWorkspaceSize()'s return + // value, even though the names are close. requiredWorkspaceSize() is the + // separate, caller-facing implementation of the same reserve-or-not rule (it + // is what the allocator sizes from) and it differs today: it always uses + // getSKReduction(), and for parallel reduction it sizes with + // requiredWorkspaceSizeGsu() rather than partialTileSize(). The two must stay + // consistent about WHETHER a workspace is needed -- change one, check the other. size_t requiredWorkspaceBytes = 0; - // recomputed: mirrors the sizing guard -- partials(+queue) bytes wanted (pre fit check). + // recomputed: partials(+work-queue) bytes wanted, before the fit check against + // givenWorkspaceBytes. Non-zero even when the fallback fires, which is what + // makes the "wanted vs given" comparison legible in the report. size_t idealWorkspaceBytes = 0; // available: problem.workspaceSize(). size_t givenWorkspaceBytes = 0; @@ -375,7 +415,11 @@ namespace TensileLite size_t numQueues = 0; // --- Fallbacks that fired (each can change selectedGrid -> finalGrid) --- - // recomputed: mirrors solve()'s workspace fallback (idealWorkspace > given -> tree + grid=tiles). + // recomputed: the workspace-insufficient fallback (idealWorkspaceBytes > + // givenWorkspaceBytes -> reduction=tree, grid=tiles). computeStreamKDecisions() + // is the sole implementation of this fallback; solve() consumes the result. + // Keep the reserve-or-not condition in sync with + // ContractionSolution::requiredWorkspaceSize(). bool workspaceDPFallbackFired = false; // available: getSKGridImpl out-param (24-bit tree-fixup bounds -> grid=tiles). bool treeBoundsFallbackFired = false; @@ -575,15 +619,16 @@ namespace TensileLite // Compute the StreamK launch-parameter DECISIONS for this solution on the // given problem/hardware. solve() consumes the reduction strategy, grid, - // and workspace/DP fallbacks from here to populate StreamKSettings; it is - // also directly callable from unit tests. Some fields are read from the - // values the launch uses (or the same helpers solve() uses: - // streamK5EffectiveDynamic, getSKReduction, getSKGrid, partialTileSize); - // others are re-derived here by mirroring the makeArgs/solve() sizing and - // packing logic. Each StreamKDecisions field documents its own provenance - // (available vs recomputed). Calling it changes no launch behaviour. - // Returns an all-default snapshot (streamKMode==0) for non-StreamK - // solutions. + // isDynamic predicate, and workspace/DP fallback from here to populate + // StreamKSettings -- this is the only place that logic lives -- and it is + // also directly callable from unit tests. Existing helpers are reused where + // possible (streamK5EffectiveDynamic, getSKReduction, getSKGridImpl, + // partialTileSize); the makeArgs packing quantities are re-derived. Each + // StreamKDecisions field documents its own provenance (available vs + // recomputed). Has no effect on the launch beyond producing these values, and + // never mutates solution or problem state. For a non-StreamK solution + // (sizeMapping.streamK <= 0) it returns immediately with a default-initialised + // snapshot whose streamKMode is sizeMapping.streamK. StreamKDecisions computeStreamKDecisions(Problem const& problem, Hardware const& hardware) const; diff --git a/projects/hipblaslt/tensilelite/include/Tensile/Debug.hpp b/projects/hipblaslt/tensilelite/include/Tensile/Debug.hpp index 252785208e41..6d01e697a502 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/Debug.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/Debug.hpp @@ -61,7 +61,7 @@ namespace TensileLite bool printWinningKernelName() const; // Re-reads TENSILE_DB, TENSILE_DB2, and TENSILE_STREAMK5_FORCE_MODE from - // the environment and updates cached state. Only these fields are + // the environment and updates cached state. Only these three fields are // refreshed; all other env-driven settings remain at their initial values. // Intended for tests that call setenv() in-process after the singleton has // already been constructed. diff --git a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp index 21910abd4eb8..dbb39f30e0cf 100644 --- a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp +++ b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp @@ -3484,12 +3484,25 @@ namespace TensileLite auto tiles = problem.getNumTiles(sizeMapping, 1); // Single source of truth: compute the StreamK launch decisions once // (also consumed by the diagnostic launch summary and unit tests) and - // consume them here. computeStreamKDecisions() applies exactly the - // reduction / grid / workspace-DP-fallback logic used at launch, - // reusing the same helpers, so the decisions match what is launched. - // It has no side effects, so computing it before the dynamic-queue - // guard below cannot change observable behaviour (the guard still - // throws before any kernel-arg packing / kernel launch). + // consume them here. computeStreamKDecisions() owns the reduction / + // grid / workspace-DP-fallback logic and reuses the same helpers, so + // the decisions ARE what is launched. + // + // The dynamic-queue guard below consumes skDecisions.isDynamic, so + // this call has to precede it -- which moves getSKReduction(), + // getSKGridImpl() and the workspace sizing ahead of that rejection. + // That is safe rather than free: the call mutates no solution or + // problem state and launches nothing, so the guard still throws before + // any kernel-arg packing or kernel launch. It is not, however, + // output-free -- the helpers it calls can write TENSILE_DB diagnostics + // to stderr (streamK5EffectiveDynamic's SK5 mode-selection note under + // 0x100000, getSKGridImpl's CU-occupancy note under + // printPropertyEvaluation). Neither is reachable on a guard-rejected + // launch as the code stands (the guard only fires on the dynamic-queue + // path, where the reduction is forced to tree without calling + // getSKReduction and getSKGridImpl takes its diagnostic-free + // branches), but a future edit that prints from those helpers would + // start emitting for solutions this guard goes on to reject. skDecisions = computeStreamKDecisions(problem, hardware); // Defensive: dynamic-queue / work-stealing StreamK solutions are // excluded from selection on devices whose runtime XCD count is not @@ -3524,8 +3537,8 @@ namespace TensileLite "Select a non-work-stealing solution instead."); } // Consume the reduction / grid decided above. computeStreamKDecisions - // applies the same logic (including the workspace-insufficient DP - // fallback) using the same helpers, so these match what is launched. + // has already applied the workspace-insufficient DP fallback, and no + // further StreamK sizing happens below, so these are the launch values. sk.reduction = skDecisions.reduction; sk.grid = skDecisions.skGrid; sk.streamKTileSchedulingMode = problem.getParams().streamKTileSchedulingMode(); @@ -4846,18 +4859,24 @@ namespace TensileLite return size; } - // Single source of truth for the StreamK launch decisions. This mirrors -- - // and is consumed by -- the sizing block in solve(): the reduction, grid, and - // workspace/DP fallbacks below are exactly what solve() applies to the - // StreamKSettings it launches with, and the skTiles/skSplit/totalItems fields - // mirror the kernel-arg packing in makeArgs(). It reuses the existing helpers - // rather than re-deriving, so the summary cannot drift from the real launch. + // Single source of truth for the StreamK launch decisions. The reduction, grid, + // and workspace/DP fallback computed here are the values solve() launches with: + // solve() reads them straight out of the returned snapshot into StreamKSettings + // and applies no further StreamK sizing of its own. Existing helpers + // (streamK5EffectiveDynamic, getSKReduction, getSKGridImpl, partialTileSize) are + // reused rather than re-derived. // - // The partials-workspace guard and sizing below reserve iff - // (reduction==parallel || tiles%grid!=0), sized as partialTileSize(grid) (+ the - // per-XCD work-queue region for the dynamic path); the reservation does not - // depend on dynamicPartialsSlots. The dynamicPartialsSlots field is still - // populated (skTiles*skSplit, computed locally) purely for reporting. + // Two mirrors here must be kept in sync with code elsewhere: + // * the reserve-or-not workspace rule below duplicates the one in + // ContractionSolution::requiredWorkspaceSize(), which is what the caller + // allocates from; + // * skTiles/skSplit/totalItems duplicate the kernel-arg packing in makeArgs(). + // + // The partials-workspace guard reserves iff (reduction==parallel || tiles%grid!=0), + // sized as partialTileSize(grid) (+ the per-XCD work-queue region on the dynamic + // path); the reservation does not depend on dynamicPartialsSlots. The + // dynamicPartialsSlots field is still populated (skTiles*skSplit, computed + // locally) purely for reporting. StreamKDecisions ContractionSolution::computeStreamKDecisions(Problem const& problem, Hardware const& hardware) const @@ -4874,7 +4893,9 @@ namespace TensileLite = (sizeMapping.streamK == 5) ? streamK5EffectiveDynamic(problem, hardware) : false; d.effectiveDynamic = effectiveDynamic; - // Reduction strategy -- mirrors solve(). + // Reduction strategy. SK4 and SK5-resolved-dynamic are unconditionally tree; + // everything else asks getSKReduction(). Note requiredWorkspaceSize() always + // asks getSKReduction() and has no such special case -- see the note above. origami::reduction_t reduction; if(sizeMapping.streamK == 4) reduction = origami::reduction_t::tree; @@ -4921,9 +4942,10 @@ namespace TensileLite d.numQueues = streamKBakedQueueCount(hardware); d.givenWorkspaceBytes = problem.workspaceSize(); - // Workspace / DP fallback -- mirrors the guard in solve(): reserve iff - // (reduction==parallel || tiles%grid!=0), sized by grid (not by - // dynamicSlots). + // Workspace / DP fallback. Reserve iff (reduction==parallel || + // tiles%grid!=0), sized by grid (not by dynamicSlots). This is the same + // reserve-or-not rule requiredWorkspaceSize() implements independently, so + // the two must be changed together. size_t idealWorkspace = 0; bool needPartials = false; if(grid > 0 @@ -5005,7 +5027,13 @@ namespace TensileLite Problem const& problem, StreamKDecisions const& d) const { - (void)problem; // reserved for future per-problem detail; keeps the API stable + // Everything printed below already lives in the snapshot, so the problem is + // currently unused. It stays in the signature because a launch summary is + // naturally reported per (solution, problem) pair and the obvious next + // additions -- the GEMM sizes, transposes, and problem-level StreamK params + // -- are only reachable from here; keeping it avoids churning every call + // site and every test when one of those is added. + (void)problem; auto reductionStr = [](origami::reduction_t r) { return r == origami::reduction_t::parallel ? "parallel(DP)" : "tree"; }; @@ -5023,10 +5051,12 @@ namespace TensileLite // Which fallback (if any) turned the initially-selected grid into the // final launch grid. Reported alongside selectedGrid vs finalGrid. - // Ordered latest-clamp-wins: the workspace-DP fallback runs in solve(), - // after getSKGridImpl; within getSKGridImpl the ForceDPOnly cluster + // Ordered latest-clamp-wins, i.e. the reverse of the order they are applied: + // the workspace-DP fallback runs last (in computeStreamKDecisions, after + // getSKGridImpl returns); inside getSKGridImpl the ForceDPOnly cluster // multicast clamp runs after the tree-bounds fallback, which in turn runs - // after the skFixedGrid override. + // after the skFixedGrid override. So the first matching branch below names + // the clamp that actually produced finalGrid. const char* gridChangedBy = "none"; if(d.workspaceDPFallbackFired) gridChangedBy = "workspaceDP"; @@ -5037,7 +5067,12 @@ namespace TensileLite else if(d.fixedGridUsed) gridChangedBy = "fixedGrid"; - // Which mechanism (if any) makes this launch data-parallel-only. + // Which mechanism (if any) makes this launch data-parallel-only. More than + // one can be set at once (e.g. the debug override on a force-DP-only + // kernel), so the ladder reports the most specific explanation first: + // the compile-time kernel param, then the process-wide debug override, + // then the runtime workspace fallback -- from "this kernel is always DP" + // to "this particular launch had to give up on StreamK". const char* dpOnlySource = "none"; if(d.forceDPOnly) dpOnlySource = "forceDPOnly(param)"; @@ -5047,7 +5082,10 @@ namespace TensileLite dpOnlySource = "workspaceDP(runtime)"; // Human-readable byte annotation, e.g. "1245184 (1.19 MiB)". SIZE_MAX is - // reported as "unbounded" (the sentinel bench passes for "no cap"). + // reported as "unbounded": that is ContractionProblem's default workspace + // size (see ContractionProblem.hpp m_workspaceSize), meaning + // setWorkspaceSize() was never called and the workspace is uncapped, so + // printing it as a byte count would be nonsense. auto humanUnit = [](size_t bytes) -> std::string { static const char* units[] = {"B", "KiB", "MiB", "GiB", "TiB"}; double v = static_cast(bytes); @@ -5079,14 +5117,13 @@ namespace TensileLite return fmtBytes(bytes); }; - // Deeply-indented, multi-line labeled block. The leading token - // "TensileLite::StreamK LAUNCH SUMMARY" is preserved verbatim (with the - // kernel name on the same first line) so existing log scrapers / test - // substring matches keep working. Each section header sits on its own - // line and its fields are indented beneath it as aligned "key = value" - // pairs, one field per line. This is a pure LAYOUT change: no field, - // value, label, or the NA-when-!isDynamic behaviour is altered, and no - // launch behaviour is affected. + // Output shape: one header line carrying the fixed token + // "TensileLite::StreamK LAUNCH SUMMARY" and the kernel name, followed by + // labeled sections. Each section header sits on its own line and its + // fields are indented beneath it as "key = value" pairs, one per line, + // with the '=' column-aligned within the section. Keep the header token + // and the field key spellings stable: log scrapers and the substring + // assertions in tests/StreamKLaunchSummary_test.cpp match on them. // // 'field' prints one indented, '='-aligned pair. 'width' is the per-section // key column width chosen so every '=' in that section lines up. From 7b4711c8a669fea1431981ffe59f1c241b545175 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 31 Aug 2026 16:53:01 +0000 Subject: [PATCH 7/8] docs(tensilelite): correct StreamK grid fallback docs and skFullTiles default Three consistency fixes around the StreamK launch summary: - skGridPreFallback documented the workspace-DP fallback as living in solve(); it is applied in computeStreamKDecisions(), which is what the workspaceDPFallbackFired doc a few lines below already says. - finalGrid's list of the fallbacks it accounts for omitted the ForceDPOnly cluster-multicast clamp. It now names all four clamps in application order using the same names the report's "changedBy" ladder prints, so the doc and the ladder can be compared directly. - The static-split path in computeStreamKDecisions() fell back to skFullTiles=0 when the AMDGPU downcast yielded null. AMDGPU::skFullTiles defaults to 1, and checkUniformSummationOrder() already uses 1 for the same fallback. The path is not reachable today (getSKGridImpl asserts and dereferences the same pointer first), so this only keeps the constant consistent with its two siblings. No behavioural change. --- .../tensilelite/include/Tensile/ContractionSolution.hpp | 6 ++++-- projects/hipblaslt/tensilelite/src/ContractionSolution.cpp | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp index 222361531ecc..4b224e263175 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp @@ -338,10 +338,12 @@ namespace TensileLite size_t selectedGrid = 0; // available: getSKGridImpl return -- AFTER the tree-fixup-bounds fallback and // the ForceDPOnly cluster-multicast clamp, but BEFORE the workspace-DP - // fallback in solve(). Diagnostic intermediate. + // fallback in computeStreamKDecisions(). Diagnostic intermediate. size_t skGridPreFallback = 0; // available: wired from sk.grid, the FINAL grid solve() launches with (after all - // fallbacks: fixed-grid override, tree-fixup-bounds, workspace-insufficient DP). + // fallbacks, in the order they are applied: fixedGrid, treeBounds, + // clusterDPMulticast, workspaceDP -- the same four names the report's + // "changedBy" ladder uses, which lists them latest-clamp-wins). size_t finalGrid = 0; // available: the grid solve() initialises sk.grid from. finalGrid and skGrid // always hold the same value -- computeStreamKDecisions() sets both to its final diff --git a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp index dbb39f30e0cf..b01781ecd8ac 100644 --- a/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp +++ b/projects/hipblaslt/tensilelite/src/ContractionSolution.cpp @@ -5006,7 +5006,7 @@ namespace TensileLite // arithmetic: makeArgs() packs skTiles from exactly this call, so the // report cannot drift from the launch. AMDGPU const* pAMDGPU = dynamic_cast(&hardware); - const int fullTiles = pAMDGPU ? pAMDGPU->skFullTiles : 0; + const int fullTiles = pAMDGPU ? pAMDGPU->skFullTiles : 1; const StreamKStaticSplit split = streamKStaticSplit(tiles, itersPerTile, grid, fullTiles, forceDPOnly); d.skTiles = split.skTiles; From 833e0f283763457535de002d62ba9dd980824869 Mon Sep 17 00:00:00 2001 From: "Joao P. L. de Carvalho" Date: Mon, 31 Aug 2026 17:15:58 +0000 Subject: [PATCH 8/8] test(tensilelite): cover StreamK parallel reduction and grid attribution The StreamK launch-summary tests left four behaviours unasserted. Add five tests that close them: - Sk3ParallelReductionReservesPartialsWorkspace: the parallel (split-K DP) reduction, which no existing test reached. A 256x4096x4096 SK3 launch on the 256-CU analytical mock lands in select_reduction's parallel band, so the snapshot's packing (skSplit = grid/tiles) and its unconditional partials reservation are pinned. - Sk3ParallelReductionWorkspaceStarvedFallsBackToTree: the same shape with no workspace, demoting to tree with grid = tiles while the workspace-DP fallback is still credited even though the grid number did not move. - LaunchSummaryDebugBitIsOffByDefault: the summary is opt-in. Asserts the Debug predicate is clear with TENSILE_DB unset, set by 0x200000, not set by the neighbouring 0x100000 mode-selection bit, restored to the ambient value afterwards, and that building the snapshot writes nothing to stderr. - TreeBoundsFallbackWinsGridAttribution and FixedGridOverrideWinsGridAttribution: the two remaining changedBy attributions, each with a complementary run showing the grid is left alone when the clamp does not apply. Adds a ScopedTensileDb RAII guard that copies the prior TENSILE_DB value before setenv invalidates it and restores that value on scope exit, since the Debug singleton is process-wide. The four decision tests assert TENSILE_STREAMK_DATA_PARALLEL is clear, because it is latched at Debug construction and would otherwise change the answers silently. Also correct the StreamKDecisions::requiredWorkspaceBytes comment. It claimed the snapshot and requiredWorkspaceSize() must stay consistent about whether a workspace is needed; they do not. Parallel reduction with a grid equal to tiles gives a k-split of 1, requiredWorkspaceSizeGsu() short-circuits to 0 for a split of 1, and the snapshot still reserves partialTileSize(tiles). The comment now describes that case, why it is an over-reservation rather than an under-allocation, and why the allocate-then-launch loop settles at a self-consistent fixed point. --- .../include/Tensile/ContractionSolution.hpp | 30 +- .../tests/StreamKLaunchSummary_test.cpp | 503 ++++++++++++++++++ 2 files changed, 528 insertions(+), 5 deletions(-) diff --git a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp index 4b224e263175..b4764d6103ad 100644 --- a/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp +++ b/projects/hipblaslt/tensilelite/include/Tensile/ContractionSolution.hpp @@ -397,11 +397,31 @@ namespace TensileLite // // WARNING: this is NOT ContractionSolution::requiredWorkspaceSize()'s return // value, even though the names are close. requiredWorkspaceSize() is the - // separate, caller-facing implementation of the same reserve-or-not rule (it - // is what the allocator sizes from) and it differs today: it always uses - // getSKReduction(), and for parallel reduction it sizes with - // requiredWorkspaceSizeGsu() rather than partialTileSize(). The two must stay - // consistent about WHETHER a workspace is needed -- change one, check the other. + // separate, caller-facing implementation of the reserve-or-not rule -- it is + // what the allocator sizes the workspace buffer from -- and it computes the + // answer differently: it always asks getSKReduction(), and for parallel + // reduction it sizes with requiredWorkspaceSizeGsu(problem, hardware, + // grid / tiles) instead of partialTileSize(grid). + // + // The two can therefore disagree about WHETHER a workspace is needed, not + // just about how many bytes. The case where they do is parallel reduction + // whose selected grid comes back equal to tiles: the k-split factor + // grid / tiles is then 1, and requiredWorkspaceSizeGsu() short-circuits to 0 + // for a split of 1, while this field still reserves partialTileSize(tiles) + // because parallel reduction always needs a partials region. Concretely, a + // 256x4096x4096 SK3 launch with a 128x128x64 macro tile on a 256-CU device, + // handed a workspace between 4 MiB and 16 MiB, reports 4 MiB here and 0 from + // requiredWorkspaceSize(). + // + // That divergence is an over-reservation of a buffer the caller already + // supplied, never an under-allocation, so it is benign in the real + // allocate-then-launch flow: the allocator sizes from + // requiredWorkspaceSize(), that size is what problem.workspaceSize() reports + // on the subsequent launch, and re-deriving this snapshot against it reaches + // a self-consistent fixed point (0 bytes -> workspace-DP fallback -> 0 bytes + // reserved). It only becomes visible when a caller hands in a workspace it + // sized some other way. Both implementations still encode the same intended + // rule -- change one, check the other. size_t requiredWorkspaceBytes = 0; // recomputed: partials(+work-queue) bytes wanted, before the fit check against // givenWorkspaceBytes. Non-zero even when the fallback fires, which is what diff --git a/projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp b/projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp index 5b1ca89af3f3..48533d5729cd 100644 --- a/projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp +++ b/projects/hipblaslt/tensilelite/tests/StreamKLaunchSummary_test.cpp @@ -16,6 +16,8 @@ // and whether a partials workspace is reserved. #include +#include +#include #include #include #include @@ -127,6 +129,52 @@ namespace } return out; } + + // Sets (or clears) TENSILE_DB for the lifetime of the object and refreshes the + // Debug singleton from it, restoring the PREVIOUS value -- not merely unsetting + // -- on scope exit. Debug::Instance() is a process-wide function-local static + // (see Singleton.hpp, LazySingleton), so an unrestored TENSILE_DB would leak + // into every later test in this binary; the destructor still runs when an + // ASSERT_* aborts the enclosing test, which a TearDown() could not be relied on + // to do for a scoped state change. + // + // reloadDebugBitsForTest() refreshes exactly TENSILE_DB, TENSILE_DB2 and + // TENSILE_STREAMK5_FORCE_MODE. It deliberately does NOT refresh + // TENSILE_STREAMK_DATA_PARALLEL, so nothing here can toggle that flag. + class ScopedTensileDb + { + public: + // value == nullptr means "unset TENSILE_DB", i.e. the shipped default. + explicit ScopedTensileDb(const char* value) + { + const char* prior = std::getenv("TENSILE_DB"); + m_had = (prior != nullptr); + if(m_had) + m_saved = prior; // copy BEFORE setenv invalidates the pointer + + if(value) + setenv("TENSILE_DB", value, /*overwrite=*/1); + else + unsetenv("TENSILE_DB"); + Debug::Instance().reloadDebugBitsForTest(); + } + + ~ScopedTensileDb() + { + if(m_had) + setenv("TENSILE_DB", m_saved.c_str(), /*overwrite=*/1); + else + unsetenv("TENSILE_DB"); + Debug::Instance().reloadDebugBitsForTest(); + } + + ScopedTensileDb(ScopedTensileDb const&) = delete; + ScopedTensileDb& operator=(ScopedTensileDb const&) = delete; + + private: + bool m_had = false; + std::string m_saved; + }; } // namespace // --------------------------------------------------------------------------- @@ -866,3 +914,458 @@ TEST(StreamKLaunchSummaryTest, ClusterDpClampWinsAttributionOverFixedGrid) EXPECT_NE(line.find("fixedGrid = yes"), std::string::npos); EXPECT_NE(line.find("clusterDPMulticast = yes"), std::string::npos); } + +// --------------------------------------------------------------------------- +// The parallel (split-K data-parallel) reduction path. +// +// Every other test in this file runs a tree-reduction launch. getSKReduction() +// delegates to origami's select_reduction() whenever the device's skDynamicGrid is +// k_split_aware (== 6, the AMDGPU default), and select_reduction returns parallel +// only on the narrow band "tiles < cuCount && itersPerTile >= 64 && +// tiles <= cuCount/4". 256x4096x4096 with a 128x128x64 macro tile lands exactly +// there on the 256-CU analytical mock: tiles = 2*32 = 64 == cuCount/4, and +// itersPerTile = 4096/64 = 64. SK4 and SK5-dynamic are unconditionally tree, so +// this has to be SK3. +// +// Parallel is the one reduction that reserves a partials workspace unconditionally +// (the tree path only reserves when tiles % grid != 0). The snapshot sizes that +// reservation with partialTileSize(finalGrid), while the caller-facing +// requiredWorkspaceSize() sizes it with requiredWorkspaceSizeGsu(problem, hw, +// grid/tiles) -- two different formulas for the same region. They agree here +// because the parallel grid is an exact multiple of tiles (grid == tiles*skSplit +// == 64*4), which makes requiredWorkspaceSizeGsu's tiles*gsu equal +// partialTileSize's grid. They do NOT agree when the parallel grid comes back +// equal to tiles; see the comment on StreamKDecisions::requiredWorkspaceBytes. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, Sk3ParallelReductionReservesPartialsWorkspace) +{ + AnalyticalEnv env; + ContractionSolution solution; + solution.kernelName = "test_streamk_parallel"; + initStreamKSolution(solution, 3); // SK3 static; SK4 is unconditionally tree + + auto problem = makeGemmProblem(256, 4096, 4096); + problem.setWorkspaceSize(std::numeric_limits::max()); + + // TENSILE_STREAMK_DATA_PARALLEL is latched when Debug is constructed and is not + // among the variables reloadDebugBitsForTest() refreshes, so it can only be + // cleared before the process starts. Fail loudly rather than silently asserting + // something else if it is set. + ASSERT_FALSE(Debug::Instance().useStreamKDataParrallel()) + << "unset TENSILE_STREAMK_DATA_PARALLEL before running this suite"; + + auto d = solution.computeStreamKDecisions(problem, env.device); + + // Anti-vacuity: the whole point of this test is the parallel branch. + ASSERT_EQ(d.reduction, origami::reduction_t::parallel) + << "scenario must actually select parallel reduction, otherwise every " + "assertion below is about the already-covered tree path"; + // ...and it came from the real helper, not from a special case in the snapshot. + EXPECT_EQ(solution.getSKReduction(problem, env.device), origami::reduction_t::parallel); + + EXPECT_EQ(d.streamKMode, 3); + EXPECT_FALSE(d.isDynamic) << "isDynamic implies tree reduction"; + EXPECT_EQ(d.tiles, 64u) << "256/128 * 4096/128 = 2 * 32"; + EXPECT_EQ(problem.getItersPerTile(solution.sizeMapping), 64u) + << "4096/64; select_reduction needs itersPerTile >= 64"; + + // No clamp fires: selected == preFallback == final. + EXPECT_EQ(d.selectedGrid, 256u); + EXPECT_EQ(d.skGridPreFallback, 256u); + EXPECT_EQ(d.finalGrid, 256u); + EXPECT_EQ(d.skGrid, d.finalGrid); + EXPECT_EQ(solution.getSKGrid(problem, env.device, d.tiles, d.reduction), d.finalGrid); + EXPECT_FALSE(d.fixedGridUsed); + EXPECT_FALSE(d.treeBoundsFallbackFired) << "tree-bounds fixup is tree-reduction only"; + EXPECT_FALSE(d.clusterDPGridClamped); + EXPECT_FALSE(d.workspaceDPFallbackFired); + + // Parallel packing: skSplit = grid/tiles, and skTiles mirrors skSplit. + EXPECT_EQ(d.skSplit, 4u) << "grid/tiles = 256/64"; + EXPECT_EQ(d.skTiles, d.skSplit) << "the parallel path packs skTiles = skSplit"; + EXPECT_EQ(d.totalItems, d.tiles); + EXPECT_TRUE(d.partialsPresent); + EXPECT_FALSE(d.dpOnly); + EXPECT_FALSE(d.forceDPOnly); + EXPECT_FALSE(d.streamKDP); + + // Parallel reduction always reserves partials, sized by the FINAL grid. + ASSERT_TRUE(d.workspaceAllocated) + << "parallel reduction must reserve a partials region, otherwise the sizing " + "checks below assert nothing"; + EXPECT_EQ(d.requiredWorkspaceBytes, solution.partialTileSize(d.finalGrid)); + EXPECT_EQ(d.requiredWorkspaceBytes, 16777216u) << "128*128*4 bytes * 256 work-groups"; + EXPECT_EQ(d.idealWorkspaceBytes, d.requiredWorkspaceBytes) << "it fits, so nothing was trimmed"; + + // In this regime -- and only because finalGrid is an exact multiple of tiles -- + // the two independent sizings land on the same byte count. + EXPECT_EQ(d.finalGrid, d.tiles * d.skSplit) << "why the two sizings coincide here"; + EXPECT_GT(solution.requiredWorkspaceSize(problem, env.device), 0u); + EXPECT_EQ(solution.requiredWorkspaceSize(problem, env.device), d.requiredWorkspaceBytes); + + // The report names the reduction and attributes nothing to a fallback. + std::ostringstream os; + solution.printStreamKLaunchSummary(os, problem, d); + const std::string line = collapseSpaces(os.str()); + EXPECT_NE(line.find("reduction = parallel(DP)"), std::string::npos); + EXPECT_NE(line.find("changedBy = none"), std::string::npos); + EXPECT_EQ(line.find("preFallback"), std::string::npos) << "nothing moved the grid"; + EXPECT_NE(line.find("isDynamic = no"), std::string::npos); + EXPECT_NE(line.find("NA (work-queues not used)"), std::string::npos); +} + +// --------------------------------------------------------------------------- +// The parallel branch's workspace-DP fallback. +// +// Same shape as the test above, but with no workspace at all. getSKReduction() +// still says parallel -- it never looks at the workspace -- while the snapshot +// demotes the launch to tree with grid = tiles because the partials region does +// not fit. This is the parallel-side complement of WorkspaceDpFallbackFires +// (which covers the tree / indivisible-tiles side), and it is also a second +// demonstration that changedBy is about ATTRIBUTION rather than about the numbers +// differing: with no workspace, origami's grid selection already returns tiles, so +// selectedGrid == finalGrid while the fallback is still correctly credited. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, Sk3ParallelReductionWorkspaceStarvedFallsBackToTree) +{ + AnalyticalEnv env; + ContractionSolution solution; + solution.kernelName = "test_streamk_parallel_starved"; + initStreamKSolution(solution, 3); + + auto problem = makeGemmProblem(256, 4096, 4096); + problem.setWorkspaceSize(0); // no workspace at all + + ASSERT_FALSE(Debug::Instance().useStreamKDataParrallel()) + << "unset TENSILE_STREAMK_DATA_PARALLEL before running this suite"; + + // Anti-vacuity: the pre-fallback reduction really is parallel. (getSKReduction + // is workspace-independent, so it reports what the snapshot started from.) + ASSERT_EQ(solution.getSKReduction(problem, env.device), origami::reduction_t::parallel) + << "scenario must start from parallel reduction for the fallback to be the " + "parallel-path fallback"; + + auto d = solution.computeStreamKDecisions(problem, env.device); + + EXPECT_EQ(d.reduction, origami::reduction_t::tree) << "the fallback demotes to tree"; + EXPECT_TRUE(d.workspaceDPFallbackFired); + EXPECT_TRUE(d.dpOnly); + EXPECT_FALSE(d.forceDPOnly) << "runtime fallback, not the compile-time param"; + + EXPECT_EQ(d.tiles, 64u); + EXPECT_EQ(d.finalGrid, d.tiles) << "the DP fallback sets grid = tiles"; + // With no workspace the grid selection already returns tiles, so the fallback is + // credited even though it did not move the number. + EXPECT_EQ(d.selectedGrid, d.tiles); + EXPECT_EQ(d.selectedGrid, d.finalGrid); + + EXPECT_EQ(d.idealWorkspaceBytes, solution.partialTileSize(d.tiles)) + << "the launch wanted a partials region sized by the parallel grid"; + EXPECT_EQ(d.idealWorkspaceBytes, 4194304u); + EXPECT_EQ(d.requiredWorkspaceBytes, 0u); + EXPECT_FALSE(d.workspaceAllocated); + // Both sizings agree that nothing is reserved. + EXPECT_EQ(solution.requiredWorkspaceSize(problem, env.device), 0u); + + std::ostringstream os; + solution.printStreamKLaunchSummary(os, problem, d); + const std::string line = collapseSpaces(os.str()); + EXPECT_NE(line.find("changedBy = workspaceDP"), std::string::npos) + << "attribution is by which clamp produced the launch grid, not by whether " + "the number changed"; + EXPECT_NE(line.find("source = workspaceDP(runtime)"), std::string::npos); + EXPECT_NE(line.find("reduction = tree"), std::string::npos); + EXPECT_EQ(line.find("preFallback"), std::string::npos); +} + +// --------------------------------------------------------------------------- +// The launch summary is OFF unless TENSILE_DB bit 0x200000 is set. +// +// That the diagnostic is opt-in is the feature's central safety claim, so pin it +// directly on the Debug accessor the production gate reads rather than inferring +// it. Three states are checked: TENSILE_DB unset (the shipped default), the +// summary bit, and the neighbouring StreamK mode-selection bit 0x100000 -- which +// must NOT enable the summary. +// +// What this test cannot cover: the production gate is a call site inside +// ContractionSolution::solve() ("if(Debug::Instance().printStreamKLaunchSummary()) +// printStreamKLaunchSummary(std::cerr, ...)"), and solve() needs real device input +// pointers and passes a work-queue guard that throws on these mock devices, so a +// host-only unit test cannot reach it. The two host-reachable halves of the claim +// are asserted instead: the predicate the gate reads is false by default, and the +// snapshot builder this suite drives writes nothing to stderr on its own. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, LaunchSummaryDebugBitIsOffByDefault) +{ + // Whatever TENSILE_DB the suite was launched with, the guards below must put it + // back. Compared against the ambient value rather than against false, so that + // running the whole suite under TENSILE_DB=0x200000 to eyeball the summaries + // does not fail this test. + const bool ambientSummaryBit = Debug::Instance().printStreamKLaunchSummary(); + + // (1) Unset -> off. The compile-time debug mask default is 0, so an unset + // TENSILE_DB leaves every debug bit clear. + { + ScopedTensileDb noDb(nullptr); + EXPECT_FALSE(Debug::Instance().printStreamKLaunchSummary()) + << "the StreamK launch summary must be opt-in"; + } + + // (2) The documented bit turns it on. + { + ScopedTensileDb withBit("0x200000"); + ASSERT_TRUE(Debug::Instance().printStreamKLaunchSummary()) + << "TENSILE_DB=0x200000 must enable the launch summary; if this fails the " + "rest of the test proves nothing about the bit being distinct"; + } + + // (3) The neighbouring StreamK bit does NOT turn it on -- they are distinct + // opt-ins, so enabling mode-selection tracing does not also spam launch + // summaries. + { + ScopedTensileDb siblingBit("0x100000"); + EXPECT_TRUE(Debug::Instance().printStreamKModeSelection()); + EXPECT_FALSE(Debug::Instance().printStreamKLaunchSummary()); + } + + // (4) State is restored: the guards above must not leak the bit into later + // tests sharing this process-wide singleton. + EXPECT_EQ(Debug::Instance().printStreamKLaunchSummary(), ambientSummaryBit) + << "ScopedTensileDb must restore the prior TENSILE_DB value and reload"; + + // (5) Building the snapshot is silent -- nothing reaches stderr unless the + // caller explicitly prints. + { + ScopedTensileDb noDb(nullptr); + + ContractionSolution solution; + initStreamKSolution(solution, 3); + auto problem = makeGemmProblem(4096, 4224, 512); + problem.setWorkspaceSize(std::numeric_limits::max()); + auto device = makeDevice(_MI350_CHIP_ID, _CPX_CU, "mi350cpx"); + device.skDynamicGrid = 0; + + std::ostringstream captured; + std::streambuf* const savedCerr = std::cerr.rdbuf(captured.rdbuf()); + auto d = solution.computeStreamKDecisions(problem, device); + std::cerr.rdbuf(savedCerr); + + EXPECT_TRUE(captured.str().empty()) + << "computeStreamKDecisions() must not write to stderr; got: " << captured.str(); + + // ...and the printer is not self-gating: the bit gates the CALL SITE, so an + // explicit call still emits. That is what lets every other test in this file + // exercise the report with the bit clear. + std::ostringstream os; + solution.printStreamKLaunchSummary(os, problem, d); + EXPECT_NE(os.str().find("LAUNCH SUMMARY"), std::string::npos); + } +} + +// --------------------------------------------------------------------------- +// changedBy == treeBounds, as the WINNER of the attribution. +// +// The tree fixup indexes with 24-bit divide/remainder arithmetic, so the grid +// selection resets the grid to tiles when itersPerTile >= 65536, itersPerWG >= +// 65536, or tiles*itersPerTile >= 2^24. Existing tests only ever assert that this +// fallback did NOT fire; this one makes it the credited clamp. +// +// Shape: 1152x131072x131072 with a 128x128x64 macro tile. +// tiles = ceil(1152/128) * ceil(131072/128) = 9 * 1024 = 9216 +// itersPerTile = ceil(131072/64) = 2048 +// tiles*itersPerTile = 18874368 >= 16777216 -> bound tripped +// (itersPerWG = 9216*2048/64 = 294912 also exceeds 65536; on a device with at most +// 256 CUs the two bounds cannot be tripped independently, since tiles*itersPerTile +// >= 2^24 forces itersPerWG >= 2^24/256 == 65536. itersPerTile itself stays under +// 65536, which the test asserts, so the trigger is not that bound.) +// +// n == k == 131072 keeps ldb == n >= k, which the {k, n} B descriptor built by +// makeGemmProblem with strides {1, ldb} requires. +// +// The clamp sets grid = tiles, so tiles % grid == 0 and the later workspace-DP +// fallback cannot fire and steal the attribution -- asserted below rather than +// assumed. A plain mock AMDGPU with skDynamicGrid = 0 is used so the pre-clamp grid +// is exactly the CU count rather than a performance-model output. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, TreeBoundsFallbackWinsGridAttribution) +{ + ContractionSolution solution; + solution.kernelName = "test_streamk_tree_bounds"; + initStreamKSolution(solution, 3); + + auto problem = makeGemmProblem(1152, 131072, 131072); + problem.setWorkspaceSize(std::numeric_limits::max()); + + auto device = makeDevice(_MI350_CHIP_ID, _CPX_CU, "mi350cpx"); + device.skDynamicGrid = 0; + + ASSERT_FALSE(Debug::Instance().useStreamKDataParrallel()) + << "unset TENSILE_STREAMK_DATA_PARALLEL before running this suite"; + + auto d = solution.computeStreamKDecisions(problem, device); + + // The arithmetic that trips the bound, spelled out so that a shape change fails + // here with a legible message instead of silently un-tripping the fallback. + const size_t itersPerTile = problem.getItersPerTile(solution.sizeMapping); + ASSERT_EQ(d.tiles, 9216u) << "ceil(1152/128) * ceil(131072/128) = 9 * 1024"; + ASSERT_EQ(itersPerTile, 2048u) << "ceil(131072/64)"; + ASSERT_GE(d.tiles * itersPerTile, 16777216u) << "tiles*itersPerTile must reach 2^24"; + EXPECT_LT(itersPerTile, 65536u) << "itersPerTile alone is not what trips it"; + + // Anti-vacuity: the fallback actually fired, and it is the credited one. + ASSERT_TRUE(d.treeBoundsFallbackFired) << "scenario must actually trip the tree-bounds fixup"; + EXPECT_EQ(d.reduction, origami::reduction_t::tree) << "the fixup is tree-reduction only"; + EXPECT_FALSE(d.fixedGridUsed); + EXPECT_FALSE(d.clusterDPGridClamped); + // The clamp sets grid = tiles, which makes tiles % grid == 0, which is exactly + // why the later workspace-DP fallback cannot fire and take the attribution. + ASSERT_EQ(d.tiles % d.finalGrid, 0u) << "clamped grid must divide tiles"; + EXPECT_FALSE(d.workspaceDPFallbackFired) + << "grid == tiles leaves no partial tiles, so nothing to reserve"; + + // selected vs final: the clamp lives inside the grid selection, so the + // pre-fallback grid moved with it. + EXPECT_EQ(d.selectedGrid, static_cast(_CPX_CU)) + << "selection picks the CU-count grid before the fixup"; + EXPECT_EQ(d.skGridPreFallback, d.tiles); + EXPECT_EQ(d.finalGrid, d.tiles); + EXPECT_NE(d.selectedGrid, d.finalGrid); + EXPECT_EQ(d.skGrid, d.finalGrid); + EXPECT_EQ(solution.getSKGrid(problem, device, d.tiles, d.reduction), d.finalGrid); + + EXPECT_EQ(d.requiredWorkspaceBytes, 0u); + EXPECT_EQ(d.idealWorkspaceBytes, 0u); + EXPECT_FALSE(d.workspaceAllocated); + EXPECT_EQ(d.requiredWorkspaceBytes, solution.requiredWorkspaceSize(problem, device)); + + std::ostringstream os; + solution.printStreamKLaunchSummary(os, problem, d); + const std::string line = collapseSpaces(os.str()); + + EXPECT_NE(line.find("changedBy = treeBounds"), std::string::npos); + EXPECT_NE(line.find("treeBoundsFallback = yes"), std::string::npos); + // Unambiguous: no other clamp is credited or reported as fired. + EXPECT_NE(line.find("fixedGrid = no"), std::string::npos); + EXPECT_NE(line.find("workspaceDPFallback = no"), std::string::npos); + EXPECT_NE(line.find("clusterDPMulticast = no"), std::string::npos); + EXPECT_NE(line.find("selected = " + std::to_string(d.selectedGrid)), std::string::npos); + EXPECT_NE(line.find("final = " + std::to_string(d.tiles)), std::string::npos); + EXPECT_NE(line.find("preFallback = " + std::to_string(d.tiles)), std::string::npos); + + // Complement: the identical shape with a small K leaves the grid alone, so the + // fallback is a property of the bound and not of the tile count. + ContractionSolution small; + small.kernelName = "test_streamk_tree_bounds_ok"; + initStreamKSolution(small, 3); + auto smallProblem = makeGemmProblem(1152, 131072, 512); + smallProblem.setWorkspaceSize(std::numeric_limits::max()); + + auto ds = small.computeStreamKDecisions(smallProblem, device); + EXPECT_LT(ds.tiles * smallProblem.getItersPerTile(small.sizeMapping), 16777216u); + EXPECT_FALSE(ds.treeBoundsFallbackFired); + EXPECT_EQ(ds.finalGrid, static_cast(_CPX_CU)); + EXPECT_EQ(ds.selectedGrid, ds.finalGrid); +} + +// --------------------------------------------------------------------------- +// changedBy == fixedGrid, as the WINNER of the attribution. +// +// fixedGridUsed is otherwise only exercised by +// ClusterDpClampWinsAttributionOverFixedGrid, where the override deliberately +// LOSES to a later clamp. Here it is the last clamp standing: SK3, skFixedGrid +// set, clusterDim {1, 1, 1} (so no multicast clamp), K small enough that the tree +// bounds are nowhere near, and skFixedGrid chosen to divide tiles so the +// workspace-DP fallback has nothing to reserve and cannot steal the attribution. +// +// The subtlety this test exists to pin: the selected grid is captured AFTER the +// skFixedGrid override is applied, so selectedGrid == finalGrid == 32 and the +// printed "changedBy = fixedGrid" is NOT derived from selected != final. The +// evidence that the override did something is that the launch grid is 32 rather +// than the CU count the same solution and problem pick without it -- asserted via +// the baseline run at the end. +// --------------------------------------------------------------------------- +TEST(StreamKLaunchSummaryTest, FixedGridOverrideWinsGridAttribution) +{ + ContractionSolution solution; + solution.kernelName = "test_streamk_fixed_grid"; + initStreamKSolution(solution, 3); // clusterDim stays {1, 1, 1} + + // 4096x4224 -> tiles = 32*33 = 1056, and 1056 % 32 == 0, so the fixed grid + // leaves no partial tiles. K=512 -> itersPerTile = 8, far below every tree bound. + auto problem = makeGemmProblem(4096, 4224, 512); + problem.setWorkspaceSize(std::numeric_limits::max()); + + auto device = makeDevice(_MI350_CHIP_ID, _CPX_CU, "mi350cpx"); + device.skDynamicGrid = 0; + device.skFixedGrid = 32; + + ASSERT_FALSE(Debug::Instance().useStreamKDataParrallel()) + << "unset TENSILE_STREAMK_DATA_PARALLEL before running this suite"; + + auto d = solution.computeStreamKDecisions(problem, device); + + // Anti-vacuity: the override actually ran. + ASSERT_TRUE(d.fixedGridUsed) << "scenario must actually take the skFixedGrid override"; + EXPECT_EQ(d.streamKMode, 3); + EXPECT_FALSE(d.isDynamic); + EXPECT_EQ(d.reduction, origami::reduction_t::tree); + EXPECT_EQ(d.tiles, 1056u); + + // No later clamp supersedes it -- this is what makes fixedGrid the winner. + EXPECT_FALSE(d.treeBoundsFallbackFired); + EXPECT_FALSE(d.clusterDPGridClamped); + ASSERT_EQ(d.tiles % d.finalGrid, 0u) + << "the fixed grid must divide tiles, otherwise the workspace-DP fallback " + "would reserve partials and could steal the attribution"; + EXPECT_FALSE(d.workspaceDPFallbackFired); + EXPECT_EQ(d.requiredWorkspaceBytes, 0u); + EXPECT_EQ(d.idealWorkspaceBytes, 0u); + EXPECT_FALSE(d.workspaceAllocated); + EXPECT_EQ(d.requiredWorkspaceBytes, solution.requiredWorkspaceSize(problem, device)); + EXPECT_FALSE(d.dpOnly); + + // The override is applied BEFORE the selected grid is captured, so "selected" is + // already the overridden value: changedBy is attribution, not a diff. + EXPECT_EQ(d.selectedGrid, 32u) << "selection reports the overridden grid"; + EXPECT_EQ(d.skGridPreFallback, 32u); + EXPECT_EQ(d.finalGrid, 32u); + EXPECT_EQ(d.selectedGrid, d.finalGrid) + << "the fixed-grid override cannot make selected differ from final"; + EXPECT_EQ(d.skGrid, d.finalGrid); + EXPECT_EQ(solution.getSKGrid(problem, device, d.tiles, d.reduction), 32u); + + std::ostringstream os; + solution.printStreamKLaunchSummary(os, problem, d); + const std::string line = collapseSpaces(os.str()); + + EXPECT_NE(line.find("changedBy = fixedGrid"), std::string::npos); + EXPECT_NE(line.find("fixedGrid = yes"), std::string::npos); + EXPECT_NE(line.find("selected = 32"), std::string::npos); + EXPECT_NE(line.find("final = 32"), std::string::npos); + // preFallback is suppressed because no clamp inside the grid selection moved the + // grid away from what selection produced. + EXPECT_EQ(line.find("preFallback"), std::string::npos); + EXPECT_NE(line.find("treeBoundsFallback = no"), std::string::npos); + EXPECT_NE(line.find("workspaceDPFallback = no"), std::string::npos); + EXPECT_NE(line.find("clusterDPMulticast = no"), std::string::npos); + + // Baseline: same solution and problem, no override. This is the evidence that 32 + // came from skFixedGrid -- without it the launch uses the CU count. + ContractionSolution baseline; + baseline.kernelName = "test_streamk_no_fixed_grid"; + initStreamKSolution(baseline, 3); + auto baseDevice = makeDevice(_MI350_CHIP_ID, _CPX_CU, "mi350cpx"); + baseDevice.skDynamicGrid = 0; // skFixedGrid stays 0 + + auto db = baseline.computeStreamKDecisions(problem, baseDevice); + EXPECT_FALSE(db.fixedGridUsed); + EXPECT_EQ(db.selectedGrid, static_cast(_CPX_CU)); + EXPECT_EQ(db.finalGrid, static_cast(_CPX_CU)); + EXPECT_NE(db.finalGrid, d.finalGrid) << "the override is what produced grid 32"; + + std::ostringstream osb; + baseline.printStreamKLaunchSummary(osb, problem, db); + const std::string baseLine = collapseSpaces(osb.str()); + EXPECT_NE(baseLine.find("changedBy = none"), std::string::npos); + EXPECT_NE(baseLine.find("fixedGrid = no"), std::string::npos); +}