refactor(stinkytofu): derive the WMMA refill line and rank prefetch above the WMMA queue

shared/stinkytofu/docs/developer/dag-wait-and-prefetch-placement.md

@@ -66,7 +66,7 @@ is in [Scheduler tuning parameters](../user/scheduler-tuning-parameters.md). `st
 | `EvenSpreadFillers` | true | – | filler quota per window |
 | `WarGateWmmas` | -1 | `--war-gate-wmmas=N` | WMMA-src → ds_load overwrite gap; -1 = derived |
 | `WmmaQueueDepth` | 8 (`stinkytofu-opt`: 1) | `--wmma-queue-depth=N` | WMMAs outstanding in the matrix pipe (it buffers ~8); a WMMA is appended whenever fewer are outstanding. 1 = one WMMA at a time (queue model off). > 1 also merges the waits of back-to-back WMMAs onto the first one |
-| `WmmaQueueCoverCycles` | 56 (`stinkytofu-opt`: 0 = off) | `--wmma-queue-cover-cycles=N` | cycles of queued WMMA work that must remain before a ds_load, filler or tensor_load may issue; below it, with room in the queue, the next ready WMMA goes first. Ignored at depth 1; forced picks (a promoted barrier) are not held |
+| `WmmaQueueCoverCycles` | -1 = automatic, (`WmmaQueueDepth` - 1) x the next WMMA's latency (`stinkytofu-opt`: 0 = off) | `--wmma-queue-cover-cycles=N` | cycles of queued WMMA work that must remain before a ds_load, filler or tensor_load may issue; below it, with room in the queue, the next ready WMMA goes first. Ignored at depth 1; forced picks (a promoted barrier) are not held |
 
 ## Measured effect
 

shared/stinkytofu/docs/user/scheduler-tuning-parameters.md

@@ -21,7 +21,7 @@ Defaults below are the production (TensileLite → `Gfx1250Backend`) defaults.
 defaults instead, where `EvenSpreadFillers`, the prefetch lead,
 the wait-alu hold and the knob heuristic are off, `DsSpread` is on, and the WMMA queue model is
 off (`WmmaQueueDepth` 1, `WmmaQueueCoverCycles` 0, `DsReadCyclesPerLoad` 0). The production defaults
-turn the queue on (8, 56, 6), so `stinkytofu-opt` output matches TensileLite only with those three
+turn the queue on (8, -1, 6), where -1 is the automatic refill line, so `stinkytofu-opt` output matches TensileLite only with those three
 set.
 
 ### How an unset value is resolved
@@ -55,9 +55,9 @@ Each parameter resolves independently, first match wins:
 | Module option | Default | CLI | Meaning |
 |---|---|---|---|
 | `WmmaQueueDepth` | 8 (`stinkytofu-opt`: 1) | `--wmma-queue-depth=N` | Max WMMAs outstanding in the pipe. A WMMA is appended whenever fewer than N are outstanding; it waits only when the queue is full. 1 = one WMMA at a time (queue model off). |
-| `WmmaQueueCoverCycles` | 56 (`stinkytofu-opt`: 0 = off) | `--wmma-queue-cover-cycles=N` | Cycles of queued WMMA work that must remain before a ds_load, filler or `tensor_load` may issue. Below N, and with room in the queue, the next ready WMMA goes first, so the pipe never runs dry. Forced picks (a promoted barrier) are not held. Useful range is up to about 64 (the pipe buffers ~8 WMMAs of 8 cycles). |
+| `WmmaQueueCoverCycles` | -1 = automatic, (`WmmaQueueDepth` - 1) x the next WMMA's latency (`stinkytofu-opt`: 0 = off) | `--wmma-queue-cover-cycles=N` | Cycles of queued WMMA work that must remain before a ds_load, filler or `tensor_load` may issue. Below N, and with room in the queue, the next ready WMMA goes first, so the pipe never runs dry. Forced picks (a promoted barrier) are not held. Useful range is up to about 64 (the pipe buffers ~8 WMMAs of 8 cycles). |
 
-The queue model runs only with **both** `WmmaQueueDepth > 1` and `WmmaQueueCoverCycles > 0`, which
+The queue model runs only with **both** `WmmaQueueDepth > 1` and `WmmaQueueCoverCycles != 0`, which
 is the production default. With either one off, the schedule is the original single-window one.
 
 ```

shared/stinkytofu/include/stinkytofu/bindings/python/Module.hpp

@@ -149,8 +149,8 @@
     X(DsBatchWmmas, int, 4)               /* WMMAs per ds_load pacing batch */                     \
     X(DsCatchUpPace, int, 2)              /* extra ds_loads per batch when behind the even line */ \
     X(WmmaQueueDepth, int, 8)             /* WMMAs the matrix pipe buffers; 1 = queue model off */ \
-    X(WmmaQueueCoverCycles, int, 56) /* queued WMMA cover kept (8 WMMAs of 8-14 cycles ~ 50-56) */ \
-    X(DsReadCyclesPerLoad, int, 6)   /* LDS return rate (cycles per ds_load); 0 = off */           \
+    X(WmmaQueueCoverCycles, int, -1)      /* queued WMMA cover kept; < 0 = automatic, 0 = off */   \
+    X(DsReadCyclesPerLoad, int, 6)        /* LDS return rate (cycles per ds_load); 0 = off */      \
     X(ClusterBarrierRule3SignalLeadCycles, int, -1)                                                \
     X(ReduceClusterBarrierPerLoop, int, -1)                                                        \
     X(TensorLoadDsLoadGapCycles, int, 64)

shared/stinkytofu/include/stinkytofu/core/Types.hpp

@@ -106,7 +106,8 @@ struct PassFeatureConfig {
         /// Cycles of queued WMMA work that must remain before a non-WMMA pick (ds_load,
         /// filler, tensor_load) may issue; below it, and with room in the queue, the next
         /// ready WMMA goes first so the pipe never runs dry. Picks the scheduler is forced
-        /// to make (a promoted barrier) are not held. 0 = off; ignored at depth 1.
+        /// to make (a promoted barrier) are not held. < 0 = automatic: one WMMA short of a full
+        /// queue, (wmmaQueueDepth - 1) x the next WMMA's latency. 0 = off; ignored at depth 1.
         int wmmaQueueCoverCycles = 0;
         /// How many WMMAs one refill issues once the queue drops below the refill line
         /// (wmmaQueueCoverCycles). Fixed: wmmaBatchSize of them. Dynamic: as many as it takes

shared/stinkytofu/src/pipeline/backend/Gfx1250Backend.cpp

@@ -232,7 +232,7 @@ bool buildGfx1250Pipeline(ModulePassManager& mpm, StinkyAsmModule& module, const
                     moduleOptions.EnableLoopCarriedTokenDeps;
                 // A deep WMMA queue issues WMMAs back-to-back: one wait before the first.
                 waitCntOptions.mergeWaitsInWmmaRuns =
-                    moduleOptions.WmmaQueueDepth > 1 && moduleOptions.WmmaQueueCoverCycles > 0;
+                    moduleOptions.WmmaQueueDepth > 1 && moduleOptions.WmmaQueueCoverCycles != 0;
                 innerPM.addPass(createStinkyWaitCntInsertionPass(waitCntOptions));
                 if (runScheduler) innerPM.addPass(createRemoveDscntPass());
             }

shared/stinkytofu/src/transforms/asm/dag/AGENTS.md

@@ -36,15 +36,17 @@ When two rules want different things, the higher level wins. `PlacementLevel` in
 | --- | --- | --- |
 | L1 | `Barrier` | The barrier lands where it is planned. |
 | L2 | `LastDs` | The last ds before a barrier issues in time for that barrier. |
-| L3 | `WmmaQueue` | The WMMA queue stays busy: no window is wasted (the refill). |
-| L4 | `Prefetch` | Prefetches and global loads issue at their planned window. |
+| L3 | `Prefetch` | Prefetches and global loads issue at their planned window. |
+| L4 | `WmmaQueue` | The WMMA queue stays busy: no window is wasted (the refill). |
 | L5 | `DsPacer` | ds loads are spread over the region (`DsSpreadPacer`). |
 | L6 | `Filler` | SALU and VALU fill whatever slots are left. |
 
 Consequences:
 
 - A lower level never vetoes a higher one. The ds pacer (L5) must not delay the last ds
-  before a barrier (L2). A filler (L6) must not take a prefetch's window (L4).
+  before a barrier (L2). A filler (L6) must not take a prefetch's window (L3).
+- A prefetch goes before the refill. Refill first measured 1.8% slower on `mxf8_tn_medium`,
+  the only perf yaml it changes.
 - A barrier issues when its WMMA threshold is met. Do not hold it for an extra tail of
   WMMAs after the threshold. The threshold includes the last WMMA a feeding ds_load depends
   on and that load's drain (`depThreshold`). Two exclusive groups whose claim windows

shared/stinkytofu/src/transforms/asm/dag/CDNA5.hpp

@@ -469,37 +469,58 @@ class CDNA5ReadyQueue : public ReadyQueue {
     // The queue model runs only with depth > 1 and a cover > 0; otherwise the depth is 1.
     int wmmaQueueDepth() const {
         const auto& f = getPassContext().getPassFeatureConfig().dagFeatures;
-        return f.wmmaQueueCoverCycles > 0 ? std::max(1, f.wmmaQueueDepth) : 1;
+        return f.wmmaQueueCoverCycles != 0 ? std::max(1, f.wmmaQueueDepth) : 1;
     }
-    // Cycles of queued WMMA work needed before a non-WMMA pick may issue (0 = off).
-    int wmmaQueueCover() const {
-        if (wmmaQueueDepth() <= 1) return 0;
-        return std::max(0,
-                        getPassContext().getPassFeatureConfig().dagFeatures.wmmaQueueCoverCycles);
+    bool wmmaQueueOn() const {
+        return wmmaQueueDepth() > 1;
+    }
+    // The refill line: cycles of queued WMMA work needed before a non-WMMA pick may issue.
+    // Automatic (the option below 0): one WMMA short of a full queue, sized by the next WMMA's
+    // latency, so a region that mixes WMMA latencies gets the line of the WMMA it refills for.
+    int wmmaQueueCover(int wmmaLatency) const {
+        if (!wmmaQueueOn()) return 0;
+        const int cover = getPassContext().getPassFeatureConfig().dagFeatures.wmmaQueueCoverCycles;
+        if (cover >= 0) return cover;
+        return (wmmaQueueDepth() - 1) * std::max(1, wmmaLatency);
     }
     // WMMA refill (see PassFeatureConfig::DagFeatures::wmmaRefillMode). A refill starts when the
-    // queued cover drops below the refill line (wmmaQueueCover()) with room in the queue, and
+    // queued cover drops below the refill line (wmmaQueueCover) with room in the queue, and
     // keeps issuing WMMAs ahead of other work until it is done: Fixed after wmmaBatchSize
-    // WMMAs, Dynamic once the queued cover reaches the refill target. Returns whether a refill
-    // is under way. wmmaLatency is the next WMMA's latency (sizes the auto target).
-    bool wmmaRefillActive(int wmmaLatency) {
-        if (wmmaQueueCover() <= 0) return false;
+    // WMMAs, Dynamic once the queued cover reaches the refill target. wmmaLatency is the next
+    // WMMA's latency (sizes the auto target).
+    // wmmaRefillState() reads the state and says whether a refill is under way and how many
+    // WMMAs it still has; wmmaRefillActive() also records that state.
+    struct WmmaRefillState {
+        bool active = false;
+        int left = 0;
+    };
+    WmmaRefillState wmmaRefillState(int wmmaLatency) const {
+        if (!wmmaQueueOn()) return {wmmaRefilling_, wmmaRefillLeft_};
         const auto& f = getPassContext().getPassFeatureConfig().dagFeatures;
         const bool room = outstandingWmmas() < wmmaQueueDepth();
-        if (wmmaRefilling_) {
+        WmmaRefillState st{wmmaRefilling_, wmmaRefillLeft_};
+        if (st.active) {
             const int target = f.wmmaRefillToCycles > 0
                                    ? f.wmmaRefillToCycles
                                    : wmmaQueueDepth() * std::max(1, wmmaLatency);
             const bool more = f.wmmaRefillMode == PassFeatureConfig::WmmaRefillMode::Fixed
-                                  ? wmmaRefillLeft_ > 0
+                                  ? st.left > 0
                                   : queuedCoverCycles() < target;
-            if (!room || !more) wmmaRefilling_ = false;
+            if (!room || !more) st.active = false;
         }
-        if (!wmmaRefilling_ && room && queuedCoverCycles() < wmmaQueueCover()) {
-            wmmaRefilling_ = true;
-            wmmaRefillLeft_ = std::max(1, f.wmmaBatchSize);
+        if (!st.active && room && queuedCoverCycles() < wmmaQueueCover(wmmaLatency)) {
+            st.active = true;
+            st.left = std::max(1, f.wmmaBatchSize);
         }
-        return wmmaRefilling_;
+        return st;
+    }
+    // Whether a refill is under way, as the pick sees it.
+    bool wmmaRefillActive(int wmmaLatency) {
+        if (!wmmaQueueOn()) return false;
+        const WmmaRefillState st = wmmaRefillState(wmmaLatency);
+        wmmaRefilling_ = st.active;
+        wmmaRefillLeft_ = st.left;
+        return st.active;
     }
     // Cycles until the pipe has run everything queued.
     int queuedCoverCycles() const {
@@ -967,7 +988,7 @@ class CDNA5ReadyQueue : public ReadyQueue {
     // new rule is a promoteDue*() function plus one row in decidePromote()'s rule
     // table at its level; a rule at an in-between level is a new enumerator.
     // A rule returns true once it has set promotedNode_.
-    enum class PlacementLevel { Barrier, LastDs, WmmaQueue, Prefetch, DsPacer, Filler };
+    enum class PlacementLevel { Barrier, LastDs, Prefetch, WmmaQueue, DsPacer, Filler };
     void decidePromote();
     void promote(PromotePhase phase, DAGNode* node, int kind = -1);
     bool promoteDueBarrier();
@@ -1083,7 +1104,7 @@ void CDNA5ReadyQueue::resetActiveWindow() {
 // the WAR gate (pointer-free; the nodes die with the region). Queue model only.
 void CDNA5ReadyQueue::carryQueuedWar() {
     std::vector<CarriedWar> next;
-    if (wmmaQueueCover() > 0) {
+    if (wmmaQueueOn()) {
         for (CarriedWar& c : carriedWar_) {
             if (c.remaining > clock_) next.push_back({std::move(c.srcs), c.remaining - clock_});
         }
@@ -1151,7 +1172,7 @@ int CDNA5ReadyQueue::preFirstWmmaStallShadow() const {
 void CDNA5ReadyQueue::advanceTime(int cycles, TimeKind kind) {
     if (kind == TimeKind::ValuIssue) {
         cycles = computeValuAdvanceCycles(cycles);  // skips the slots a VALU cannot use
-    } else if (kind == TimeKind::Issue && wmmaQueueCover() > 0) {
+    } else if (kind == TimeKind::Issue && wmmaQueueOn()) {
         int at = coIssueCyclePos_;
         for (int c = 0; c < cycles; ++c) {
             while (isBlockedCycle(at)) ++at;
@@ -2541,6 +2562,8 @@ DAGNode* CDNA5ReadyQueue::pickOne() {
         const bool refillActive = wmmaRefillActive(bestWMMA->inst->latencyCycles);
         // The refill only hides a stall: a non-WMMA pick that carries no wait issues now, so a
         // ready ds_load at a region start does not wait for the queue to fill.
+        // A promoted prefetch (decidePromote) goes before the refill: refill first measured 1.8%
+        // slower on mxf8_tn_medium, the only yaml it changes.
         const bool preemptForQueue = refillActive && (!hasPickableNonWmma || pickWait > 0);
         const bool blockWmmaForActiveWindow = !preemptForQueue && !quotaClosedWindow &&
                                               (coIssueCyclePos_ < activeWmmaLatency_) &&
@@ -2638,7 +2661,7 @@ DAGNode* CDNA5ReadyQueue::pickOne() {
         if (bestLatency > 0 && heldMemOpPending())
             if (DAGNode* n = pickFallback()) return n;
         // The queued WMMA can only start once its sources arrive: that stall elapses.
-        if (wmmaQueueCover() > 0 && bestLatency > 0) advanceTime(bestLatency, TimeKind::Elapsed);
+        if (wmmaQueueOn() && bestLatency > 0) advanceTime(bestLatency, TimeKind::Elapsed);
         DAGNode* node = pickOneFromWMMA(bestWMMA);
         return rememberPick(node);
     }

shared/stinkytofu/tests/unit/asm/CDNA5ReadyQueueTest.cpp

@@ -148,6 +148,9 @@ struct CDNA5ReadyQueueTestPeer {
     static void carry(CDNA5ReadyQueue& q) {
         q.carryQueuedWar();
     }
+    static int cover(const CDNA5ReadyQueue& q, int wmmaLatency) {
+        return q.wmmaQueueCover(wmmaLatency);
+    }
     static void resetWindow(CDNA5ReadyQueue& q) {
         q.resetActiveWindow();
     }
@@ -361,6 +364,29 @@ TEST_F(CDNA5ReadyQueueTest, AdvanceTimeReadsItsArgumentByKind) {
 
 // A queued WMMA keeps reading its sources after a region cut. The WAR gate must still hold a
 // ds_load into those registers until the pipe has read them, then release it.
+// The automatic refill line is one WMMA short of a full queue, sized by the latency of the WMMA
+// the refill is for, not by the region's first WMMA: depth 8 gives 28 for a 4-cycle WMMA and 56
+// for an 8-cycle one. An explicit cover is the same for both and 0 turns the queue model off.
+TEST_F(CDNA5ReadyQueueTest, AutomaticRefillLineFollowsTheNextWmmaLatency) {
+    using Peer = CDNA5ReadyQueueTestPeer;
+    PassContext autoCtx = makeQueueCtx(8, -1);
+    CDNA5ReadyQueue autoQueue(autoCtx);
+    EXPECT_EQ(Peer::cover(autoQueue, 4), 28);
+    EXPECT_EQ(Peer::cover(autoQueue, 8), 56);
+
+    PassContext explicitCtx = makeQueueCtx(8, 40);
+    CDNA5ReadyQueue explicitQueue(explicitCtx);
+    EXPECT_EQ(Peer::cover(explicitQueue, 4), 40);
+    EXPECT_EQ(Peer::cover(explicitQueue, 8), 40);
+
+    PassContext offCtx = makeQueueCtx(8, 0);
+    CDNA5ReadyQueue offQueue(offCtx);
+    EXPECT_EQ(Peer::cover(offQueue, 8), 0);
+    PassContext depthOneCtx = makeQueueCtx(1, -1);
+    CDNA5ReadyQueue depthOneQueue(depthOneCtx);
+    EXPECT_EQ(Peer::cover(depthOneQueue, 8), 0);
+}
+
 TEST_F(CDNA5ReadyQueueTest, QueuedWmmaWarStateSurvivesARegionCut) {
     PassContext ctx = makeQueueCtx(8, 16);
     CDNA5ReadyQueue queue(ctx);

shared/stinkytofu/tests/unit/asm/DAGSchedulerPassTest.cpp

@@ -2467,6 +2467,32 @@ TEST_F(DAGSchedulerPassTest, WmmaQueue_LongRunKeepsEveryInstruction) {
     EXPECT_EQ(std::count(shape.begin(), shape.end(), 'd'), 12) << shape;
 }
 
+// An automatic refill line (cover < 0) is one WMMA short of a full queue: the same schedule as
+// asking for (depth - 1) x the next WMMA's latency by hand.
+TEST_F(DAGSchedulerPassTest, WmmaQueue_AutomaticCoverIsOneWmmaShortOfFull) {
+    constexpr int kDepth = 4;
+    std::string shapes[2];
+    for (int v = 0; v < 2; ++v) {
+        SetUp();
+        for (int i = 0; i < 8; i++) createMovableDsLoad(200 + i * 4, 80, i + 1);
+        int latency = 0;
+        for (int i = 0; i < 8; i++)
+            latency = createWmmaF32_16x16x16_bf16(8 * i, 100 + 8 * i)->latencyCycles;
+        PassContext ctx;
+        ctx.setGemmTileConfig(config);
+        PassFeatureConfig pfc;
+        pfc.loopConfig.unrollGemm = true;
+        pfc.dagFeatures.dsSpread = false;
+        pfc.dagFeatures.dsReadQueueDepth = 2;
+        pfc.dagFeatures.wmmaQueueDepth = kDepth;
+        pfc.dagFeatures.wmmaQueueCoverCycles = v == 0 ? -1 : (kDepth - 1) * latency;
+        ctx.setPassFeatureConfig(pfc);
+        pass->run(*func, ctx, am);
+        shapes[v] = wdShape(mnemonicSequence(*bb));
+    }
+    EXPECT_EQ(shapes[0], shapes[1]);
+}
+
 // Depth 4, cover 16: ds_loads go while at least 16 cycles of WMMA work are queued, and the
 // next WMMA goes first once less is, so the queue never runs dry.
 TEST_F(DAGSchedulerPassTest, WmmaQueue_KeepsTheQueueFed) {