update GCNHazardRecognizer

llvm · Jan 24, 2024 · d3f9c4f · d3f9c4f
1 parent fd460a7
commit d3f9c4f
Show file tree

Hide file tree

Showing 3 changed files with 91 additions and 93 deletions.
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -1722,7 +1722,7 @@ bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) {
   const SIInstrInfo *TII = ST.getInstrInfo();
   const SIRegisterInfo *TRI = ST.getRegisterInfo();
 
-  auto IsHazardFn = [MI, TII, TRI](const MachineInstr &I) {
+  auto IsHazardFn = [MI, TII, TRI, this](const MachineInstr &I) {
     if (!SIInstrInfo::isWMMA(I) && !SIInstrInfo::isSWMMAC(I))
       return false;
 
@@ -1741,6 +1741,18 @@ bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) {
       return true;
     }
 
+    // GFX12+ allows overlap of matrix C with PrevDstReg (hardware will stall)
+    // but Index can't overlap with PrevDstReg.
+    if (AMDGPU::isGFX12Plus(ST)) {
+      if (SIInstrInfo::isSWMMAC(*MI)) {
+        const Register CurIndex =
+            TII->getNamedOperand(*MI, AMDGPU::OpName::src2)->getReg();
+        if (TRI->regsOverlap(PrevDstReg, CurIndex))
+          return true;
+      }
+      return false;
+    }
+
     return false;
   };