Back-port of some patches from llvm16 version of Translator.

lib/SPIRV/PreprocessMetadata.cpp

@@ -224,18 +224,38 @@ void PreprocessMetadataBase::visit(Module *M) {
     if (MDNode *Interface =
             Kernel.getMetadata(kSPIR2MD::IntelFPGAIPInterface)) {
       std::set<std::string> InterfaceStrSet;
-      // Default mode is 'csr' aka !ip_interface !N
-      //                           !N = !{!”csr”}
-      // don't emit any particular SPIR-V for it
+      for (size_t I = 0; I != Interface->getNumOperands(); ++I)
+        InterfaceStrSet.insert(getMDOperandAsString(Interface, I).str());
+
+      // ip_interface metadata will either have Register Map metadata or
+      // Streaming metadata.
+      //
+      // Register Map mode metadata:
+      // Not 'WaitForDoneWrite' mode (to be mapped on '0' literal)
+      // !ip_interface !N
+      // !N = !{!"csr"}
+      // 'WaitForDoneWrite' mode (to be mapped on '1' literal)
+      // !ip_interface !N
+      // !N = !{!"csr", !"wait_for_done_write"}
+      if (InterfaceStrSet.find("csr") != InterfaceStrSet.end()) {
+        int32_t InterfaceMode = 0;
+        if (InterfaceStrSet.find("wait_for_done_write") !=
+            InterfaceStrSet.end())
+          InterfaceMode = 1;
+        EM.addOp()
+            .add(&Kernel)
+            .add(spv::ExecutionModeRegisterMapInterfaceINTEL)
+            .add(InterfaceMode)
+            .done();
+      }
+
       // Streaming mode metadata be like:
       // Not 'stall free' mode (to be mapped on '0' literal)
       // !ip_interface !N
       // !N = !{!"streaming"}
       // 'stall free' mode (to be mapped on '1' literal)
       // !ip_interface !N
       // !N = !{!"streaming", !"stall_free_return"}
-      for (size_t I = 0; I != Interface->getNumOperands(); ++I)
-        InterfaceStrSet.insert(getMDOperandAsString(Interface, I).str());
       if (InterfaceStrSet.find("streaming") != InterfaceStrSet.end()) {
         int32_t InterfaceMode = 0;
         if (InterfaceStrSet.find("stall_free_return") != InterfaceStrSet.end())
@@ -319,17 +339,16 @@ void PreprocessMetadataBase::preprocessVectorComputeMetadata(Module *M,
           FPRoundingModeExecModeMap::map(getFPRoundingMode(Mode));
       spv::ExecutionMode ExecFloatMode =
           FPOperationModeExecModeMap::map(getFPOperationMode(Mode));
-      VCFloatTypeSizeMap::foreach (
-          [&](VCFloatType FloatType, unsigned TargetWidth) {
-            EM.addOp().add(&F).add(ExecRoundMode).add(TargetWidth).done();
-            EM.addOp().add(&F).add(ExecFloatMode).add(TargetWidth).done();
-            EM.addOp()
-                .add(&F)
-                .add(FPDenormModeExecModeMap::map(
-                    getFPDenormMode(Mode, FloatType)))
-                .add(TargetWidth)
-                .done();
-          });
+      VCFloatTypeSizeMap::foreach ([&](VCFloatType FloatType,
+                                       unsigned TargetWidth) {
+        EM.addOp().add(&F).add(ExecRoundMode).add(TargetWidth).done();
+        EM.addOp().add(&F).add(ExecFloatMode).add(TargetWidth).done();
+        EM.addOp()
+            .add(&F)
+            .add(FPDenormModeExecModeMap::map(getFPDenormMode(Mode, FloatType)))
+            .add(TargetWidth)
+            .done();
+      });
     }
     if (Attrs.hasFnAttr(kVCMetadata::VCSLMSize)) {
       SPIRVWord SLMSize = 0;

lib/SPIRV/SPIRVReader.cpp

@@ -932,6 +932,14 @@ void SPIRVToLLVM::setLLVMLoopMetadata(const LoopInstType *LM,
           "llvm.loop.intel.loopcount_avg", static_cast<int64_t>(LoopCountAvg)));
     }
   }
+  if (LC & spv::internal::LoopControlMaxReinvocationDelayINTELMask) {
+    Metadata.push_back(llvm::MDNode::get(
+        *Context, getMetadataFromNameAndParameter(
+                      "llvm.loop.intel.max_reinvocation_delay.count",
+                      LoopControlParameters[NumParam++])));
+    assert(NumParam <= LoopControlParameters.size() &&
+           "Missing loop control parameter!");
+  }
   llvm::MDNode *Node = llvm::MDNode::get(*Context, Metadata);
 
   // Set the first operand to refer itself
@@ -4091,6 +4099,27 @@ bool SPIRVToLLVM::transMetadata() {
       F->setMetadata(kSPIR2MD::FmaxMhz,
                      getMDNodeStringIntVec(Context, EM->getLiterals()));
     }
+    // Generate metadata for Intel FPGA register map interface
+    if (auto *EM =
+            BF->getExecutionMode(ExecutionModeRegisterMapInterfaceINTEL)) {
+      std::vector<uint32_t> InterfaceVec = EM->getLiterals();
+      assert(InterfaceVec.size() == 1 &&
+             "Expected RegisterMapInterfaceINTEL to have exactly 1 literal");
+      std::vector<Metadata *> InterfaceMDVec =
+          [&]() -> std::vector<Metadata *> {
+        switch (InterfaceVec[0]) {
+        case 0:
+          return {MDString::get(*Context, "csr")};
+        case 1:
+          return {MDString::get(*Context, "csr"),
+                  MDString::get(*Context, "wait_for_done_write")};
+        default:
+          llvm_unreachable("Invalid register map interface mode");
+        }
+      }();
+      F->setMetadata(kSPIR2MD::IntelFPGAIPInterface,
+                     MDNode::get(*Context, InterfaceMDVec));
+    }
     // Generate metadata for Intel FPGA streaming interface
     if (auto *EM = BF->getExecutionMode(
             internal::ExecutionModeStreamingInterfaceINTEL)) {

lib/SPIRV/SPIRVWriter.cpp

@@ -973,6 +973,8 @@ void LLVMToSPIRVBase::transVectorComputeMetadata(Function *F) {
   }
 }
 
+static void transMetadataDecorations(Metadata *MD, SPIRVEntry *Target);
+
 void LLVMToSPIRVBase::transFPGAFunctionMetadata(SPIRVFunction *BF,
                                                 Function *F) {
   if (MDNode *StallEnable = F->getMetadata(kSPIR2MD::StallEnable)) {
@@ -1021,6 +1023,10 @@ void LLVMToSPIRVBase::transFPGAFunctionMetadata(SPIRVFunction *BF,
       BF->addDecorate(new SPIRVDecoratePipelineEnableINTEL(BF, !Disable));
     }
   }
+
+  // In addition, process the decorations on the function
+  if (auto *FDecoMD = F->getMetadata(SPIRV_MD_DECORATIONS))
+    transMetadataDecorations(FDecoMD, BF);
 }
 
 void LLVMToSPIRVBase::transAuxDataInst(SPIRVFunction *BF, Function *F) {
@@ -1563,6 +1569,14 @@ LLVMToSPIRVBase::getLoopControl(const BranchInst *Branch,
           BM->addCapability(CapabilityFPGALoopControlsINTEL);
           LoopCount.Avg = getMDOperandAsInt(Node, 1);
           LoopControl |= spv::internal::LoopControlLoopCountINTELMask;
+        } else if (S == "llvm.loop.intel.max_reinvocation_delay.count") {
+          BM->addExtension(ExtensionID::SPV_INTEL_fpga_loop_controls);
+          BM->addCapability(CapabilityFPGALoopControlsINTEL);
+          size_t I = getMDOperandAsInt(Node, 1);
+          ParametersToSort.emplace_back(
+              spv::internal::LoopControlMaxReinvocationDelayINTELMask, I);
+          LoopControl |=
+              spv::internal::LoopControlMaxReinvocationDelayINTELMask;
         }
       }
     }
@@ -2820,6 +2834,12 @@ using DecorationsInfoVec =
 struct AnnotationDecorations {
   DecorationsInfoVec MemoryAttributesVec;
   DecorationsInfoVec MemoryAccessesVec;
+  DecorationsInfoVec CacheControlVec;
+
+  bool empty() {
+    return (MemoryAttributesVec.empty() && MemoryAccessesVec.empty() &&
+            CacheControlVec.empty());
+  }
 };
 
 struct IntelLSUControlsInfo {
@@ -2996,6 +3016,8 @@ AnnotationDecorations tryParseAnnotationString(SPIRVModule *BM,
       BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fpga_memory_accesses);
   const bool AllowFPGAMemAttr = BM->isAllowedToUseExtension(
       ExtensionID::SPV_INTEL_fpga_memory_attributes);
+  const bool AllowCacheControls =
+      BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_cache_controls);
 
   bool ValidDecorationFound = false;
   DecorationsInfoVec DecorationsVec;
@@ -3014,8 +3036,15 @@ AnnotationDecorations tryParseAnnotationString(SPIRVModule *BM,
       std::vector<std::string> DecValues;
       if (tryParseAnnotationDecoValues(ValueStr, DecValues)) {
         ValidDecorationFound = true;
-        DecorationsVec.emplace_back(static_cast<Decoration>(DecorationKind),
-                                    std::move(DecValues));
+
+        if (AllowCacheControls &&
+            DecorationKind == internal::DecorationCacheControlLoadINTEL) {
+          Decorates.CacheControlVec.emplace_back(
+              static_cast<Decoration>(DecorationKind), std::move(DecValues));
+        } else {
+          DecorationsVec.emplace_back(static_cast<Decoration>(DecorationKind),
+                                      std::move(DecValues));
+        }
       }
       continue;
     }
@@ -3117,7 +3146,7 @@ void addAnnotationDecorations(SPIRVEntry *E, DecorationsInfoVec &Decorations) {
       continue;
     }
 
-    switch (I.first) {
+    switch (static_cast<size_t>(I.first)) {
     case DecorationUserSemantic:
       M->getErrorLog().checkError(I.second.size() == 1,
                                   SPIRVEC_InvalidLlvmModule,
@@ -3201,6 +3230,21 @@ void addAnnotationDecorations(SPIRVEntry *E, DecorationsInfoVec &Decorations) {
         E->addDecorate(I.first, Result);
       }
     } break;
+    case spv::internal::DecorationCacheControlLoadINTEL: {
+      if (M->isAllowedToUseExtension(ExtensionID::SPV_INTEL_cache_controls)) {
+        M->getErrorLog().checkError(
+            I.second.size() == 2, SPIRVEC_InvalidLlvmModule,
+            "CacheControlLoadINTEL requires exactly 2 extra operands");
+        SPIRVWord CacheLevel = 0;
+        SPIRVWord CacheControl = 0;
+        StringRef(I.second[0]).getAsInteger(10, CacheLevel);
+        StringRef(I.second[1]).getAsInteger(10, CacheControl);
+        E->addDecorate(new SPIRVDecorateCacheControlLoadINTEL(
+            E, CacheLevel,
+            static_cast<internal::LoadCacheControlINTEL>(CacheControl)));
+      }
+    }
+
     default:
       // Other decorations are either not supported by the translator or
       // handled in other places.
@@ -3931,15 +3975,17 @@ SPIRVValue *LLVMToSPIRVBase::transIntrinsicInst(IntrinsicInst *II,
       } else {
         // Memory accesses to a standalone pointer variable
         auto *DecSubj = transValue(II->getArgOperand(0), BB);
-        if (Decorations.MemoryAccessesVec.empty())
+        if (Decorations.empty())
           DecSubj->addDecorate(new SPIRVDecorateUserSemanticAttr(
               DecSubj, AnnotationString.c_str()));
-        else
+        else {
           // Apply the LSU parameter decoration to the pointer result of an
           // instruction. Note it's the address to the accessed memory that's
           // loaded from the original pointer variable, and not the value
           // accessed by the latter.
           addAnnotationDecorations(DecSubj, Decorations.MemoryAccessesVec);
+          addAnnotationDecorations(DecSubj, Decorations.CacheControlVec);
+        }
         II->replaceAllUsesWith(II->getOperand(0));
       }
     }
@@ -4762,6 +4808,19 @@ bool LLVMToSPIRVBase::transExecutionMode() {
       case spv::ExecutionModeNumSIMDWorkitemsINTEL:
       case spv::ExecutionModeSchedulerTargetFmaxMhzINTEL:
       case spv::ExecutionModeMaxWorkDimINTEL:
+      case spv::ExecutionModeRegisterMapInterfaceINTEL: {
+        if (!BM->isAllowedToUseExtension(
+                ExtensionID::SPV_INTEL_kernel_attributes))
+          break;
+        AddSingleArgExecutionMode(static_cast<ExecutionMode>(EMode));
+        BM->addExtension(ExtensionID::SPV_INTEL_kernel_attributes);
+        BM->addCapability(CapabilityFPGAKernelAttributesINTEL);
+        // RegisterMapInterfaceINTEL mode is defined by the
+        // CapabilityFPGAKernelAttributesv2INTEL capability and that
+        // capability implicitly defines CapabilityFPGAKernelAttributesINTEL
+        if (EMode == spv::ExecutionModeRegisterMapInterfaceINTEL)
+          BM->addCapability(CapabilityFPGAKernelAttributesv2INTEL);
+      } break;
       case spv::internal::ExecutionModeStreamingInterfaceINTEL: {
         if (!BM->isAllowedToUseExtension(
                 ExtensionID::SPV_INTEL_kernel_attributes))

lib/SPIRV/libSPIRV/SPIRVEntry.cpp

@@ -626,6 +626,7 @@ void SPIRVExecutionMode::decode(std::istream &I) {
   case ExecutionModeMaxWorkDimINTEL:
   case ExecutionModeNumSIMDWorkitemsINTEL:
   case ExecutionModeSchedulerTargetFmaxMhzINTEL:
+  case ExecutionModeRegisterMapInterfaceINTEL:
   case internal::ExecutionModeStreamingInterfaceINTEL:
     WordLiterals.resize(1);
     break;

lib/SPIRV/libSPIRV/SPIRVEnum.h

@@ -273,6 +273,8 @@ template <> inline void SPIRVMap<SPIRVExecutionModeKind, SPIRVCapVec>::init() {
                {CapabilityVectorComputeINTEL});
   ADD_VEC_INIT(internal::ExecutionModeFastCompositeKernelINTEL,
                {internal::CapabilityFastCompositeINTEL});
+  ADD_VEC_INIT(ExecutionModeRegisterMapInterfaceINTEL,
+               {CapabilityFPGAKernelAttributesv2INTEL});
   ADD_VEC_INIT(internal::ExecutionModeStreamingInterfaceINTEL,
                {CapabilityFPGAKernelAttributesINTEL});
   ADD_VEC_INIT(ExecutionModeNamedBarrierCountINTEL,

lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h

@@ -71,6 +71,7 @@ inline bool isValid(spv::ExecutionModel V) {
   case ExecutionModelClosestHitKHR:
   case ExecutionModelMissKHR:
   case ExecutionModelCallableKHR:
+  case ExecutionModeRegisterMapInterfaceINTEL:
   case internal::ExecutionModeStreamingInterfaceINTEL:
     return true;
   default:

lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h

@@ -568,6 +568,7 @@ template <> inline void SPIRVMap<Capability, std::string>::init() {
   add(CapabilityFPGALoopControlsINTEL, "FPGALoopControlsINTEL");
   add(CapabilityKernelAttributesINTEL, "KernelAttributesINTEL");
   add(CapabilityFPGAKernelAttributesINTEL, "FPGAKernelAttributesINTEL");
+  add(CapabilityFPGAKernelAttributesv2INTEL, "FPGAKernelAttributesv2INTEL");
   add(CapabilityFPGAMemoryAccessesINTEL, "FPGAMemoryAccessesINTEL");
   add(CapabilityFPGAClusterAttributesINTEL, "FPGAClusterAttributesINTEL");
   add(CapabilityLoopFuseINTEL, "LoopFuseINTEL");

lib/SPIRV/libSPIRV/spirv_internal.hpp

@@ -123,7 +123,10 @@ enum InternalExecutionMode {
   IExecModeStreamingInterfaceINTEL = 6154
 };
 
-enum InternalLoopControlMask { ILoopControlLoopCountINTELMask = 0x1000000 };
+enum InternalLoopControlMask {
+  ILoopControlLoopCountINTELMask = 0x1000000,
+  ILoopControlMaxReinvocationDelayINTELMask = 0x2000000
+};
 
 constexpr LinkageType LinkageTypeInternal =
     static_cast<LinkageType>(ILTInternal);
@@ -293,6 +296,8 @@ constexpr ExecutionMode ExecutionModeStreamingInterfaceINTEL =
 
 constexpr LoopControlMask LoopControlLoopCountINTELMask =
     static_cast<LoopControlMask>(ILoopControlLoopCountINTELMask);
+static const LoopControlMask LoopControlMaxReinvocationDelayINTELMask =
+    static_cast<LoopControlMask>(ILoopControlMaxReinvocationDelayINTELMask);
 
 } // namespace internal
 } // namespace spv