Add support for SPV_INTEL_usm_storage_classes extension

With this extension 2 new Storage Classes are introduced: DeviceOnlyINTEL and HostOnlyINTEL appropriately mapped on global_device and global_host SYCL/OpenCL address spaces which are part of SYCL_INTEL_usm_address_spaces extension. In case if device doesn't support SPV_INTEL_usm_storage_classes extension global_device and global_host address spaces are lowered to just global address space and therefore new Storage Classes are lowered to CrossWorkgroup storage class. Also this patch introduces a flag 'enable-usm-addrspaces' that enables generation of global_device and global_host address spaces during reversed translation. Co-authored-by: Viktoria Maksimova <[email protected]> Signed-off-by: Dmitry Sidorov <[email protected]> Signed-off-by: Viktoria Maksimova <[email protected]>
KhronosGroup · Jun 18, 2020 · 1b33fbf · 1b33fbf
1 parent f8b112a
commit 1b33fbf
Show file tree

Hide file tree

Showing 18 changed files with 375 additions and 13 deletions.
diff --git a/include/LLVMSPIRVExtensions.inc b/include/LLVMSPIRVExtensions.inc
@@ -21,3 +21,4 @@ EXT(SPV_INTEL_arbitrary_precision_integers)
 EXT(SPV_INTEL_optimization_hints)
 EXT(SPV_INTEL_float_controls2)
 EXT(SPV_INTEL_vector_compute)
+EXT(SPV_INTEL_usm_storage_classes)
diff --git a/include/LLVMSPIRVOpts.h b/include/LLVMSPIRVOpts.h
@@ -137,6 +137,10 @@ class TranslatorOpts {
 
   FPContractMode getFPContractMode() const { return FPCMode; }
 
+  void enableUsmAddrspaces() { EnableUsmAddrspaces = true; }
+
+  bool isUsmAddrspacesEnabled() { return EnableUsmAddrspaces; }
+
 private:
   // Common translation options
   VersionNumber MaxVersion = VersionNumber::MaximumVersion;
@@ -159,6 +163,13 @@ class TranslatorOpts {
   // - FPContractMode::Fast allows *all* operations to be contracted
   //   for all entry points
   FPContractMode FPCMode = FPContractMode::On;
+  // Controls translation of USM storage classes to USM address spaces.
+  //
+  // - true allows to use USM storage/addrspace
+  //
+  // - false prohibit translation of USM storage classess to USM addrspaces.
+  //   Instead, USM storage is translated to common global addrspace.
+  bool EnableUsmAddrspaces = false;
 };
 
 } // namespace SPIRV

diff --git a/lib/SPIRV/Mangler/ParameterType.h b/lib/SPIRV/Mangler/ParameterType.h
@@ -136,7 +136,9 @@ enum TypeAttributeEnum {
   ATTR_CONSTANT,
   ATTR_LOCAL,
   ATTR_GENERIC,
-  ATTR_ADDR_SPACE_LAST = ATTR_GENERIC,
+  ATTR_GLOBAL_DEVICE,
+  ATTR_GLOBAL_HOST,
+  ATTR_ADDR_SPACE_LAST = ATTR_GLOBAL_HOST,
   ATTR_NONE,
   ATTR_NUM = ATTR_NONE
 };

diff --git a/lib/SPIRV/OCLUtil.cpp b/lib/SPIRV/OCLUtil.cpp
@@ -349,6 +349,10 @@ static SPIR::TypeAttributeEnum mapAddrSpaceEnums(SPIRAddressSpace Addrspace) {
     return SPIR::ATTR_LOCAL;
   case SPIRAS_Generic:
     return SPIR::ATTR_GENERIC;
+  case SPIRAS_GlobalDevice:
+    return SPIR::ATTR_GLOBAL_DEVICE;
+  case SPIRAS_GlobalHost:
+    return SPIR::ATTR_GLOBAL_HOST;
   default:
     llvm_unreachable("Invalid addrspace enum member");
   }

diff --git a/lib/SPIRV/SPIRVInternal.h b/lib/SPIRV/SPIRVInternal.h
@@ -191,6 +191,8 @@ enum SPIRAddressSpace {
   SPIRAS_Input,
   SPIRAS_Output,
   SPIRAS_Count,
+  SPIRAS_GlobalDevice = 11,
+  SPIRAS_GlobalHost = 12,
 };
 
 template <> inline void SPIRVMap<SPIRAddressSpace, std::string>::init() {
@@ -200,6 +202,8 @@ template <> inline void SPIRVMap<SPIRAddressSpace, std::string>::init() {
   add(SPIRAS_Local, "Local");
   add(SPIRAS_Generic, "Generic");
   add(SPIRAS_Input, "Input");
+  add(SPIRAS_GlobalDevice, "GlobalDevice");
+  add(SPIRAS_GlobalHost, "GlobalHost");
 }
 typedef SPIRVMap<SPIRAddressSpace, SPIRVStorageClassKind>
     SPIRAddrSpaceCapitalizedNameMap;
@@ -212,6 +216,8 @@ inline void SPIRVMap<SPIRAddressSpace, SPIRVStorageClassKind>::init() {
   add(SPIRAS_Local, StorageClassWorkgroup);
   add(SPIRAS_Generic, StorageClassGeneric);
   add(SPIRAS_Input, StorageClassInput);
+  add(SPIRAS_GlobalDevice, StorageClassDeviceOnlyINTEL);
+  add(SPIRAS_GlobalHost, StorageClassHostOnlyINTEL);
 }
 typedef SPIRVMap<SPIRAddressSpace, SPIRVStorageClassKind> SPIRSPIRVAddrSpaceMap;
 

diff --git a/lib/SPIRV/SPIRVReader.cpp b/lib/SPIRV/SPIRVReader.cpp
@@ -385,11 +385,18 @@ Type *SPIRVToLLVM::transType(SPIRVType *T, bool IsClassMember) {
   case OpTypeArray:
     return mapType(T, ArrayType::get(transType(T->getArrayElementType()),
                                      T->getArrayLength()));
-  case OpTypePointer:
+  case OpTypePointer: {
+    SPIRAddressSpace AS =
+      SPIRSPIRVAddrSpaceMap::rmap(T->getPointerStorageClass());
+    // If generation of SYCL USM address spaces isn't allowed in the reversed
+    // translation - transform them into global address space
+    if (!BM->isUsmAddrspacesEnabled() && (AS == SPIRAS_GlobalDevice ||
+                                          AS == SPIRAS_GlobalHost))
+      AS = SPIRAS_Global;
     return mapType(
         T, PointerType::get(
-               transType(T->getPointerElementType(), IsClassMember),
-               SPIRSPIRVAddrSpaceMap::rmap(T->getPointerStorageClass())));
+               transType(T->getPointerElementType(), IsClassMember), AS));
+  }
   case OpTypeVector:
     return mapType(T, VectorType::get(transType(T->getVectorComponentType()),
                                       T->getVectorComponentCount()));
@@ -931,8 +938,19 @@ Value *SPIRVToLLVM::transConvertInst(SPIRVValue *BV, Function *F,
   switch (BC->getOpCode()) {
   case OpPtrCastToGeneric:
   case OpGenericCastToPtr:
+  case OpPtrCastToCrossWorkgroupINTEL:
+  case OpCrossWorkgroupCastToPtrINTEL: {
+    // If module has pointers with DeviceOnlyINTEL and HostOnlyINTEL storage
+    // classes and if during reversed translation 'enable-usm-addrspaces'
+    // wasn't passed there will be a situation, when global_device/global_host
+    // address space will be lowered to just global address space. If there also
+    // is an addrspacecast - we need to replace it with source pointer.
+    if (Src->getType()->getPointerAddressSpace() ==
+        Dst->getPointerAddressSpace())
+      return Src;
     CO = Instruction::AddrSpaceCast;
     break;
+  }
   case OpSConvert:
     CO = IsExt ? Instruction::SExt : Instruction::Trunc;
     break;
@@ -3358,15 +3376,20 @@ bool SPIRVToLLVM::transOCLMetadata(SPIRVFunction *BF) {
   if (F->getCallingConv() != CallingConv::SPIR_KERNEL)
     return true;
 
-  // Generate metadata for kernel_arg_address_spaces
+  // Generate metadata for kernel_arg_addr_space
   addOCLKernelArgumentMetadata(
       Context, SPIR_MD_KERNEL_ARG_ADDR_SPACE, BF, F,
       [=](SPIRVFunctionParameter *Arg) {
         SPIRVType *ArgTy = Arg->getType();
         SPIRAddressSpace AS = SPIRAS_Private;
-        if (ArgTy->isTypePointer())
+        if (ArgTy->isTypePointer()) {
           AS = SPIRSPIRVAddrSpaceMap::rmap(ArgTy->getPointerStorageClass());
-        else if (ArgTy->isTypeOCLImage() || ArgTy->isTypePipe())
+          // If generation of SYCL USM address spaces isn't allowed in the reversed
+          // translation - transform them into global address space
+          if (!BM->isUsmAddrspacesEnabled() && (AS == SPIRAS_GlobalDevice ||
+                                                AS == SPIRAS_GlobalHost))
+            AS = SPIRAS_Global;
+        } else if (ArgTy->isTypeOCLImage() || ArgTy->isTypePipe())
           AS = SPIRAS_Global;
         return ConstantAsMetadata::get(
             ConstantInt::get(Type::getInt32Ty(*Context), AS));

diff --git a/lib/SPIRV/SPIRVWriter.cpp b/lib/SPIRV/SPIRVWriter.cpp
@@ -303,6 +303,14 @@ SPIRVType *LLVMToSPIRV::transType(Type *T) {
       return nullptr;
     auto ST = dyn_cast<StructType>(ET);
     auto AddrSpc = T->getPointerAddressSpace();
+    // Lower global_device and global_host address spaces that were added in
+    // SYCL as part of SYCL_INTEL_usm_address_spaces extension to just global
+    // address space if device doesn't support SPV_INTEL_usm_storage_classes
+    // extension
+    if (!BM->isAllowedToUseExtension(
+            ExtensionID::SPV_INTEL_usm_storage_classes) &&
+        ((AddrSpc == SPIRAS_GlobalDevice) || (AddrSpc == SPIRAS_GlobalHost)))
+      AddrSpc = SPIRAS_Global;
     if (ST && !ST->isSized()) {
       Op OpCode;
       StringRef STName = ST->getName();
@@ -756,14 +764,41 @@ SPIRV::SPIRVInstruction *LLVMToSPIRV::transUnaryInst(UnaryInstruction *U,
   Op BOC = OpNop;
   SPIRVValue *Op = nullptr;
   if (auto Cast = dyn_cast<AddrSpaceCastInst>(U)) {
-    if (Cast->getDestTy()->getPointerAddressSpace() == SPIRAS_Generic) {
-      assert(Cast->getSrcTy()->getPointerAddressSpace() != SPIRAS_Constant &&
+    const auto SrcAddrSpace = Cast->getSrcTy()->getPointerAddressSpace();
+    const auto DestAddrSpace = Cast->getDestTy()->getPointerAddressSpace();
+    if (DestAddrSpace == SPIRAS_Generic) {
+      assert(SrcAddrSpace != SPIRAS_Constant &&
              "Casts from constant address space to generic are illegal");
       BOC = OpPtrCastToGeneric;
+    // In SPIR-V only casts to/from generic are allowed. But with
+    // SPV_INTEL_usm_storage_classes we can also have casts from global_device
+    // and global_host to global addr space and vice versa.
+    } else if (SrcAddrSpace == SPIRAS_GlobalDevice ||
+               SrcAddrSpace == SPIRAS_GlobalHost) {
+      assert(
+          (DestAddrSpace == SPIRAS_Global || DestAddrSpace == SPIRAS_Generic) &&
+          "Casts from global_device/global_host only allowed to \
+             global/generic");
+      if (!BM->isAllowedToUseExtension(
+              ExtensionID::SPV_INTEL_usm_storage_classes))
+        BOC = OpPtrCastToGeneric;
+      else
+        BOC = OpPtrCastToCrossWorkgroupINTEL;
+    } else if (DestAddrSpace == SPIRAS_GlobalDevice ||
+                DestAddrSpace == SPIRAS_GlobalHost) {
+      assert(
+          (SrcAddrSpace == SPIRAS_Global || SrcAddrSpace == SPIRAS_Generic) &&
+          "Casts to global_device/global_host only allowed from \
+             global/generic");
+      if (!BM->isAllowedToUseExtension(
+              ExtensionID::SPV_INTEL_usm_storage_classes))
+        BOC = OpGenericCastToPtr;
+      else
+        BOC = OpCrossWorkgroupCastToPtrINTEL;
     } else {
-      assert(Cast->getDestTy()->getPointerAddressSpace() != SPIRAS_Constant &&
+      assert(DestAddrSpace != SPIRAS_Constant &&
              "Casts from generic address space to constant are illegal");
-      assert(Cast->getSrcTy()->getPointerAddressSpace() == SPIRAS_Generic);
+      assert(SrcAddrSpace == SPIRAS_Generic);
       BOC = OpGenericCastToPtr;
     }
   } else {
@@ -1052,8 +1087,18 @@ SPIRVValue *LLVMToSPIRV::transValueWithoutDecoration(Value *V,
     if (IsVectorCompute)
       StorageClass =
           VectorComputeUtil::getVCGlobalVarStorageClass(AddressSpace);
-    else
+    else {
+      // Lower global_device and global_host address spaces that were added in
+      // SYCL as part of SYCL_INTEL_usm_address_spaces extension to just global
+      // address space if device doesn't support SPV_INTEL_usm_storage_classes
+      // extension
+      if (!BM->isAllowedToUseExtension(
+              ExtensionID::SPV_INTEL_usm_storage_classes) &&
+          ((AddressSpace == SPIRAS_GlobalDevice) ||
+           (AddressSpace == SPIRAS_GlobalHost)))
+        AddressSpace = SPIRAS_Global;
       StorageClass = SPIRSPIRVAddrSpaceMap::map(AddressSpace);
+    }
 
     auto BVar = static_cast<SPIRVVariable *>(
         BM->addVariable(transType(Ty), GV->isConstant(), transLinkageType(GV),

diff --git a/lib/SPIRV/libSPIRV/SPIRVEnum.h b/lib/SPIRV/libSPIRV/SPIRVEnum.h
@@ -266,6 +266,8 @@ template <> inline void SPIRVMap<SPIRVStorageClassKind, SPIRVCapVec>::init() {
   ADD_VEC_INIT(StorageClassGeneric, {CapabilityGenericPointer});
   ADD_VEC_INIT(StorageClassPushConstant, {CapabilityShader});
   ADD_VEC_INIT(StorageClassAtomicCounter, {CapabilityAtomicStorage});
+  ADD_VEC_INIT(StorageClassDeviceOnlyINTEL, {CapabilityUSMStorageClassesINTEL});
+  ADD_VEC_INIT(StorageClassHostOnlyINTEL, {CapabilityUSMStorageClassesINTEL});
 }
 
 template <> inline void SPIRVMap<SPIRVImageDimKind, SPIRVCapVec>::init() {

diff --git a/lib/SPIRV/libSPIRV/SPIRVInstruction.cpp b/lib/SPIRV/libSPIRV/SPIRVInstruction.cpp
@@ -183,6 +183,8 @@ bool isSpecConstantOpAllowedOp(Op OC) {
       OpConvertUToPtr,
       OpGenericCastToPtr,
       OpPtrCastToGeneric,
+      OpCrossWorkgroupCastToPtrINTEL,
+      OpPtrCastToCrossWorkgroupINTEL,
       OpBitcast,
       OpQuantizeToF16,
       OpSNegate,

diff --git a/lib/SPIRV/libSPIRV/SPIRVInstruction.h b/lib/SPIRV/libSPIRV/SPIRVInstruction.h
@@ -1603,6 +1603,8 @@ _SPIRV_OP(ConvertPtrToU)
 _SPIRV_OP(ConvertUToPtr)
 _SPIRV_OP(PtrCastToGeneric)
 _SPIRV_OP(GenericCastToPtr)
+_SPIRV_OP(CrossWorkgroupCastToPtrINTEL)
+_SPIRV_OP(PtrCastToCrossWorkgroupINTEL)
 _SPIRV_OP(Bitcast)
 _SPIRV_OP(SNegate)
 _SPIRV_OP(FNegate)

diff --git a/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h b/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h
@@ -174,6 +174,8 @@ inline bool isValid(spv::StorageClass V) {
   case StorageClassPushConstant:
   case StorageClassAtomicCounter:
   case StorageClassImage:
+  case StorageClassDeviceOnlyINTEL:
+  case StorageClassHostOnlyINTEL:
     return true;
   default:
     return false;
@@ -728,6 +730,8 @@ inline bool isValid(spv::Op V) {
   case OpConvertUToPtr:
   case OpPtrCastToGeneric:
   case OpGenericCastToPtr:
+  case OpPtrCastToCrossWorkgroupINTEL:
+  case OpCrossWorkgroupCastToPtrINTEL:
   case OpGenericCastToPtrExplicit:
   case OpBitcast:
   case OpSNegate:

diff --git a/lib/SPIRV/libSPIRV/SPIRVModule.h b/lib/SPIRV/libSPIRV/SPIRVModule.h
@@ -476,6 +476,10 @@ class SPIRVModule {
     return TranslationOpts.getFPContractMode();
   }
 
+  bool isUsmAddrspacesEnabled() {
+    return TranslationOpts.isUsmAddrspacesEnabled();
+  }
+
   // I/O functions
   friend spv_ostream &operator<<(spv_ostream &O, SPIRVModule &M);
   friend std::istream &operator>>(std::istream &I, SPIRVModule &M);

diff --git a/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h b/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h
@@ -150,6 +150,8 @@ template <> inline void SPIRVMap<StorageClass, std::string>::init() {
   add(StorageClassPushConstant, "PushConstant");
   add(StorageClassAtomicCounter, "AtomicCounter");
   add(StorageClassImage, "Image");
+  add(StorageClassDeviceOnlyINTEL, "DeviceOnlyINTEL");
+  add(StorageClassHostOnlyINTEL, "HostOnlyINTEL");
 }
 SPIRV_DEF_NAMEMAP(StorageClass, SPIRVStorageClassNameMap)
 
@@ -549,6 +551,7 @@ template <> inline void SPIRVMap<Capability, std::string>::init() {
   add(CapabilityGroupNonUniformShuffleRelative,
       "GroupNonUniformShuffleRelative");
   add(CapabilityGroupNonUniformClustered, "GroupNonUniformClustered");
+  add(CapabilityUSMStorageClassesINTEL, "USMStorageClassesINTEL");
 }
 SPIRV_DEF_NAMEMAP(Capability, SPIRVCapabilityNameMap)
 

diff --git a/lib/SPIRV/libSPIRV/SPIRVOpCode.h b/lib/SPIRV/libSPIRV/SPIRVOpCode.h
@@ -96,7 +96,9 @@ inline bool isCmpOpCode(Op OpCode) {
 
 inline bool isCvtOpCode(Op OpCode) {
   return ((unsigned)OpCode >= OpConvertFToU && (unsigned)OpCode <= OpBitcast) ||
-         OpCode == OpSatConvertSToU || OpCode == OpSatConvertUToS;
+         OpCode == OpSatConvertSToU || OpCode == OpSatConvertUToS ||
+         OpCode == OpPtrCastToCrossWorkgroupINTEL ||
+         OpCode == OpCrossWorkgroupCastToPtrINTEL;
 }
 
 inline bool isCvtToUnsignedOpCode(Op OpCode) {

diff --git a/lib/SPIRV/libSPIRV/SPIRVOpCodeEnum.h b/lib/SPIRV/libSPIRV/SPIRVOpCodeEnum.h
@@ -469,6 +469,8 @@ _SPIRV_OP(SubgroupAvcSicGetPackedSkcLumaCountThresholdINTEL, 5814)
 _SPIRV_OP(SubgroupAvcSicGetPackedSkcLumaSumThresholdINTEL, 5815)
 _SPIRV_OP(SubgroupAvcSicGetInterRawSadsINTEL, 5816)
 _SPIRV_OP(LoopControlINTEL, 5887)
+_SPIRV_OP(PtrCastToCrossWorkgroupINTEL, 5934)
+_SPIRV_OP(CrossWorkgroupCastToPtrINTEL, 5938)
 _SPIRV_OP(ReadPipeBlockingINTEL, 5946)
 _SPIRV_OP(WritePipeBlockingINTEL, 5947)
 _SPIRV_OP(FPGARegINTEL, 5949)
diff --git a/lib/SPIRV/libSPIRV/spirv.hpp b/lib/SPIRV/libSPIRV/spirv.hpp
@@ -196,6 +196,8 @@ enum StorageClass {
     StorageClassShaderRecordBufferNV = 5343,
     StorageClassPhysicalStorageBuffer = 5349,
     StorageClassPhysicalStorageBufferEXT = 5349,
+    StorageClassDeviceOnlyINTEL = 5936,
+    StorageClassHostOnlyINTEL = 5937,
     StorageClassMax = 0x7fffffff,
 };
 
@@ -953,6 +955,7 @@ enum Capability {
   CapabilityFPGARegINTEL = 5948,
   CapabilityKernelAttributesINTEL = 5892,
   CapabilityFPGAKernelAttributesINTEL = 5897,
+  CapabilityUSMStorageClassesINTEL = 5935,
   CapabilityIOPipeINTEL = 5943,
   CapabilityMax = 0x7fffffff,
 };
@@ -1493,6 +1496,8 @@ enum Op {
   OpSubgroupAvcSicGetPackedSkcLumaSumThresholdINTEL = 5815,
   OpSubgroupAvcSicGetInterRawSadsINTEL = 5816,
   OpLoopControlINTEL = 5887,
+  OpPtrCastToCrossWorkgroupINTEL = 5934,
+  OpCrossWorkgroupCastToPtrINTEL = 5938,
   OpReadPipeBlockingINTEL = 5946,
   OpWritePipeBlockingINTEL = 5947,
   OpFPGARegINTEL = 5949,