Skip to content

Commit 12b64ce

Browse files
MiloszSkobejkoMiloszSkobejko
authored
Merge branch 'llvm_release_150' into fpbuiltin-max-error-backport-llvm-15
2 parents 626e879 + a59ddf8 commit 12b64ce

21 files changed

+532
-56
lines changed

‎include/LLVMSPIRVExtensions.inc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,3 +64,4 @@ EXT(SPV_INTEL_tensor_float32_rounding)
6464
EXT(SPV_EXT_relaxed_printf_string_address_space)
6565
EXT(SPV_INTEL_fp_max_error)
6666
EXT(SPV_INTEL_cache_controls)
67+
EXT(SPV_INTEL_maximum_registers)

‎lib/SPIRV/SPIRVInternal.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ using namespace llvm;
5959

6060
namespace llvm {
6161
class IntrinsicInst;
62+
class IRBuilderBase;
6263
}
6364

6465
namespace SPIRV {
@@ -573,6 +574,10 @@ std::string mapLLVMTypeToOCLType(const Type *Ty, bool Signed,
573574
Type *PointerElementType = nullptr);
574575
SPIRVDecorate *mapPostfixToDecorate(StringRef Postfix, SPIRVEntry *Target);
575576

577+
/// Return vector V extended with poison elements to match the number of
578+
/// components of NewType.
579+
Value *extendVector(Value *V, FixedVectorType *NewType, IRBuilderBase &Builder);
580+
576581
/// Add decorations to a SPIR-V entry.
577582
/// \param Decs Each string is a postfix without _ at the beginning.
578583
SPIRVValue *addDecorations(SPIRVValue *Target,

‎lib/SPIRV/SPIRVReader.cpp‎

Lines changed: 91 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2305,10 +2305,37 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F,
23052305
if (BB) {
23062306
Builder.SetInsertPoint(BB);
23072307
}
2308-
return mapValue(BV, Builder.CreateShuffleVector(
2309-
transValue(VS->getVector1(), F, BB),
2310-
transValue(VS->getVector2(), F, BB),
2311-
ConstantVector::get(Components), BV->getName()));
2308+
Value *Vec1 = transValue(VS->getVector1(), F, BB);
2309+
Value *Vec2 = transValue(VS->getVector2(), F, BB);
2310+
auto *Vec1Ty = cast<FixedVectorType>(Vec1->getType());
2311+
auto *Vec2Ty = cast<FixedVectorType>(Vec2->getType());
2312+
if (Vec1Ty->getNumElements() != Vec2Ty->getNumElements()) {
2313+
// LLVM's shufflevector requires that the two vector operands have the
2314+
// same type; SPIR-V's OpVectorShuffle allows the vector operands to
2315+
// differ in the number of components. Adjust for that by extending
2316+
// the smaller vector.
2317+
if (Vec1Ty->getNumElements() < Vec2Ty->getNumElements()) {
2318+
Vec1 = extendVector(Vec1, Vec2Ty, Builder);
2319+
// Extending Vec1 requires offsetting any Vec2 indices in Components by
2320+
// the number of new elements.
2321+
unsigned Offset = Vec2Ty->getNumElements() - Vec1Ty->getNumElements();
2322+
unsigned Vec2Start = Vec1Ty->getNumElements();
2323+
for (auto &C : Components) {
2324+
if (auto *CI = dyn_cast<ConstantInt>(C)) {
2325+
uint64_t V = CI->getZExtValue();
2326+
if (V >= Vec2Start) {
2327+
// This is a Vec2 index; add the offset to it.
2328+
C = ConstantInt::get(Int32Ty, V + Offset);
2329+
}
2330+
}
2331+
}
2332+
} else {
2333+
Vec2 = extendVector(Vec2, Vec1Ty, Builder);
2334+
}
2335+
}
2336+
return mapValue(
2337+
BV, Builder.CreateShuffleVector(
2338+
Vec1, Vec2, ConstantVector::get(Components), BV->getName()));
23122339
}
23132340

23142341
case OpBitReverse: {
@@ -2470,6 +2497,22 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F,
24702497
case OpSignBitSet:
24712498
return mapValue(BV,
24722499
transRelational(static_cast<SPIRVInstruction *>(BV), BB));
2500+
case OpIAddCarry: {
2501+
IRBuilder<> Builder(BB);
2502+
auto *BC = static_cast<SPIRVBinary *>(BV);
2503+
return mapValue(BV, Builder.CreateBinaryIntrinsic(
2504+
Intrinsic::uadd_with_overflow,
2505+
transValue(BC->getOperand(0), F, BB),
2506+
transValue(BC->getOperand(1), F, BB)));
2507+
}
2508+
case OpISubBorrow: {
2509+
IRBuilder<> Builder(BB);
2510+
auto *BC = static_cast<SPIRVBinary *>(BV);
2511+
return mapValue(BV, Builder.CreateBinaryIntrinsic(
2512+
Intrinsic::usub_with_overflow,
2513+
transValue(BC->getOperand(0), F, BB),
2514+
transValue(BC->getOperand(1), F, BB)));
2515+
}
24732516
case OpGetKernelWorkGroupSize:
24742517
case OpGetKernelPreferredWorkGroupSizeMultiple:
24752518
return mapValue(
@@ -4124,6 +4167,50 @@ bool SPIRVToLLVM::transMetadata() {
41244167
F->setMetadata(kSPIR2MD::IntelFPGAIPInterface,
41254168
MDNode::get(*Context, InterfaceMDVec));
41264169
}
4170+
if (auto *EM = BF->getExecutionMode(
4171+
internal::ExecutionModeMaximumRegistersINTEL)) {
4172+
NamedMDNode *ExecModeMD =
4173+
M->getOrInsertNamedMetadata(kSPIRVMD::ExecutionMode);
4174+
4175+
SmallVector<Metadata *, 4> ValueVec;
4176+
ValueVec.push_back(ConstantAsMetadata::get(F));
4177+
ValueVec.push_back(
4178+
ConstantAsMetadata::get(getUInt32(M, EM->getExecutionMode())));
4179+
ValueVec.push_back(
4180+
ConstantAsMetadata::get(getUInt32(M, EM->getLiterals()[0])));
4181+
ExecModeMD->addOperand(MDNode::get(*Context, ValueVec));
4182+
}
4183+
if (auto *EM = BF->getExecutionMode(
4184+
internal::ExecutionModeMaximumRegistersIdINTEL)) {
4185+
NamedMDNode *ExecModeMD =
4186+
M->getOrInsertNamedMetadata(kSPIRVMD::ExecutionMode);
4187+
4188+
SmallVector<Metadata *, 4> ValueVec;
4189+
ValueVec.push_back(ConstantAsMetadata::get(F));
4190+
ValueVec.push_back(
4191+
ConstantAsMetadata::get(getUInt32(M, EM->getExecutionMode())));
4192+
4193+
auto *ExecOp = BF->getModule()->getValue(EM->getLiterals()[0]);
4194+
ValueVec.push_back(
4195+
MDNode::get(*Context, ConstantAsMetadata::get(cast<ConstantInt>(
4196+
transValue(ExecOp, nullptr, nullptr)))));
4197+
ExecModeMD->addOperand(MDNode::get(*Context, ValueVec));
4198+
}
4199+
if (auto *EM = BF->getExecutionMode(
4200+
internal::ExecutionModeNamedMaximumRegistersINTEL)) {
4201+
NamedMDNode *ExecModeMD =
4202+
M->getOrInsertNamedMetadata(kSPIRVMD::ExecutionMode);
4203+
4204+
SmallVector<Metadata *, 4> ValueVec;
4205+
ValueVec.push_back(ConstantAsMetadata::get(F));
4206+
ValueVec.push_back(
4207+
ConstantAsMetadata::get(getUInt32(M, EM->getExecutionMode())));
4208+
4209+
assert(EM->getLiterals()[0] == 0 &&
4210+
"Invalid named maximum number of registers");
4211+
ValueVec.push_back(MDString::get(*Context, "AutoINTEL"));
4212+
ExecModeMD->addOperand(MDNode::get(*Context, ValueVec));
4213+
}
41274214
}
41284215
NamedMDNode *MemoryModelMD =
41294216
M->getOrInsertNamedMetadata(kSPIRVMD::MemoryModel);

‎lib/SPIRV/SPIRVUtil.cpp‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,24 @@ void removeFnAttr(CallInst *Call, Attribute::AttrKind Attr) {
9090
Call->removeFnAttr(Attr);
9191
}
9292

93+
Value *extendVector(Value *V, FixedVectorType *NewType,
94+
IRBuilderBase &Builder) {
95+
unsigned OldSize = cast<FixedVectorType>(V->getType())->getNumElements();
96+
unsigned NewSize = NewType->getNumElements();
97+
assert(OldSize < NewSize);
98+
std::vector<Constant *> Components;
99+
IntegerType *Int32Ty = Builder.getInt32Ty();
100+
for (unsigned I = 0; I < NewSize; I++) {
101+
if (I < OldSize)
102+
Components.push_back(ConstantInt::get(Int32Ty, I));
103+
else
104+
Components.push_back(PoisonValue::get(Int32Ty));
105+
}
106+
107+
return Builder.CreateShuffleVector(V, PoisonValue::get(V->getType()),
108+
ConstantVector::get(Components), "vecext");
109+
}
110+
93111
void saveLLVMModule(Module *M, const std::string &OutputFile) {
94112
std::error_code EC;
95113
ToolOutputFile Out(OutputFile.c_str(), EC, sys::fs::OF_None);

‎lib/SPIRV/SPIRVWriter.cpp‎

Lines changed: 65 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -915,6 +915,9 @@ SPIRVFunction *LLVMToSPIRVBase::transFunctionDecl(Function *F) {
915915

916916
transFPGAFunctionMetadata(BF, F);
917917

918+
if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_maximum_registers))
919+
transFunctionMetadataAsExecutionMode(BF, F);
920+
918921
transAuxDataInst(BF, F);
919922

920923
SPIRVDBG(dbgs() << "[transFunction] " << *F << " => ";
@@ -1041,6 +1044,38 @@ void LLVMToSPIRVBase::transFPGAFunctionMetadata(SPIRVFunction *BF,
10411044
}
10421045
}
10431046

1047+
void LLVMToSPIRVBase::transFunctionMetadataAsExecutionMode(SPIRVFunction *BF,
1048+
Function *F) {
1049+
SmallVector<MDNode *, 1> RegisterAllocModeMDs;
1050+
F->getMetadata("RegisterAllocMode", RegisterAllocModeMDs);
1051+
1052+
for (unsigned I = 0; I < RegisterAllocModeMDs.size(); I++) {
1053+
auto *RegisterAllocMode = RegisterAllocModeMDs[I]->getOperand(0).get();
1054+
if (isa<MDString>(RegisterAllocMode)) {
1055+
const StringRef Str = getMDOperandAsString(RegisterAllocModeMDs[I], 0);
1056+
const internal::InternalNamedMaximumNumberOfRegisters NamedValue =
1057+
SPIRVNamedMaximumNumberOfRegistersNameMap::rmap(Str.str());
1058+
BF->addExecutionMode(BM->add(new SPIRVExecutionMode(
1059+
OpExecutionMode, BF,
1060+
internal::ExecutionModeNamedMaximumRegistersINTEL, NamedValue)));
1061+
} else if (isa<MDNode>(RegisterAllocMode)) {
1062+
auto *RegisterAllocNodeMDOp =
1063+
getMDOperandAsMDNode(RegisterAllocModeMDs[I], 0);
1064+
const int Num = getMDOperandAsInt(RegisterAllocNodeMDOp, 0);
1065+
auto *Const =
1066+
BM->addConstant(transType(Type::getInt32Ty(F->getContext())), Num);
1067+
BF->addExecutionMode(BM->add(new SPIRVExecutionModeId(
1068+
BF, internal::ExecutionModeMaximumRegistersIdINTEL, Const->getId())));
1069+
} else {
1070+
const int64_t RegisterAllocVal =
1071+
mdconst::dyn_extract<ConstantInt>(RegisterAllocMode)->getZExtValue();
1072+
BF->addExecutionMode(BM->add(new SPIRVExecutionMode(
1073+
OpExecutionMode, BF, internal::ExecutionModeMaximumRegistersINTEL,
1074+
RegisterAllocVal)));
1075+
}
1076+
}
1077+
}
1078+
10441079
void LLVMToSPIRVBase::transAuxDataInst(SPIRVFunction *BF, Function *F) {
10451080
auto *BM = BF->getModule();
10461081
if (!BM->preserveAuxData())
@@ -3376,6 +3411,8 @@ bool LLVMToSPIRVBase::isKnownIntrinsic(Intrinsic::ID Id) {
33763411
case Intrinsic::dbg_label:
33773412
case Intrinsic::trap:
33783413
case Intrinsic::arithmetic_fence:
3414+
case Intrinsic::uadd_with_overflow:
3415+
case Intrinsic::usub_with_overflow:
33793416
return true;
33803417
default:
33813418
// Unknown intrinsics' declarations should always be translated
@@ -3809,6 +3846,16 @@ SPIRVValue *LLVMToSPIRVBase::transIntrinsicInst(IntrinsicInst *II,
38093846
SPIRVValue *Zero = transValue(Constant::getNullValue(II->getType()), BB);
38103847
return BM->addSelectInst(Cmp, Sub, Zero, BB);
38113848
}
3849+
case Intrinsic::uadd_with_overflow: {
3850+
return BM->addBinaryInst(OpIAddCarry, transType(II->getType()),
3851+
transValue(II->getArgOperand(0), BB),
3852+
transValue(II->getArgOperand(1), BB), BB);
3853+
}
3854+
case Intrinsic::usub_with_overflow: {
3855+
return BM->addBinaryInst(OpISubBorrow, transType(II->getType()),
3856+
transValue(II->getArgOperand(0), BB),
3857+
transValue(II->getArgOperand(1), BB), BB);
3858+
}
38123859
case Intrinsic::memset: {
38133860
// Generally there is no direct mapping of memset to SPIR-V. But it turns
38143861
// out that memset is emitted by Clang for initialization in default
@@ -4885,19 +4932,20 @@ bool LLVMToSPIRVBase::transExecutionMode() {
48854932
auto AddSingleArgExecutionMode = [&](ExecutionMode EMode) {
48864933
uint32_t Arg = 0;
48874934
N.get(Arg);
4888-
BF->addExecutionMode(BM->add(new SPIRVExecutionMode(BF, EMode, Arg)));
4935+
BF->addExecutionMode(
4936+
BM->add(new SPIRVExecutionMode(OpExecutionMode, BF, EMode, Arg)));
48894937
};
48904938

48914939
switch (EMode) {
48924940
case spv::ExecutionModeContractionOff:
4893-
BF->addExecutionMode(BM->add(
4894-
new SPIRVExecutionMode(BF, static_cast<ExecutionMode>(EMode))));
4941+
BF->addExecutionMode(BM->add(new SPIRVExecutionMode(
4942+
OpExecutionMode, BF, static_cast<ExecutionMode>(EMode))));
48954943
break;
48964944
case spv::ExecutionModeInitializer:
48974945
case spv::ExecutionModeFinalizer:
48984946
if (BM->isAllowedToUseVersion(VersionNumber::SPIRV_1_1)) {
4899-
BF->addExecutionMode(BM->add(
4900-
new SPIRVExecutionMode(BF, static_cast<ExecutionMode>(EMode))));
4947+
BF->addExecutionMode(BM->add(new SPIRVExecutionMode(
4948+
OpExecutionMode, BF, static_cast<ExecutionMode>(EMode))));
49014949
} else {
49024950
getErrorLog().checkError(false, SPIRVEC_Requires1_1,
49034951
"Initializer/Finalizer Execution Mode");
@@ -4909,15 +4957,16 @@ bool LLVMToSPIRVBase::transExecutionMode() {
49094957
unsigned X = 0, Y = 0, Z = 0;
49104958
N.get(X).get(Y).get(Z);
49114959
BF->addExecutionMode(BM->add(new SPIRVExecutionMode(
4912-
BF, static_cast<ExecutionMode>(EMode), X, Y, Z)));
4960+
OpExecutionMode, BF, static_cast<ExecutionMode>(EMode), X, Y, Z)));
49134961
} break;
49144962
case spv::ExecutionModeMaxWorkgroupSizeINTEL: {
49154963
if (BM->isAllowedToUseExtension(
49164964
ExtensionID::SPV_INTEL_kernel_attributes)) {
49174965
unsigned X = 0, Y = 0, Z = 0;
49184966
N.get(X).get(Y).get(Z);
49194967
BF->addExecutionMode(BM->add(new SPIRVExecutionMode(
4920-
BF, static_cast<ExecutionMode>(EMode), X, Y, Z)));
4968+
OpExecutionMode, BF, static_cast<ExecutionMode>(EMode), X, Y,
4969+
Z)));
49214970
BM->addExtension(ExtensionID::SPV_INTEL_kernel_attributes);
49224971
BM->addCapability(CapabilityKernelAttributesINTEL);
49234972
}
@@ -4926,8 +4975,8 @@ bool LLVMToSPIRVBase::transExecutionMode() {
49264975
if (!BM->isAllowedToUseExtension(
49274976
ExtensionID::SPV_INTEL_kernel_attributes))
49284977
break;
4929-
BF->addExecutionMode(BM->add(
4930-
new SPIRVExecutionMode(BF, static_cast<ExecutionMode>(EMode))));
4978+
BF->addExecutionMode(BM->add(new SPIRVExecutionMode(
4979+
OpExecutionMode, BF, static_cast<ExecutionMode>(EMode))));
49314980
BM->addExtension(ExtensionID::SPV_INTEL_kernel_attributes);
49324981
BM->addCapability(CapabilityKernelAttributesINTEL);
49334982
} break;
@@ -4957,8 +5006,9 @@ bool LLVMToSPIRVBase::transExecutionMode() {
49575006
break;
49585007
unsigned NBarrierCnt = 0;
49595008
N.get(NBarrierCnt);
4960-
BF->addExecutionMode(new SPIRVExecutionMode(
4961-
BF, static_cast<ExecutionMode>(EMode), NBarrierCnt));
5009+
BF->addExecutionMode(BM->add(new SPIRVExecutionMode(
5010+
OpExecutionMode, BF, static_cast<ExecutionMode>(EMode),
5011+
NBarrierCnt)));
49625012
BM->addExtension(ExtensionID::SPV_INTEL_vector_compute);
49635013
BM->addCapability(CapabilityVectorComputeINTEL);
49645014
} break;
@@ -4988,8 +5038,8 @@ bool LLVMToSPIRVBase::transExecutionMode() {
49885038
} break;
49895039
case spv::internal::ExecutionModeFastCompositeKernelINTEL: {
49905040
if (BM->isAllowedToUseExtension(ExtensionID::SPV_INTEL_fast_composite))
4991-
BF->addExecutionMode(BM->add(
4992-
new SPIRVExecutionMode(BF, static_cast<ExecutionMode>(EMode))));
5041+
BF->addExecutionMode(BM->add(new SPIRVExecutionMode(
5042+
OpExecutionMode, BF, static_cast<ExecutionMode>(EMode))));
49935043
} break;
49945044
default:
49955045
llvm_unreachable("invalid execution mode");
@@ -5034,8 +5084,8 @@ void LLVMToSPIRVBase::transFPContract() {
50345084
}
50355085

50365086
if (DisableContraction) {
5037-
BF->addExecutionMode(BF->getModule()->add(
5038-
new SPIRVExecutionMode(BF, spv::ExecutionModeContractionOff)));
5087+
BF->addExecutionMode(BF->getModule()->add(new SPIRVExecutionMode(
5088+
OpExecutionMode, BF, spv::ExecutionModeContractionOff)));
50395089
}
50405090
}
50415091
}

‎lib/SPIRV/SPIRVWriter.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ class LLVMToSPIRVBase {
132132
void transVectorComputeMetadata(Function *F);
133133
void transFPGAFunctionMetadata(SPIRVFunction *BF, Function *F);
134134
void transAuxDataInst(SPIRVFunction *BF, Function *F);
135-
135+
void transFunctionMetadataAsExecutionMode(SPIRVFunction *BF, Function *F);
136136
bool transGlobalVariables();
137137

138138
Op transBoolOpCode(SPIRVValue *Opn, Op OC);

‎lib/SPIRV/libSPIRV/SPIRVEntry.cpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -581,7 +581,7 @@ SPIRVEntryPoint::SPIRVEntryPoint(SPIRVModule *TheModule,
581581
SPIRVExecutionModelKind TheExecModel,
582582
SPIRVId TheId, const std::string &TheName,
583583
std::vector<SPIRVId> Variables)
584-
: SPIRVAnnotation(TheModule->get<SPIRVFunction>(TheId),
584+
: SPIRVAnnotation(OpEntryPoint, TheModule->get<SPIRVFunction>(TheId),
585585
getSizeInWords(TheName) + Variables.size() + 3),
586586
ExecModel(TheExecModel), Name(TheName), Variables(Variables) {}
587587

@@ -628,6 +628,9 @@ void SPIRVExecutionMode::decode(std::istream &I) {
628628
case ExecutionModeNumSIMDWorkitemsINTEL:
629629
case ExecutionModeSchedulerTargetFmaxMhzINTEL:
630630
case internal::ExecutionModeStreamingInterfaceINTEL:
631+
case internal::ExecutionModeMaximumRegistersINTEL:
632+
case internal::ExecutionModeMaximumRegistersIdINTEL:
633+
case internal::ExecutionModeNamedMaximumRegistersINTEL:
631634
WordLiterals.resize(1);
632635
break;
633636
default:
@@ -649,7 +652,8 @@ SPIRVForward *SPIRVAnnotationGeneric::getOrCreateTarget() const {
649652
}
650653

651654
SPIRVName::SPIRVName(const SPIRVEntry *TheTarget, const std::string &TheStr)
652-
: SPIRVAnnotation(TheTarget, getSizeInWords(TheStr) + 2), Str(TheStr) {}
655+
: SPIRVAnnotation(OpName, TheTarget, getSizeInWords(TheStr) + 2),
656+
Str(TheStr) {}
653657

654658
void SPIRVName::encode(spv_ostream &O) const { getEncoder(O) << Target << Str; }
655659

0 commit comments

Comments
 (0)