Skip to content

Commit 118abf2

Browse files
huihzhanghuihzhang
committed
[SVE] Update API ConstantVector::getSplat() to use ElementCount.
Summary: Support ConstantInt::get() and Constant::getAllOnesValue() for scalable vector type, this requires ConstantVector::getSplat() to take in 'ElementCount', instead of 'unsigned' number of element count. This change is needed for D73753. Reviewers: sdesmalen, efriedma, apazos, spatel, huntergr, willlovett Reviewed By: efriedma Subscribers: tschuett, hiraditya, rkruppe, psnobl, cfe-commits, llvm-commits Tags: #llvm Differential Revision: https://reviews.llvm.org/D74386
1 parent e91feee commit 118abf2

File tree

14 files changed

+180
-67
lines changed

14 files changed

+180
-67
lines changed

‎clang/lib/CodeGen/CGBuiltin.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4496,8 +4496,8 @@ static llvm::VectorType *GetFloatNeonType(CodeGenFunction *CGF,
44964496
}
44974497

44984498
Value *CodeGenFunction::EmitNeonSplat(Value *V, Constant *C) {
4499-
unsigned nElts = V->getType()->getVectorNumElements();
4500-
Value* SV = llvm::ConstantVector::getSplat(nElts, C);
4499+
ElementCount EC = V->getType()->getVectorElementCount();
4500+
Value *SV = llvm::ConstantVector::getSplat(EC, C);
45014501
return Builder.CreateShuffleVector(V, V, SV, "lane");
45024502
}
45034503

@@ -8701,7 +8701,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
87018701
llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) :
87028702
VTy;
87038703
llvm::Constant *cst = cast<Constant>(Ops[3]);
8704-
Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst);
8704+
Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
87058705
Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
87068706
Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane");
87078707

@@ -8730,7 +8730,7 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
87308730
llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(),
87318731
VTy->getNumElements() * 2);
87328732
Ops[2] = Builder.CreateBitCast(Ops[2], STy);
8733-
Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(),
8733+
Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
87348734
cast<ConstantInt>(Ops[3]));
87358735
Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
87368736

‎llvm/include/llvm/Analysis/Utils/Local.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ Value *EmitGEPOffset(IRBuilderTy *Builder, const DataLayout &DL, User *GEP,
6363

6464
// Splat the constant if needed.
6565
if (IntIdxTy->isVectorTy() && !OpC->getType()->isVectorTy())
66-
OpC = ConstantVector::getSplat(IntIdxTy->getVectorNumElements(), OpC);
66+
OpC = ConstantVector::getSplat(IntIdxTy->getVectorElementCount(), OpC);
6767

6868
Constant *Scale = ConstantInt::get(IntIdxTy, Size);
6969
Constant *OC = ConstantExpr::getIntegerCast(OpC, IntIdxTy, true /*SExt*/);

‎llvm/include/llvm/IR/Constants.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -517,7 +517,7 @@ class ConstantVector final : public ConstantAggregate {
517517

518518
public:
519519
/// Return a ConstantVector with the specified constant in each element.
520-
static Constant *getSplat(unsigned NumElts, Constant *Elt);
520+
static Constant *getSplat(ElementCount EC, Constant *Elt);
521521

522522
/// Specialize the getType() method to always return a VectorType,
523523
/// which reduces the amount of casting needed in parts of the compiler.

‎llvm/lib/Analysis/InstructionSimplify.cpp‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -707,9 +707,8 @@ static Constant *stripAndComputeConstantOffsets(const DataLayout &DL, Value *&V,
707707
Offset = Offset.sextOrTrunc(IntIdxTy->getIntegerBitWidth());
708708

709709
Constant *OffsetIntPtr = ConstantInt::get(IntIdxTy, Offset);
710-
if (V->getType()->isVectorTy())
711-
return ConstantVector::getSplat(V->getType()->getVectorNumElements(),
712-
OffsetIntPtr);
710+
if (VectorType *VecTy = dyn_cast<VectorType>(V->getType()))
711+
return ConstantVector::getSplat(VecTy->getElementCount(), OffsetIntPtr);
713712
return OffsetIntPtr;
714713
}
715714

‎llvm/lib/CodeGen/CodeGenPrepare.cpp‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6565,19 +6565,23 @@ class VectorPromoteHelper {
65656565
UseSplat = true;
65666566
}
65676567

6568-
unsigned End = getTransitionType()->getVectorNumElements();
6568+
ElementCount EC = getTransitionType()->getVectorElementCount();
65696569
if (UseSplat)
6570-
return ConstantVector::getSplat(End, Val);
6571-
6572-
SmallVector<Constant *, 4> ConstVec;
6573-
UndefValue *UndefVal = UndefValue::get(Val->getType());
6574-
for (unsigned Idx = 0; Idx != End; ++Idx) {
6575-
if (Idx == ExtractIdx)
6576-
ConstVec.push_back(Val);
6577-
else
6578-
ConstVec.push_back(UndefVal);
6579-
}
6580-
return ConstantVector::get(ConstVec);
6570+
return ConstantVector::getSplat(EC, Val);
6571+
6572+
if (!EC.Scalable) {
6573+
SmallVector<Constant *, 4> ConstVec;
6574+
UndefValue *UndefVal = UndefValue::get(Val->getType());
6575+
for (unsigned Idx = 0; Idx != EC.Min; ++Idx) {
6576+
if (Idx == ExtractIdx)
6577+
ConstVec.push_back(Val);
6578+
else
6579+
ConstVec.push_back(UndefVal);
6580+
}
6581+
return ConstantVector::get(ConstVec);
6582+
} else
6583+
llvm_unreachable(
6584+
"Generate scalable vector for non-splat is unimplemented");
65816585
}
65826586

65836587
/// Check if promoting to a vector type an operand at \p OperandIdx

‎llvm/lib/IR/ConstantFold.cpp‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2229,8 +2229,7 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C,
22292229
Constant *Idx0 = cast<Constant>(Idxs[0]);
22302230
if (Idxs.size() == 1 && (Idx0->isNullValue() || isa<UndefValue>(Idx0)))
22312231
return GEPTy->isVectorTy() && !C->getType()->isVectorTy()
2232-
? ConstantVector::getSplat(
2233-
cast<VectorType>(GEPTy)->getNumElements(), C)
2232+
? ConstantVector::getSplat(GEPTy->getVectorElementCount(), C)
22342233
: C;
22352234

22362235
if (C->isNullValue()) {

‎llvm/lib/IR/Constants.cpp‎

Lines changed: 53 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -370,7 +370,7 @@ Constant *Constant::getIntegerValue(Type *Ty, const APInt &V) {
370370

371371
// Broadcast a scalar to a vector, if necessary.
372372
if (VectorType *VTy = dyn_cast<VectorType>(Ty))
373-
C = ConstantVector::getSplat(VTy->getNumElements(), C);
373+
C = ConstantVector::getSplat(VTy->getElementCount(), C);
374374

375375
return C;
376376
}
@@ -387,7 +387,7 @@ Constant *Constant::getAllOnesValue(Type *Ty) {
387387
}
388388

389389
VectorType *VTy = cast<VectorType>(Ty);
390-
return ConstantVector::getSplat(VTy->getNumElements(),
390+
return ConstantVector::getSplat(VTy->getElementCount(),
391391
getAllOnesValue(VTy->getElementType()));
392392
}
393393

@@ -681,15 +681,15 @@ Constant *ConstantInt::getTrue(Type *Ty) {
681681
assert(Ty->isIntOrIntVectorTy(1) && "Type not i1 or vector of i1.");
682682
ConstantInt *TrueC = ConstantInt::getTrue(Ty->getContext());
683683
if (auto *VTy = dyn_cast<VectorType>(Ty))
684-
return ConstantVector::getSplat(VTy->getNumElements(), TrueC);
684+
return ConstantVector::getSplat(VTy->getElementCount(), TrueC);
685685
return TrueC;
686686
}
687687

688688
Constant *ConstantInt::getFalse(Type *Ty) {
689689
assert(Ty->isIntOrIntVectorTy(1) && "Type not i1 or vector of i1.");
690690
ConstantInt *FalseC = ConstantInt::getFalse(Ty->getContext());
691691
if (auto *VTy = dyn_cast<VectorType>(Ty))
692-
return ConstantVector::getSplat(VTy->getNumElements(), FalseC);
692+
return ConstantVector::getSplat(VTy->getElementCount(), FalseC);
693693
return FalseC;
694694
}
695695

@@ -712,7 +712,7 @@ Constant *ConstantInt::get(Type *Ty, uint64_t V, bool isSigned) {
712712

713713
// For vectors, broadcast the value.
714714
if (VectorType *VTy = dyn_cast<VectorType>(Ty))
715-
return ConstantVector::getSplat(VTy->getNumElements(), C);
715+
return ConstantVector::getSplat(VTy->getElementCount(), C);
716716

717717
return C;
718718
}
@@ -736,7 +736,7 @@ Constant *ConstantInt::get(Type *Ty, const APInt& V) {
736736

737737
// For vectors, broadcast the value.
738738
if (VectorType *VTy = dyn_cast<VectorType>(Ty))
739-
return ConstantVector::getSplat(VTy->getNumElements(), C);
739+
return ConstantVector::getSplat(VTy->getElementCount(), C);
740740

741741
return C;
742742
}
@@ -781,7 +781,7 @@ Constant *ConstantFP::get(Type *Ty, double V) {
781781

782782
// For vectors, broadcast the value.
783783
if (VectorType *VTy = dyn_cast<VectorType>(Ty))
784-
return ConstantVector::getSplat(VTy->getNumElements(), C);
784+
return ConstantVector::getSplat(VTy->getElementCount(), C);
785785

786786
return C;
787787
}
@@ -793,7 +793,7 @@ Constant *ConstantFP::get(Type *Ty, const APFloat &V) {
793793

794794
// For vectors, broadcast the value.
795795
if (auto *VTy = dyn_cast<VectorType>(Ty))
796-
return ConstantVector::getSplat(VTy->getNumElements(), C);
796+
return ConstantVector::getSplat(VTy->getElementCount(), C);
797797

798798
return C;
799799
}
@@ -806,7 +806,7 @@ Constant *ConstantFP::get(Type *Ty, StringRef Str) {
806806

807807
// For vectors, broadcast the value.
808808
if (VectorType *VTy = dyn_cast<VectorType>(Ty))
809-
return ConstantVector::getSplat(VTy->getNumElements(), C);
809+
return ConstantVector::getSplat(VTy->getElementCount(), C);
810810

811811
return C;
812812
}
@@ -817,7 +817,7 @@ Constant *ConstantFP::getNaN(Type *Ty, bool Negative, uint64_t Payload) {
817817
Constant *C = get(Ty->getContext(), NaN);
818818

819819
if (VectorType *VTy = dyn_cast<VectorType>(Ty))
820-
return ConstantVector::getSplat(VTy->getNumElements(), C);
820+
return ConstantVector::getSplat(VTy->getElementCount(), C);
821821

822822
return C;
823823
}
@@ -828,7 +828,7 @@ Constant *ConstantFP::getQNaN(Type *Ty, bool Negative, APInt *Payload) {
828828
Constant *C = get(Ty->getContext(), NaN);
829829

830830
if (VectorType *VTy = dyn_cast<VectorType>(Ty))
831-
return ConstantVector::getSplat(VTy->getNumElements(), C);
831+
return ConstantVector::getSplat(VTy->getElementCount(), C);
832832

833833
return C;
834834
}
@@ -839,7 +839,7 @@ Constant *ConstantFP::getSNaN(Type *Ty, bool Negative, APInt *Payload) {
839839
Constant *C = get(Ty->getContext(), NaN);
840840

841841
if (VectorType *VTy = dyn_cast<VectorType>(Ty))
842-
return ConstantVector::getSplat(VTy->getNumElements(), C);
842+
return ConstantVector::getSplat(VTy->getElementCount(), C);
843843

844844
return C;
845845
}
@@ -850,7 +850,7 @@ Constant *ConstantFP::getNegativeZero(Type *Ty) {
850850
Constant *C = get(Ty->getContext(), NegZero);
851851

852852
if (VectorType *VTy = dyn_cast<VectorType>(Ty))
853-
return ConstantVector::getSplat(VTy->getNumElements(), C);
853+
return ConstantVector::getSplat(VTy->getElementCount(), C);
854854

855855
return C;
856856
}
@@ -898,7 +898,7 @@ Constant *ConstantFP::getInfinity(Type *Ty, bool Negative) {
898898
Constant *C = get(Ty->getContext(), APFloat::getInf(Semantics, Negative));
899899

900900
if (VectorType *VTy = dyn_cast<VectorType>(Ty))
901-
return ConstantVector::getSplat(VTy->getNumElements(), C);
901+
return ConstantVector::getSplat(VTy->getElementCount(), C);
902902

903903
return C;
904904
}
@@ -1204,15 +1204,35 @@ Constant *ConstantVector::getImpl(ArrayRef<Constant*> V) {
12041204
return nullptr;
12051205
}
12061206

1207-
Constant *ConstantVector::getSplat(unsigned NumElts, Constant *V) {
1208-
// If this splat is compatible with ConstantDataVector, use it instead of
1209-
// ConstantVector.
1210-
if ((isa<ConstantFP>(V) || isa<ConstantInt>(V)) &&
1211-
ConstantDataSequential::isElementTypeCompatible(V->getType()))
1212-
return ConstantDataVector::getSplat(NumElts, V);
1207+
Constant *ConstantVector::getSplat(ElementCount EC, Constant *V) {
1208+
if (!EC.Scalable) {
1209+
// If this splat is compatible with ConstantDataVector, use it instead of
1210+
// ConstantVector.
1211+
if ((isa<ConstantFP>(V) || isa<ConstantInt>(V)) &&
1212+
ConstantDataSequential::isElementTypeCompatible(V->getType()))
1213+
return ConstantDataVector::getSplat(EC.Min, V);
12131214

1214-
SmallVector<Constant*, 32> Elts(NumElts, V);
1215-
return get(Elts);
1215+
SmallVector<Constant *, 32> Elts(EC.Min, V);
1216+
return get(Elts);
1217+
}
1218+
1219+
Type *VTy = VectorType::get(V->getType(), EC);
1220+
1221+
if (V->isNullValue())
1222+
return ConstantAggregateZero::get(VTy);
1223+
else if (isa<UndefValue>(V))
1224+
return UndefValue::get(VTy);
1225+
1226+
Type *I32Ty = Type::getInt32Ty(VTy->getContext());
1227+
1228+
// Move scalar into vector.
1229+
Constant *UndefV = UndefValue::get(VTy);
1230+
V = ConstantExpr::getInsertElement(UndefV, V, ConstantInt::get(I32Ty, 0));
1231+
// Build shuffle mask to perform the splat.
1232+
Type *MaskTy = VectorType::get(I32Ty, EC);
1233+
Constant *Zeros = ConstantAggregateZero::get(MaskTy);
1234+
// Splat.
1235+
return ConstantExpr::getShuffleVector(V, UndefV, Zeros);
12161236
}
12171237

12181238
ConstantTokenNone *ConstantTokenNone::get(LLVMContext &Context) {
@@ -2098,15 +2118,15 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C,
20982118
unsigned AS = C->getType()->getPointerAddressSpace();
20992119
Type *ReqTy = DestTy->getPointerTo(AS);
21002120

2101-
unsigned NumVecElts = 0;
2102-
if (C->getType()->isVectorTy())
2103-
NumVecElts = C->getType()->getVectorNumElements();
2121+
ElementCount EltCount = {0, false};
2122+
if (VectorType *VecTy = dyn_cast<VectorType>(C->getType()))
2123+
EltCount = VecTy->getElementCount();
21042124
else for (auto Idx : Idxs)
2105-
if (Idx->getType()->isVectorTy())
2106-
NumVecElts = Idx->getType()->getVectorNumElements();
2125+
if (VectorType *VecTy = dyn_cast<VectorType>(Idx->getType()))
2126+
EltCount = VecTy->getElementCount();
21072127

2108-
if (NumVecElts)
2109-
ReqTy = VectorType::get(ReqTy, NumVecElts);
2128+
if (EltCount.Min != 0)
2129+
ReqTy = VectorType::get(ReqTy, EltCount);
21102130

21112131
if (OnlyIfReducedTy == ReqTy)
21122132
return nullptr;
@@ -2117,12 +2137,12 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C,
21172137
ArgVec.push_back(C);
21182138
for (unsigned i = 0, e = Idxs.size(); i != e; ++i) {
21192139
assert((!Idxs[i]->getType()->isVectorTy() ||
2120-
Idxs[i]->getType()->getVectorNumElements() == NumVecElts) &&
2140+
Idxs[i]->getType()->getVectorElementCount() == EltCount) &&
21212141
"getelementptr index type missmatch");
21222142

21232143
Constant *Idx = cast<Constant>(Idxs[i]);
2124-
if (NumVecElts && !Idxs[i]->getType()->isVectorTy())
2125-
Idx = ConstantVector::getSplat(NumVecElts, Idx);
2144+
if (EltCount.Min != 0 && !Idxs[i]->getType()->isVectorTy())
2145+
Idx = ConstantVector::getSplat(EltCount, Idx);
21262146
ArgVec.push_back(Idx);
21272147
}
21282148

@@ -2759,7 +2779,7 @@ Constant *ConstantDataVector::getSplat(unsigned NumElts, Constant *V) {
27592779
return getFP(V->getContext(), Elts);
27602780
}
27612781
}
2762-
return ConstantVector::getSplat(NumElts, V);
2782+
return ConstantVector::getSplat({NumElts, false}, V);
27632783
}
27642784

27652785

‎llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5379,8 +5379,9 @@ static Instruction *foldVectorCmp(CmpInst &Cmp,
53795379
if (ScalarC && ScalarM) {
53805380
// We allow undefs in matching, but this transform removes those for safety.
53815381
// Demanded elements analysis should be able to recover some/all of that.
5382-
C = ConstantVector::getSplat(V1Ty->getVectorNumElements(), ScalarC);
5383-
M = ConstantVector::getSplat(M->getType()->getVectorNumElements(), ScalarM);
5382+
C = ConstantVector::getSplat(V1Ty->getVectorElementCount(), ScalarC);
5383+
M = ConstantVector::getSplat(M->getType()->getVectorElementCount(),
5384+
ScalarM);
53845385
Value *NewCmp = IsFP ? Builder.CreateFCmp(Pred, V1, C)
53855386
: Builder.CreateICmp(Pred, V1, C);
53865387
return new ShuffleVectorInst(NewCmp, UndefValue::get(NewCmp->getType()), M);

‎llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -774,7 +774,7 @@ Instruction *InstCombiner::FoldShiftByConstant(Value *Op0, Constant *Op1,
774774
APInt Bits = APInt::getHighBitsSet(TypeBits, TypeBits - Op1Val);
775775
Constant *Mask = ConstantInt::get(I.getContext(), Bits);
776776
if (VectorType *VT = dyn_cast<VectorType>(X->getType()))
777-
Mask = ConstantVector::getSplat(VT->getNumElements(), Mask);
777+
Mask = ConstantVector::getSplat(VT->getElementCount(), Mask);
778778
return BinaryOperator::CreateAnd(X, Mask);
779779
}
780780

@@ -809,7 +809,7 @@ Instruction *InstCombiner::FoldShiftByConstant(Value *Op0, Constant *Op1,
809809
APInt Bits = APInt::getHighBitsSet(TypeBits, TypeBits - Op1Val);
810810
Constant *Mask = ConstantInt::get(I.getContext(), Bits);
811811
if (VectorType *VT = dyn_cast<VectorType>(X->getType()))
812-
Mask = ConstantVector::getSplat(VT->getNumElements(), Mask);
812+
Mask = ConstantVector::getSplat(VT->getElementCount(), Mask);
813813
return BinaryOperator::CreateAnd(X, Mask);
814814
}
815815

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1717,9 +1717,10 @@ void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
17171717
// FIXME: If the step is non-constant, we create the vector splat with
17181718
// IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't
17191719
// handle a constant vector splat.
1720-
Value *SplatVF = isa<Constant>(Mul)
1721-
? ConstantVector::getSplat(VF, cast<Constant>(Mul))
1722-
: Builder.CreateVectorSplat(VF, Mul);
1720+
Value *SplatVF =
1721+
isa<Constant>(Mul)
1722+
? ConstantVector::getSplat({VF, false}, cast<Constant>(Mul))
1723+
: Builder.CreateVectorSplat(VF, Mul);
17231724
Builder.restoreIP(CurrIP);
17241725

17251726
// We may need to add the step a number of times, depending on the unroll
@@ -3731,7 +3732,7 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi) {
37313732
// incoming scalar reduction.
37323733
VectorStart = ReductionStartValue;
37333734
} else {
3734-
Identity = ConstantVector::getSplat(VF, Iden);
3735+
Identity = ConstantVector::getSplat({VF, false}, Iden);
37353736

37363737
// This vector is the Identity vector where the first element is the
37373738
// incoming scalar reduction.

0 commit comments

Comments
 (0)