From 3529e4b2e997ba032acc07d2d9ec45fe2e0f93f6 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Tue, 12 Nov 2024 21:56:37 +0000 Subject: [PATCH 1/2] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?= =?UTF-8?q?itial=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Created using spr 1.3.5 --- .../Transforms/Vectorize/SLPVectorizer.cpp | 273 +++++++++++++----- .../AArch64/reused-scalar-repeated-in-node.ll | 30 +- .../vectorize-free-extracts-inserts.ll | 12 +- .../SLPVectorizer/X86/landing_pad.ll | 11 +- 4 files changed, 231 insertions(+), 95 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 1bf082d57b8bb..bbc2560778541 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -456,15 +456,18 @@ static std::string shortBundleName(ArrayRef VL, int Idx = -1) { /// \returns true if all of the instructions in \p VL are in the same block or /// false otherwise. static bool allSameBlock(ArrayRef VL) { - Instruction *I0 = dyn_cast(VL[0]); - if (!I0) + auto *It = find_if(VL, IsaPred); + if (It == VL.end()) return false; + Instruction *I0 = cast(*It); if (all_of(VL, isVectorLikeInstWithConstOps)) return true; BasicBlock *BB = I0->getParent(); - for (int I = 1, E = VL.size(); I < E; I++) { - auto *II = dyn_cast(VL[I]); + for (Value *V : iterator_range(It, VL.end())) { + if (isa(V)) + continue; + auto *II = dyn_cast(V); if (!II) return false; @@ -893,10 +896,19 @@ static bool isCmpSameOrSwapped(const CmpInst *BaseCI, const CmpInst *CI, static InstructionsState getSameOpcode(ArrayRef VL, const TargetLibraryInfo &TLI) { // Make sure these are all Instructions. - if (!all_of(VL, IsaPred)) + if (!all_of(VL, IsaPred)) + return InstructionsState::invalid(); + + auto *It = find_if(VL, IsaPred); + if (It == VL.end()) + return InstructionsState::invalid(); + + Value *V = *It; + unsigned InstCnt = std::count_if(It, VL.end(), IsaPred); + if ((VL.size() > 2 && !isa(V) && InstCnt < VL.size() / 2) || + (VL.size() == 2 && InstCnt < 2)) return InstructionsState::invalid(); - Value *V = VL.front(); bool IsCastOp = isa(V); bool IsBinOp = isa(V); bool IsCmpOp = isa(V); @@ -904,7 +916,7 @@ static InstructionsState getSameOpcode(ArrayRef VL, IsCmpOp ? cast(V)->getPredicate() : CmpInst::BAD_ICMP_PREDICATE; unsigned Opcode = cast(V)->getOpcode(); unsigned AltOpcode = Opcode; - unsigned AltIndex = 0; + unsigned AltIndex = std::distance(VL.begin(), It); bool SwappedPredsCompatible = [&]() { if (!IsCmpOp) @@ -940,8 +952,17 @@ static InstructionsState getSameOpcode(ArrayRef VL, if (!isTriviallyVectorizable(BaseID) && BaseMappings.empty()) return InstructionsState::invalid(); } + bool AnyPoison = InstCnt != VL.size(); for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) { - auto *I = cast(VL[Cnt]); + auto *I = dyn_cast(VL[Cnt]); + if (!I) + continue; + + // Cannot combine poison and divisions. + if (AnyPoison && (I->isIntDivRem() || I->isArithmeticShift() || + I->getOpcode() == Instruction::FDiv || + I->getOpcode() == Instruction::FRem || isa(I))) + return InstructionsState::invalid(); unsigned InstOpcode = I->getOpcode(); if (IsBinOp && isa(I)) { if (InstOpcode == Opcode || InstOpcode == AltOpcode) @@ -1177,10 +1198,13 @@ static SmallBitVector getAltInstrMask(ArrayRef VL, unsigned Opcode0, Type *ScalarTy = VL[0]->getType(); unsigned ScalarTyNumElements = getNumElements(ScalarTy); SmallBitVector OpcodeMask(VL.size() * ScalarTyNumElements, false); - for (unsigned Lane : seq(VL.size())) + for (unsigned Lane : seq(VL.size())) { + if (isa(VL[Lane])) + continue; if (cast(VL[Lane])->getOpcode() == Opcode1) OpcodeMask.set(Lane * ScalarTyNumElements, Lane * ScalarTyNumElements + ScalarTyNumElements); + } return OpcodeMask; } @@ -1781,13 +1805,17 @@ class BoUpSLP { (S.MainOp->getNumOperands() <= 2 || !MainAltOps.empty() || !S.isAltShuffle()) && all_of(Ops, [&S](Value *V) { - return cast(V)->getNumOperands() == - S.MainOp->getNumOperands(); + return isa(V) || + cast(V)->getNumOperands() == + S.MainOp->getNumOperands(); })) return S.isAltShuffle() ? LookAheadHeuristics::ScoreAltOpcodes : LookAheadHeuristics::ScoreSameOpcode; } + if (I1 && isa(V2)) + return LookAheadHeuristics::ScoreSameOpcode; + if (isa(V2)) return LookAheadHeuristics::ScoreUndef; @@ -2336,17 +2364,17 @@ class BoUpSLP { assert(!VL.empty() && "Bad VL"); assert((empty() || VL.size() == getNumLanes()) && "Expected same number of lanes"); - assert(isa(VL[0]) && "Expected instruction"); constexpr unsigned IntrinsicNumOperands = 2; - unsigned NumOperands = isa(VL[0]) - ? IntrinsicNumOperands - : cast(VL[0])->getNumOperands(); + auto *VL0 = cast(*find_if(VL, IsaPred)); + unsigned NumOperands = isa(VL0) ? IntrinsicNumOperands + : VL0->getNumOperands(); OpsVec.resize(NumOperands); unsigned NumLanes = VL.size(); for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) { OpsVec[OpIdx].resize(NumLanes); for (unsigned Lane = 0; Lane != NumLanes; ++Lane) { - assert(isa(VL[Lane]) && "Expected instruction"); + assert((isa(VL[Lane]) || isa(VL[Lane])) && + "Expected instruction or poison value"); // Our tree has just 3 nodes: the root and two operands. // It is therefore trivial to get the APO. We only need to check the // opcode of VL[Lane] and whether the operand at OpIdx is the LHS or @@ -2357,6 +2385,12 @@ class BoUpSLP { // Since operand reordering is performed on groups of commutative // operations or alternating sequences (e.g., +, -), we can safely // tell the inverse operations by checking commutativity. + if (isa(VL[Lane])) { + OpsVec[OpIdx][Lane] = { + PoisonValue::get(VL0->getOperand(OpIdx)->getType()), true, + false}; + continue; + } bool IsInverseOperation = !isCommutative(cast(VL[Lane])); bool APO = (OpIdx == 0) ? false : IsInverseOperation; OpsVec[OpIdx][Lane] = {cast(VL[Lane])->getOperand(OpIdx), @@ -2451,7 +2485,7 @@ class BoUpSLP { Value *OpILn = getValue(OpI, Ln); return (L && L->isLoopInvariant(OpILn)) || (getSameOpcode({Op, OpILn}, TLI).getOpcode() && - Op->getParent() == cast(OpILn)->getParent()); + allSameBlock({Op, OpILn})); })) return true; } @@ -2463,7 +2497,8 @@ class BoUpSLP { VLOperands(ArrayRef RootVL, const BoUpSLP &R) : TLI(*R.TLI), DL(*R.DL), SE(*R.SE), R(R), L(R.LI->getLoopFor( - (cast(RootVL.front())->getParent()))) { + (cast(*find_if(RootVL, IsaPred)) + ->getParent()))) { // Append all the operands of RootVL. appendOperandsOfVL(RootVL); } @@ -3265,13 +3300,18 @@ class BoUpSLP { /// Set the operands of this bundle in their original order. void setOperandsInOrder() { assert(Operands.empty() && "Already initialized?"); - auto *I0 = cast(Scalars[0]); + auto *I0 = cast(*find_if(Scalars, IsaPred)); Operands.resize(I0->getNumOperands()); unsigned NumLanes = Scalars.size(); for (unsigned OpIdx = 0, NumOperands = I0->getNumOperands(); OpIdx != NumOperands; ++OpIdx) { Operands[OpIdx].resize(NumLanes); for (unsigned Lane = 0; Lane != NumLanes; ++Lane) { + if (isa(Scalars[Lane])) { + Operands[OpIdx][Lane] = + PoisonValue::get(I0->getOperand(OpIdx)->getType()); + continue; + } auto *I = cast(Scalars[Lane]); assert(I->getNumOperands() == NumOperands && "Expected same number of operands"); @@ -4891,8 +4931,8 @@ BoUpSLP::canVectorizeLoads(ArrayRef VL, const Value *VL0, PointerOps.resize(Sz); auto *POIter = PointerOps.begin(); for (Value *V : VL) { - auto *L = cast(V); - if (!L->isSimple()) + auto *L = dyn_cast(V); + if (!L || !L->isSimple()) return LoadsState::Gather; *POIter = L->getPointerOperand(); ++POIter; @@ -5470,6 +5510,8 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) { TE.ReuseShuffleIndices.end()); if (TE.getOpcode() == Instruction::ExtractElement && !TE.isAltShuffle() && all_of(TE.Scalars, [Sz](Value *V) { + if (isa(V)) + return true; std::optional Idx = getExtractIndex(cast(V)); return Idx && *Idx < Sz; })) { @@ -5554,7 +5596,8 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) { auto PHICompare = [&](unsigned I1, unsigned I2) { Value *V1 = TE.Scalars[I1]; Value *V2 = TE.Scalars[I2]; - if (V1 == V2 || (V1->getNumUses() == 0 && V2->getNumUses() == 0)) + if (V1 == V2 || (V1->getNumUses() == 0 && V2->getNumUses() == 0) || + isa(V1) || isa(V2)) return false; if (V1->getNumUses() < V2->getNumUses()) return true; @@ -7319,8 +7362,14 @@ bool BoUpSLP::areAltOperandsProfitable(const InstructionsState &S, for (unsigned I : seq(0, S.MainOp->getNumOperands())) { Operands.emplace_back(); // Prepare the operand vector. - for (Value *V : VL) + for (Value *V : VL) { + if (isa(V)) { + Operands.back().push_back( + PoisonValue::get(S.MainOp->getOperand(I)->getType())); + continue; + } Operands.back().push_back(cast(V)->getOperand(I)); + } } if (Operands.size() == 2) { // Try find best operands candidates. @@ -7427,8 +7476,11 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState( if (VL0->getNumOperands() > MaxPHINumOperands) return TreeEntry::NeedToGather; // Check for terminator values (e.g. invoke). - for (Value *V : VL) - for (Value *Incoming : cast(V)->incoming_values()) { + for (Value *V : VL) { + auto *PHI = dyn_cast(V); + if (!PHI) + continue; + for (Value *Incoming : PHI->incoming_values()) { Instruction *Term = dyn_cast(Incoming); if (Term && Term->isTerminator()) { LLVM_DEBUG(dbgs() @@ -7436,6 +7488,7 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState( return TreeEntry::NeedToGather; } } + } return TreeEntry::Vectorize; } @@ -7511,8 +7564,10 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState( if (DL->getTypeSizeInBits(ScalarTy) != DL->getTypeAllocSizeInBits(ScalarTy)) LLVM_DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n"); - else if (any_of(VL, - [](Value *V) { return !cast(V)->isSimple(); })) + else if (any_of(VL, [](Value *V) { + auto *LI = dyn_cast(V); + return !LI || !LI->isSimple(); + })) LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n"); else LLVM_DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n"); @@ -7536,6 +7591,8 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState( case Instruction::BitCast: { Type *SrcTy = VL0->getOperand(0)->getType(); for (Value *V : VL) { + if (isa(V)) + continue; Type *Ty = cast(V)->getOperand(0)->getType(); if (Ty != SrcTy || !isValidElementType(Ty)) { LLVM_DEBUG( @@ -7552,7 +7609,9 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState( CmpInst::Predicate SwapP0 = CmpInst::getSwappedPredicate(P0); Type *ComparedTy = VL0->getOperand(0)->getType(); for (Value *V : VL) { - CmpInst *Cmp = cast(V); + if (isa(V)) + continue; + auto *Cmp = cast(V); if ((Cmp->getPredicate() != P0 && Cmp->getPredicate() != SwapP0) || Cmp->getOperand(0)->getType() != ComparedTy) { LLVM_DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n"); @@ -7795,7 +7854,13 @@ class PHIHandler { } // Prepare the operand vector. for (auto [Idx, V] : enumerate(Phis)) { - auto *P = cast(V); + auto *P = dyn_cast(V); + if (!P) { + assert(isa(V) && + "Expected isa instruction or poison value."); + Operands[I][Idx] = V; + continue; + } if (P->getIncomingBlock(I) == InBB) Operands[I][Idx] = P->getIncomingValue(I); else @@ -7814,6 +7879,11 @@ class PHIHandler { Blocks.try_emplace(InBB).first->second.push_back(I); } for (auto [Idx, V] : enumerate(Phis)) { + if (isa(V)) { + for (unsigned I : seq(Main->getNumIncomingValues())) + Operands[I][Idx] = V; + continue; + } auto *P = cast(V); for (unsigned I : seq(0, P->getNumIncomingValues())) { BasicBlock *InBB = P->getIncomingBlock(I); @@ -7863,7 +7933,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, for (Value *V : VL) { if (isConstant(V)) { ReuseShuffleIndices.emplace_back( - isa(V) ? PoisonMaskElem : UniqueValues.size()); + isa(V) ? PoisonMaskElem : UniqueValues.size()); UniqueValues.emplace_back(V); continue; } @@ -7895,11 +7965,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, }))) { if (DoNotFail && UniquePositions.size() > 1 && NumUniqueScalarValues > 1 && S.MainOp->isSafeToRemove() && - all_of(UniqueValues, [=](Value *V) { - return isa(V) || - areAllUsersVectorized(cast(V), - UserIgnoreList); - })) { + all_of(UniqueValues, IsaPred)) { // Find the number of elements, which forms full vectors. unsigned PWSz = getFullVectorNumberOfElements( *TTI, UniqueValues.front()->getType(), UniqueValues.size()); @@ -7907,8 +7973,9 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, ReuseShuffleIndices.clear(); } else { NonUniqueValueVL.assign(UniqueValues.begin(), UniqueValues.end()); - NonUniqueValueVL.append(PWSz - UniqueValues.size(), - UniqueValues.back()); + NonUniqueValueVL.append( + PWSz - UniqueValues.size(), + PoisonValue::get(UniqueValues.front()->getType())); VL = NonUniqueValueVL; } return true; @@ -8043,7 +8110,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, return true; // Check if all operands are extracts, part of vector node or can build a // regular vectorize node. - SmallVector InstsCount(VL.size(), 0); + SmallVector InstsCount; for (Value *V : VL) { auto *I = cast(V); InstsCount.push_back(count_if(I->operand_values(), [](Value *Op) { @@ -8437,6 +8504,11 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, } else { // Collect operands - commute if it uses the swapped predicate. for (Value *V : VL) { + if (isa(V)) { + Left.push_back(PoisonValue::get(VL0->getOperand(0)->getType())); + Right.push_back(PoisonValue::get(VL0->getOperand(1)->getType())); + continue; + } auto *Cmp = cast(V); Value *LHS = Cmp->getOperand(0); Value *RHS = Cmp->getOperand(1); @@ -8636,7 +8708,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, if (isa(VL0) || CI) { ValueList Left, Right; if (!CI || all_of(VL, [](Value *V) { - return cast(V)->isCommutative(); + return isa(V) || cast(V)->isCommutative(); })) { reorderInputsAccordingToOpcode(VL, Left, Right, *this); } else { @@ -8649,6 +8721,13 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, // Collect operands - commute if it uses the swapped predicate or // alternate operation. for (Value *V : VL) { + if (isa(V)) { + Left.push_back( + PoisonValue::get(MainCI->getOperand(0)->getType())); + Right.push_back( + PoisonValue::get(MainCI->getOperand(1)->getType())); + continue; + } auto *Cmp = cast(V); Value *LHS = Cmp->getOperand(0); Value *RHS = Cmp->getOperand(1); @@ -8853,6 +8932,8 @@ void BoUpSLP::TreeEntry::buildAltOpShuffleMask( unsigned Idx = I; if (!ReorderIndices.empty()) Idx = OrderMask[I]; + if (isa(Scalars[Idx])) + continue; auto *OpInst = cast(Scalars[Idx]); if (IsAltOp(OpInst)) { Mask[I] = Sz + Idx; @@ -9627,9 +9708,11 @@ void BoUpSLP::transformNodes() { // Try to vectorize reduced values or if all users are vectorized. // For expensive instructions extra extracts might be profitable. if ((!UserIgnoreList || E.Idx != 0) && - TTI->getInstructionCost(cast(Slice.front()), - CostKind) < TTI::TCC_Expensive && + TTI->getInstructionCost(S.MainOp, CostKind) < + TTI::TCC_Expensive && !all_of(Slice, [&](Value *V) { + if (isa(V)) + return true; return areAllUsersVectorized(cast(V), UserIgnoreList); })) @@ -9652,12 +9735,13 @@ void BoUpSLP::transformNodes() { continue; } } else if (S.getOpcode() == Instruction::ExtractElement || - (TTI->getInstructionCost( - cast(Slice.front()), CostKind) < + (TTI->getInstructionCost(S.MainOp, CostKind) < TTI::TCC_Expensive && !CheckOperandsProfitability( - cast(Slice.front()), - cast(Slice.back()), S))) { + S.MainOp, + cast(*find_if(reverse(Slice), + IsaPred)), + S))) { // Do not vectorize extractelements (handled effectively // alread). Do not vectorize non-profitable instructions (with // low cost and non-vectorizable operands.) @@ -10887,7 +10971,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, const unsigned Sz = UniqueValues.size(); SmallBitVector UsedScalars(Sz, false); for (unsigned I = 0; I < Sz; ++I) { - if (getTreeEntry(UniqueValues[I]) == E) + if (isa(UniqueValues[I]) && getTreeEntry(UniqueValues[I]) == E) continue; UsedScalars.set(I); } @@ -11026,6 +11110,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, case Instruction::ExtractValue: case Instruction::ExtractElement: { auto GetScalarCost = [&](unsigned Idx) { + if (isa(UniqueValues[Idx])) + return InstructionCost(TTI::TCC_Free); + auto *I = cast(UniqueValues[Idx]); VectorType *SrcVecTy; if (ShuffleOrOp == Instruction::ExtractElement) { @@ -11214,10 +11301,10 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, VecOpcode = Instruction::UIToFP; } auto GetScalarCost = [&](unsigned Idx) -> InstructionCost { - auto *VI = cast(UniqueValues[Idx]); + assert(Idx == 0 && "Expected 0 index only"); return TTI->getCastInstrCost(Opcode, VL0->getType(), VL0->getOperand(0)->getType(), - TTI::getCastContextHint(VI), CostKind, VI); + TTI::getCastContextHint(VL0), CostKind, VL0); }; auto GetVectorCost = [=](InstructionCost CommonCost) { // Do not count cost here if minimum bitwidth is in effect and it is just @@ -11245,6 +11332,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, ? CmpInst::BAD_FCMP_PREDICATE : CmpInst::BAD_ICMP_PREDICATE; auto GetScalarCost = [&](unsigned Idx) { + if (isa(UniqueValues[Idx])) + return InstructionCost(TTI::TCC_Free); + auto *VI = cast(UniqueValues[Idx]); CmpInst::Predicate CurrentPred = ScalarTy->isFloatingPointTy() ? CmpInst::BAD_FCMP_PREDICATE @@ -11325,6 +11415,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, case Instruction::Or: case Instruction::Xor: { auto GetScalarCost = [&](unsigned Idx) { + if (isa(UniqueValues[Idx])) + return InstructionCost(TTI::TCC_Free); + auto *VI = cast(UniqueValues[Idx]); unsigned OpIdx = isa(VI) ? 0 : 1; TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(VI->getOperand(0)); @@ -11503,6 +11596,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, return false; }; auto GetScalarCost = [&](unsigned Idx) { + if (isa(UniqueValues[Idx])) + return InstructionCost(TTI::TCC_Free); + auto *VI = cast(UniqueValues[Idx]); assert(E->isOpcodeOrAlt(VI) && "Unexpected main/alternate opcode"); (void)E; @@ -13285,8 +13381,8 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) { if (E->getOpcode() == Instruction::GetElementPtr && !isa(V)) return true; - auto *I = cast(V); - return !E->isOpcodeOrAlt(I) || I->getParent() == BB || + auto *I = dyn_cast(V); + return !I || !E->isOpcodeOrAlt(I) || I->getParent() == BB || isVectorLikeInstWithConstOps(I); })) && "Expected gathered loads or GEPs or instructions from same basic " @@ -13385,8 +13481,9 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) { })) || all_of(E->Scalars, [](Value *V) { - return !isVectorLikeInstWithConstOps(V) && - isUsedOutsideBlock(V); + return isa(V) || + (!isVectorLikeInstWithConstOps(V) && + isUsedOutsideBlock(V)); }) || (E->isGather() && E->Idx == 0 && all_of(E->Scalars, [](Value *V) { return isa(V) || @@ -13911,12 +14008,16 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis { Value *V1 = E1.VectorizedValue; if (V1->getType()->isIntOrIntVectorTy()) V1 = castToScalarTyElem(V1, any_of(E1.Scalars, [&](Value *V) { + if (isa(V)) + return false; return !isKnownNonNegative( V, SimplifyQuery(*R.DL)); })); Value *V2 = E2.VectorizedValue; if (V2->getType()->isIntOrIntVectorTy()) V2 = castToScalarTyElem(V2, any_of(E2.Scalars, [&](Value *V) { + if (isa(V)) + return false; return !isKnownNonNegative( V, SimplifyQuery(*R.DL)); })); @@ -13928,6 +14029,8 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis { Value *V1 = E1.VectorizedValue; if (V1->getType()->isIntOrIntVectorTy()) V1 = castToScalarTyElem(V1, any_of(E1.Scalars, [&](Value *V) { + if (isa(V)) + return false; return !isKnownNonNegative( V, SimplifyQuery(*R.DL)); })); @@ -14089,6 +14192,8 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis { Value *V = E->VectorizedValue; if (V->getType()->isIntOrIntVectorTy()) V = castToScalarTyElem(V, any_of(E->Scalars, [&](Value *V) { + if (isa(V)) + return false; return !isKnownNonNegative( V, SimplifyQuery(*R.DL)); })); @@ -14805,6 +14910,16 @@ Value *BoUpSLP::createBuildVector(const TreeEntry *E, Type *ScalarTy, Builder, *this); } +/// \returns \p I after propagating metadata from \p VL only for instructions in +/// \p VL. +static Instruction *propagateMetadata(Instruction *Inst, ArrayRef VL) { + SmallVector Insts; + for (Value *V : VL) + if (isa(V)) + Insts.push_back(V); + return llvm::propagateMetadata(Inst, Insts); +} + Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { IRBuilderBase::InsertPointGuard Guard(Builder); @@ -14874,6 +14989,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { IsSigned = It->second.second; else IsSigned = any_of(OpE->Scalars, [&](Value *R) { + if (isa(V)) + return false; return !isKnownNonNegative(R, SimplifyQuery(*DL)); }); return IsSigned; @@ -14962,7 +15079,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { Builder.SetInsertPoint(LI); Value *Ptr = LI->getPointerOperand(); LoadInst *V = Builder.CreateAlignedLoad(VecTy, Ptr, LI->getAlign()); - Value *NewV = propagateMetadata(V, E->Scalars); + Value *NewV = ::propagateMetadata(V, E->Scalars); NewV = FinalShuffle(NewV, E); E->VectorizedValue = NewV; return NewV; @@ -15295,7 +15412,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { static_cast(E->getOpcode()), Op); propagateIRFlags(V, E->Scalars, VL0); if (auto *I = dyn_cast(V)) - V = propagateMetadata(I, E->Scalars); + V = ::propagateMetadata(I, E->Scalars); V = FinalShuffle(V, E); @@ -15389,11 +15506,11 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { RHS); propagateIRFlags(V, E->Scalars, VL0, It == MinBWs.end()); if (auto *I = dyn_cast(V)) { - V = propagateMetadata(I, E->Scalars); + V = ::propagateMetadata(I, E->Scalars); // Drop nuw flags for abs(sub(commutative), true). if (!MinBWs.contains(E) && ShuffleOrOp == Instruction::Sub && any_of(E->Scalars, [](Value *V) { - return isCommutative(cast(V)); + return isa(V) || isCommutative(cast(V)); })) I->setHasNoUnsignedWrap(/*b=*/false); } @@ -15488,7 +15605,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { Align CommonAlignment = computeCommonAlignment(E->Scalars); NewLI = Builder.CreateMaskedGather(VecTy, VecPtr, CommonAlignment); } - Value *V = propagateMetadata(NewLI, E->Scalars); + Value *V = ::propagateMetadata(NewLI, E->Scalars); V = FinalShuffle(V, E); E->VectorizedValue = V; @@ -15533,7 +15650,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { ST = Inst; } - Value *V = propagateMetadata(ST, E->Scalars); + Value *V = ::propagateMetadata(ST, E->Scalars); E->VectorizedValue = V; ++NumVectorInstructions; @@ -15566,7 +15683,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { if (isa(V)) GEPs.push_back(V); } - V = propagateMetadata(I, GEPs); + V = ::propagateMetadata(I, GEPs); } V = FinalShuffle(V, E); @@ -15678,7 +15795,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { V = Builder.CreateShuffleVector(SVSrc->getOperand(0), NewMask); propagateIRFlags(V, E->Scalars, VL0); if (auto *I = dyn_cast(V)) - V = propagateMetadata(I, E->Scalars); + V = ::propagateMetadata(I, E->Scalars); V = FinalShuffle(V, E); } else { assert(E->isAltShuffle() && @@ -15755,7 +15872,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { assert(LHS->getType() == VecTy && "Expected same type as operand."); if (auto *I = dyn_cast(LHS)) - LHS = propagateMetadata(I, E->Scalars); + LHS = ::propagateMetadata(I, E->Scalars); LHS = FinalShuffle(LHS, E); E->VectorizedValue = LHS; ++NumVectorInstructions; @@ -15796,9 +15913,10 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { if (auto *I = dyn_cast(Vec); I && Opcode == Instruction::Sub && !MinBWs.contains(E) && any_of(E->Scalars, [](Value *V) { + if (isa(V)) + return false; auto *IV = cast(V); - return IV->getOpcode() == Instruction::Sub && - isCommutative(cast(IV)); + return IV->getOpcode() == Instruction::Sub && isCommutative(IV); })) I->setHasNoUnsignedWrap(/*b=*/false); }; @@ -15811,7 +15929,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { } V = Builder.CreateShuffleVector(V0, V1, Mask); if (auto *I = dyn_cast(V)) { - V = propagateMetadata(I, E->Scalars); + V = ::propagateMetadata(I, E->Scalars); GatherShuffleExtractSeq.insert(I); CSEBlocks.insert(I->getParent()); } @@ -16389,6 +16507,8 @@ BoUpSLP::vectorizeTree(const ExtraValueToDebugLocsMap &ExternallyUsedValues, if (auto *EE = dyn_cast(Scalar); EE && IgnoredExtracts.contains(EE)) continue; + if (isa(Scalar)) + continue; #ifndef NDEBUG Type *Ty = Scalar->getType(); if (!Ty->isVoidTy()) { @@ -17287,9 +17407,13 @@ bool BoUpSLP::collectValuesToDemote( // by the insertelement instruction and not used in multiple vector nodes, it // cannot be demoted. bool IsSignedNode = any_of(E.Scalars, [&](Value *R) { + if (isa(R)) + return false; return !isKnownNonNegative(R, SimplifyQuery(*DL)); }); auto IsPotentiallyTruncated = [&](Value *V, unsigned &BitWidth) -> bool { + if (isa(V)) + return true; if (MultiNodeScalars.contains(V)) return false; // For lat shuffle of sext/zext with many uses need to check the extra bit @@ -17472,6 +17596,8 @@ bool BoUpSLP::collectValuesToDemote( // inrange amount, we can always perform a SHL in a smaller type. auto ShlChecker = [&](unsigned BitWidth, unsigned) { return all_of(E.Scalars, [&](Value *V) { + if (isa(V)) + return true; auto *I = cast(V); KnownBits AmtKnownBits = computeKnownBits(I->getOperand(1), *DL); return AmtKnownBits.getMaxValue().ult(BitWidth); @@ -17486,6 +17612,8 @@ bool BoUpSLP::collectValuesToDemote( // already zeros. auto LShrChecker = [&](unsigned BitWidth, unsigned OrigBitWidth) { return all_of(E.Scalars, [&](Value *V) { + if (isa(V)) + return true; auto *I = cast(V); KnownBits AmtKnownBits = computeKnownBits(I->getOperand(1), *DL); APInt ShiftedBits = APInt::getBitsSetFrom(OrigBitWidth, BitWidth); @@ -17755,6 +17883,8 @@ void BoUpSLP::computeMinimumValueSizes() { // Determine if the sign bit of all the roots is known to be zero. If not, // IsKnownPositive is set to False. bool IsKnownPositive = !IsSignedCmp && all_of(E.Scalars, [&](Value *R) { + if (isa(R)) + return true; KnownBits Known = computeKnownBits(R, *DL); return Known.isNonNegative(); }); @@ -17762,6 +17892,8 @@ void BoUpSLP::computeMinimumValueSizes() { // We first check if all the bits of the roots are demanded. If they're not, // we can truncate the roots to this narrower type. for (Value *Root : E.Scalars) { + if (isa(Root)) + continue; unsigned NumSignBits = ComputeNumSignBits(Root, *DL, 0, AC, nullptr, DT); TypeSize NumTypeBits = DL->getTypeSizeInBits(Root->getType()->getScalarType()); @@ -17818,9 +17950,8 @@ void BoUpSLP::computeMinimumValueSizes() { !(((Opcode == Instruction::SExt || Opcode == Instruction::ZExt) && (!IsTopRoot || !(IsStoreOrInsertElt || UserIgnoreList) || DL->getTypeSizeInBits(TreeRootIT) / - DL->getTypeSizeInBits(cast(E.Scalars.front()) - ->getOperand(0) - ->getType()) > + DL->getTypeSizeInBits( + E.getMainOp()->getOperand(0)->getType()) > 2))))) return 0u; // Round MaxBitWidth up to the next power-of-two. @@ -17836,6 +17967,8 @@ void BoUpSLP::computeMinimumValueSizes() { if (UserIgnoreList && isa(VectorizableTree.front()->Scalars.front()->getType())) { for (Value *V : *UserIgnoreList) { + if (isa(V)) + continue; auto NumSignBits = ComputeNumSignBits(V, *DL, 0, AC, nullptr, DT); auto NumTypeBits = DL->getTypeSizeInBits(V->getType()); unsigned BitWidth1 = NumTypeBits - NumSignBits; @@ -17950,8 +18083,10 @@ void BoUpSLP::computeMinimumValueSizes() { if (MinBWs.contains(TE)) continue; bool IsSigned = any_of(TE->Scalars, [&](Value *R) { - return !isKnownNonNegative(R, SimplifyQuery(*DL)); - }); + if (isa(R)) + return false; + return !isKnownNonNegative(R, SimplifyQuery(*DL)); + }); MinBWs.try_emplace(TE, MaxBitWidth, IsSigned); } } diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/reused-scalar-repeated-in-node.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/reused-scalar-repeated-in-node.ll index d6073ea4bbbae..289807a808d5d 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/reused-scalar-repeated-in-node.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/reused-scalar-repeated-in-node.ll @@ -12,7 +12,7 @@ define void @test() { ; CHECK: [[BB63]]: ; CHECK-NEXT: br label %[[BB64]] ; CHECK: [[BB64]]: -; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x float> [ poison, %[[BB61]] ], [ poison, %[[BB63]] ], [ poison, %[[BB62]] ] +; CHECK-NEXT: [[TMP25:%.*]] = phi <16 x float> [ poison, %[[BB61]] ], [ poison, %[[BB63]] ], [ poison, %[[BB62]] ] ; CHECK-NEXT: [[I66:%.*]] = load float, ptr poison, align 16 ; CHECK-NEXT: [[I67:%.*]] = load float, ptr poison, align 4 ; CHECK-NEXT: [[I68:%.*]] = load float, ptr poison, align 8 @@ -37,28 +37,30 @@ define void @test() { ; CHECK-NEXT: [[TMP11:%.*]] = insertelement <16 x float> [[TMP10]], float [[I69]], i32 15 ; CHECK-NEXT: br i1 poison, label %[[BB167:.*]], label %[[BB77:.*]] ; CHECK: [[BB77]]: -; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x float> [[TMP11]], <16 x float> poison, <8 x i32> -; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x float> [[TMP0]], <2 x float> poison, <16 x i32> -; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <2 x float> [[TMP0]], <2 x float> poison, <16 x i32> +; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x float> [[TMP11]], <16 x float> poison, <8 x i32> +; CHECK-NEXT: [[TMP17:%.*]] = insertelement <8 x float> [[TMP12]], float [[I70]], i32 0 +; CHECK-NEXT: [[TMP30:%.*]] = insertelement <2 x float> poison, float [[I68]], i32 0 +; CHECK-NEXT: [[TMP31:%.*]] = insertelement <2 x float> [[TMP30]], float [[I66]], i32 1 +; CHECK-NEXT: [[TMP39:%.*]] = shufflevector <16 x float> [[TMP25]], <16 x float> poison, <16 x i32> +; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <16 x float> [[TMP39]], <16 x float> [[TMP25]], <16 x i32> ; CHECK-NEXT: br label %[[BB78:.*]] ; CHECK: [[BB78]]: -; CHECK-NEXT: [[TMP15:%.*]] = phi <8 x float> [ [[TMP12]], %[[BB77]] ], [ [[TMP30:%.*]], %[[BB78]] ] -; CHECK-NEXT: [[TMP16:%.*]] = phi <2 x float> [ poison, %[[BB77]] ], [ [[TMP31:%.*]], %[[BB78]] ] -; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <8 x float> [[TMP15]], <8 x float> poison, <16 x i32> -; CHECK-NEXT: [[TMP18:%.*]] = fmul fast <16 x float> [[TMP17]], [[TMP13]] -; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <8 x float> [[TMP15]], <8 x float> poison, <16 x i32> +; CHECK-NEXT: [[TMP15:%.*]] = phi <8 x float> [ [[TMP17]], %[[BB77]] ], [ [[TMP36:%.*]], %[[BB78]] ] +; CHECK-NEXT: [[TMP16:%.*]] = phi <2 x float> [ [[TMP31]], %[[BB77]] ], [ [[TMP37:%.*]], %[[BB78]] ] +; CHECK-NEXT: [[TMP38:%.*]] = shufflevector <8 x float> [[TMP15]], <8 x float> poison, <16 x i32> +; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <8 x float> [[TMP15]], <8 x float> poison, <16 x i32> ; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <2 x float> [[TMP16]], <2 x float> poison, <16 x i32> ; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <16 x float> [[TMP19]], <16 x float> [[TMP20]], <16 x i32> ; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <8 x float> [[TMP15]], <8 x float> poison, <16 x i32> -; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <16 x float> [[TMP21]], <16 x float> [[TMP22]], <16 x i32> +; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <16 x float> [[TMP21]], <16 x float> [[TMP22]], <16 x i32> ; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <16 x float> [[TMP23]], <16 x float> poison, <16 x i32> -; CHECK-NEXT: [[TMP25:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v2f32(<16 x float> [[TMP14]], <2 x float> [[TMP0]], i64 2) -; CHECK-NEXT: [[TMP26:%.*]] = fmul fast <16 x float> [[TMP24]], [[TMP25]] +; CHECK-NEXT: [[TMP18:%.*]] = fmul fast <16 x float> [[TMP24]], [[TMP13]] +; CHECK-NEXT: [[TMP26:%.*]] = fmul fast <16 x float> [[TMP38]], [[TMP25]] ; CHECK-NEXT: [[TMP27:%.*]] = fadd fast <16 x float> [[TMP26]], [[TMP18]] ; CHECK-NEXT: [[TMP28:%.*]] = fadd fast <16 x float> [[TMP27]], poison ; CHECK-NEXT: [[TMP29:%.*]] = fadd fast <16 x float> [[TMP28]], poison -; CHECK-NEXT: [[TMP30]] = shufflevector <16 x float> [[TMP29]], <16 x float> poison, <8 x i32> -; CHECK-NEXT: [[TMP31]] = shufflevector <16 x float> [[TMP29]], <16 x float> poison, <2 x i32> +; CHECK-NEXT: [[TMP36]] = shufflevector <16 x float> [[TMP29]], <16 x float> poison, <8 x i32> +; CHECK-NEXT: [[TMP37]] = shufflevector <16 x float> [[TMP29]], <16 x float> poison, <2 x i32> ; CHECK-NEXT: br i1 poison, label %[[BB78]], label %[[BB167]] ; CHECK: [[BB167]]: ; CHECK-NEXT: [[TMP32:%.*]] = phi <16 x float> [ [[TMP11]], %[[BB64]] ], [ [[TMP29]], %[[BB78]] ] diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vectorize-free-extracts-inserts.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vectorize-free-extracts-inserts.ll index e2d1a29ee22de..4755c690c0711 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/vectorize-free-extracts-inserts.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vectorize-free-extracts-inserts.ll @@ -307,14 +307,14 @@ define void @noop_extracts_9_lanes(ptr %ptr.1, ptr %ptr.2) { ; CHECK-NEXT: [[V1_LANE_5:%.*]] = extractelement <9 x double> [[V_1]], i32 5 ; CHECK-NEXT: [[V_2:%.*]] = load <4 x double>, ptr [[PTR_2:%.*]], align 16 ; CHECK-NEXT: [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0 -; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> +; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> +; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> ; CHECK-NEXT: [[TMP2:%.*]] = fmul <8 x double> [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_0]] ; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <9 x i32> ; CHECK-NEXT: [[A_INS_8:%.*]] = insertelement <9 x double> [[TMP3]], double [[A_LANE_8]], i32 8 ; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> -; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> ; CHECK-NEXT: [[TMP6:%.*]] = fmul <8 x double> [[TMP4]], [[TMP5]] ; CHECK-NEXT: [[B_LANE_8:%.*]] = fmul double [[V1_LANE_5]], [[V2_LANE_0]] ; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> poison, <9 x i32> @@ -395,10 +395,10 @@ define void @first_mul_chain_jumbled(ptr %ptr.1, ptr %ptr.2) { ; CHECK-NEXT: [[V1_LANE_5:%.*]] = extractelement <9 x double> [[V_1]], i32 5 ; CHECK-NEXT: [[V_2:%.*]] = load <4 x double>, ptr [[PTR_2:%.*]], align 16 ; CHECK-NEXT: [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0 -; CHECK-NEXT: [[V2_LANE_1:%.*]] = extractelement <4 x double> [[V_2]], i32 1 -; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> +; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> ; CHECK-NEXT: [[TMP2:%.*]] = fmul <8 x double> [[TMP0]], [[TMP1]] +; CHECK-NEXT: [[V2_LANE_1:%.*]] = extractelement <4 x double> [[V_2]], i32 1 ; CHECK-NEXT: [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_1]] ; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <9 x i32> ; CHECK-NEXT: [[A_INS_8:%.*]] = insertelement <9 x double> [[TMP3]], double [[A_LANE_8]], i32 8 @@ -483,15 +483,15 @@ define void @first_and_second_mul_chain_jumbled(ptr %ptr.1, ptr %ptr.2) { ; CHECK-NEXT: [[V1_LANE_4:%.*]] = extractelement <9 x double> [[V_1]], i32 4 ; CHECK-NEXT: [[V_2:%.*]] = load <4 x double>, ptr [[PTR_2:%.*]], align 16 ; CHECK-NEXT: [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0 +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> +; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> ; CHECK-NEXT: [[V2_LANE_2:%.*]] = extractelement <4 x double> [[V_2]], i32 2 ; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> ; CHECK-NEXT: [[TMP2:%.*]] = fmul <8 x double> [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_0]] ; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <9 x i32> ; CHECK-NEXT: [[A_INS_8:%.*]] = insertelement <9 x double> [[TMP3]], double [[A_LANE_8]], i32 8 ; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> -; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> ; CHECK-NEXT: [[TMP6:%.*]] = fmul <8 x double> [[TMP4]], [[TMP5]] ; CHECK-NEXT: [[B_LANE_8:%.*]] = fmul double [[V1_LANE_4]], [[V2_LANE_2]] ; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> poison, <9 x i32> diff --git a/llvm/test/Transforms/SLPVectorizer/X86/landing_pad.ll b/llvm/test/Transforms/SLPVectorizer/X86/landing_pad.ll index 2a036cc8fe326..4282ae2ab88ec 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/landing_pad.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/landing_pad.ll @@ -33,16 +33,15 @@ define void @foo() personality ptr @bar { ; CHECK-NEXT: br i1 poison, label [[BB7]], label [[BB6]] ; CHECK: bb9: ; CHECK-NEXT: [[INDVARS_IV528799:%.*]] = phi i64 [ poison, [[BB10]] ], [ poison, [[BB12]] ] -; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ [[TMP8:%.*]], [[BB10]] ], [ [[TMP9:%.*]], [[BB12]] ] -; CHECK-NEXT: [[TMP7]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> +; CHECK-NEXT: [[TMP7]] = phi <4 x i32> [ [[TMP9:%.*]], [[BB10]] ], [ [[TMP11:%.*]], [[BB12]] ] ; CHECK-NEXT: br label [[BB2]] ; CHECK: bb10: ; CHECK-NEXT: [[LOCAL_10_38123_LCSSA:%.*]] = phi i32 [ [[TMP10]], [[BB3]] ] ; CHECK-NEXT: [[LOCAL_5_33118_LCSSA:%.*]] = phi i32 [ [[TMP4]], [[BB3]] ] ; CHECK-NEXT: [[LANDING_PAD68:%.*]] = landingpad { ptr, i32 } ; CHECK-NEXT: cleanup -; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> poison, i32 [[LOCAL_10_38123_LCSSA]], i32 0 -; CHECK-NEXT: [[TMP8]] = insertelement <2 x i32> [[TMP12]], i32 [[LOCAL_5_33118_LCSSA]], i32 1 +; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> poison, i32 [[LOCAL_10_38123_LCSSA]], i32 2 +; CHECK-NEXT: [[TMP9]] = insertelement <4 x i32> [[TMP8]], i32 [[LOCAL_5_33118_LCSSA]], i32 3 ; CHECK-NEXT: br label [[BB9]] ; CHECK: bb11: ; CHECK-NEXT: ret void @@ -51,8 +50,8 @@ define void @foo() personality ptr @bar { ; CHECK-NEXT: [[LOCAL_5_84111_LCSSA:%.*]] = phi i32 [ [[LOCAL_5_84111]], [[BB7]] ] ; CHECK-NEXT: [[LANDING_PAD149:%.*]] = landingpad { ptr, i32 } ; CHECK-NEXT: cleanup -; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> poison, i32 [[LOCAL_10_89113_LCSSA]], i32 0 -; CHECK-NEXT: [[TMP9]] = insertelement <2 x i32> [[TMP11]], i32 [[LOCAL_5_84111_LCSSA]], i32 1 +; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> poison, i32 [[LOCAL_10_89113_LCSSA]], i32 2 +; CHECK-NEXT: [[TMP11]] = insertelement <4 x i32> [[TMP12]], i32 [[LOCAL_5_84111_LCSSA]], i32 3 ; CHECK-NEXT: br label [[BB9]] ; bb1: From d90b7f3022a6610d6aced5c00f41da636442c577 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 13 Nov 2024 12:21:49 +0000 Subject: [PATCH 2/2] Address comments Created using spr 1.3.5 --- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index bbc2560778541..b12a6efff4501 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -959,8 +959,7 @@ static InstructionsState getSameOpcode(ArrayRef VL, continue; // Cannot combine poison and divisions. - if (AnyPoison && (I->isIntDivRem() || I->isArithmeticShift() || - I->getOpcode() == Instruction::FDiv || + if (AnyPoison && (I->isIntDivRem() || I->getOpcode() == Instruction::FDiv || I->getOpcode() == Instruction::FRem || isa(I))) return InstructionsState::invalid(); unsigned InstOpcode = I->getOpcode();