diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 2242f95e4226c..eb078c783d5f7 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1385,12 +1385,16 @@ static void simplifyRecipe(VPSingleDefRecipe *Def, VPTypeAnalysis &TypeInfo) { return; } - // Look through ExtractLastLane (BuildVector ....). - if (match(Def, m_ExtractLastLane(m_BuildVector()))) { - auto *BuildVector = cast(Def->getOperand(0)); - Def->replaceAllUsesWith( - BuildVector->getOperand(BuildVector->getNumOperands() - 1)); - return; + // Look through ExtractLastLane. + if (match(Def, m_ExtractLastLane(m_VPValue(A)))) { + if (match(A, m_BuildVector())) { + auto *BuildVector = cast(A); + Def->replaceAllUsesWith( + BuildVector->getOperand(BuildVector->getNumOperands() - 1)); + return; + } + if (Plan->hasScalarVFOnly()) + return Def->replaceAllUsesWith(A); } // Look through ExtractPenultimateElement (BuildVector ....). diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index 6fb706ea7d64b..7b4c524712d9a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -371,10 +371,9 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) { continue; } - if (match(&R, m_ExtractLastLaneOfLastPart(m_VPValue(Op0))) || - match(&R, m_ExtractPenultimateElement(m_VPValue(Op0)))) { - addUniformForAllParts(cast(&R)); - if (Plan.hasScalarVFOnly()) { + if (Plan.hasScalarVFOnly()) { + if (match(&R, m_ExtractLastPart(m_VPValue(Op0))) || + match(&R, m_ExtractPenultimateElement(m_VPValue(Op0)))) { auto *I = cast(&R); bool IsPenultimatePart = I->getOpcode() == VPInstruction::ExtractPenultimateElement; @@ -383,7 +382,10 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) { I->replaceAllUsesWith(getValueForPart(Op0, PartIdx)); continue; } - // For vector VF, always extract from the last part. + } + if (match(&R, m_ExtractLastLaneOfLastPart(m_VPValue(Op0))) || + match(&R, m_ExtractPenultimateElement(m_VPValue(Op0)))) { + addUniformForAllParts(cast(&R)); R.setOperand(0, getValueForPart(Op0, UF - 1)); continue; } diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll index bbd596a772c53..c77afa870e2c1 100644 --- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll +++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll @@ -220,7 +220,6 @@ exit: ; DBG-EMPTY: ; DBG-NEXT: middle.block: ; DBG-NEXT: EMIT vp<[[RESUME_1_PART:%.+]]> = extract-last-part vp<[[SCALAR_STEPS]]> -; DBG-NEXT: EMIT vp<[[RESUME_1:%.+]]> = extract-last-lane vp<[[RESUME_1_PART]]> ; DBG-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[TC]]>, vp<[[VEC_TC]]> ; DBG-NEXT: EMIT branch-on-cond vp<[[CMP]]> ; DBG-NEXT: Successor(s): ir-bb, scalar.ph @@ -230,7 +229,7 @@ exit: ; DBG-EMPTY: ; DBG-NEXT: scalar.ph: ; DBG-NEXT: EMIT-SCALAR vp<[[RESUME_IV:%.+]]> = phi [ vp<[[VTC]]>, middle.block ], [ ir<0>, ir-bb ] -; DBG-NEXT: EMIT-SCALAR vp<[[RESUME_P:%.*]]> = phi [ vp<[[RESUME_1]]>, middle.block ], [ ir<0>, ir-bb ] +; DBG-NEXT: EMIT-SCALAR vp<[[RESUME_P:%.*]]> = phi [ vp<[[RESUME_1_PART]]>, middle.block ], [ ir<0>, ir-bb ] ; DBG-NEXT: Successor(s): ir-bb ; DBG-EMPTY: ; DBG-NEXT: ir-bb: