diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 2fd0251e4bee1..b10b18268c289 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -11607,6 +11607,23 @@ class InstructionsCompatibilityAnalysis { return Operands.contains(I); })) continue; + // On a tie, keep the outer binary op as MainOp rather than replacing it + // with an inner op that appears as its direct operand. For example, in + // (2.0f * A) + B the fadd and fmul each appear once in VL; without this + // check the fmul could win and prevent vectorization of the fadd pair. + if (P.second.size() == BestOpcodeNum) { + auto *I = P.second.front(); + if (auto *MainBO = dyn_cast(MainOp)) { + auto *MainBOOp0 = dyn_cast(MainBO->getOperand(0)); + auto *MainBOOp1 = dyn_cast(MainBO->getOperand(1)); + if (MainBOOp0 && MainBOOp0->getOpcode() == I->getOpcode() && + MainBOOp0->getParent() == I->getParent()) + continue; + if (MainBOOp1 && MainBOOp1->getOpcode() == I->getOpcode() && + MainBOOp1->getParent() == I->getParent()) + continue; + } + } UsedOutside = PUsedOutside; for (Instruction *I : P.second) { if (IsSupportedInstruction(I, AnyUndef)) { diff --git a/llvm/test/Transforms/PhaseOrdering/X86/avg.ll b/llvm/test/Transforms/PhaseOrdering/X86/avg.ll index e28e294164ac5..0b0ade31da145 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/avg.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/avg.ll @@ -42,13 +42,13 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0, ; SSE2-NEXT: [[TMP18:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0 ; SSE2-NEXT: [[TMP19:%.*]] = insertelement <2 x i16> [[TMP18]], i16 [[TMP7]], i64 1 ; SSE2-NEXT: [[TMP20:%.*]] = lshr <2 x i16> [[TMP19]], splat (i16 8) +; SSE2-NEXT: [[A_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 56 ; SSE2-NEXT: [[B_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 56 ; SSE2-NEXT: [[TMP21:%.*]] = insertelement <2 x i16> poison, i16 [[TMP8]], i64 0 ; SSE2-NEXT: [[TMP22:%.*]] = insertelement <2 x i16> [[TMP21]], i16 [[TMP15]], i64 1 ; SSE2-NEXT: [[TMP23:%.*]] = lshr <2 x i16> [[TMP22]], splat (i16 8) -; SSE2-NEXT: [[A_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 56 -; SSE2-NEXT: [[CONV1_6:%.*]] = and i64 [[A_SROA_7_0_EXTRACT_SHIFT]], 255 ; SSE2-NEXT: [[B_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 56 +; SSE2-NEXT: [[CONV1_6:%.*]] = and i64 [[A_SROA_7_0_EXTRACT_SHIFT]], 255 ; SSE2-NEXT: [[CONV4_6:%.*]] = and i64 [[B_SROA_7_0_EXTRACT_SHIFT]], 255 ; SSE2-NEXT: [[TMP24:%.*]] = add nuw nsw <2 x i64> [[TMP16]], splat (i64 1) ; SSE2-NEXT: [[TMP25:%.*]] = add nuw nsw <2 x i64> [[TMP24]], [[TMP17]] @@ -144,13 +144,13 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0, ; SSE4-NEXT: [[TMP18:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0 ; SSE4-NEXT: [[TMP19:%.*]] = insertelement <2 x i16> [[TMP18]], i16 [[TMP7]], i64 1 ; SSE4-NEXT: [[TMP20:%.*]] = lshr <2 x i16> [[TMP19]], splat (i16 8) +; SSE4-NEXT: [[A_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 56 ; SSE4-NEXT: [[B_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 56 ; SSE4-NEXT: [[TMP21:%.*]] = insertelement <2 x i16> poison, i16 [[TMP8]], i64 0 ; SSE4-NEXT: [[TMP22:%.*]] = insertelement <2 x i16> [[TMP21]], i16 [[TMP15]], i64 1 ; SSE4-NEXT: [[TMP23:%.*]] = lshr <2 x i16> [[TMP22]], splat (i16 8) -; SSE4-NEXT: [[A_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 56 -; SSE4-NEXT: [[CONV1_6:%.*]] = and i64 [[A_SROA_7_0_EXTRACT_SHIFT]], 255 ; SSE4-NEXT: [[B_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 56 +; SSE4-NEXT: [[CONV1_6:%.*]] = and i64 [[A_SROA_7_0_EXTRACT_SHIFT]], 255 ; SSE4-NEXT: [[CONV4_6:%.*]] = and i64 [[B_SROA_7_0_EXTRACT_SHIFT]], 255 ; SSE4-NEXT: [[TMP24:%.*]] = add nuw nsw <2 x i64> [[TMP16]], splat (i64 1) ; SSE4-NEXT: [[TMP25:%.*]] = add nuw nsw <2 x i64> [[TMP24]], [[TMP17]] @@ -747,11 +747,11 @@ define { i64, i64 } @avgr_8_u16(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0, ; SSE2-NEXT: [[TMP19:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0 ; SSE2-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP2]], i64 1 ; SSE2-NEXT: [[TMP11:%.*]] = lshr <2 x i32> [[TMP10]], splat (i32 16) +; SSE2-NEXT: [[CONV_6:%.*]] = lshr i64 [[A_COERCE1]], 48 ; SSE2-NEXT: [[CONV2_4:%.*]] = lshr i64 [[B_COERCE0]], 48 ; SSE2-NEXT: [[TMP20:%.*]] = lshr <2 x i32> [[TMP4]], splat (i32 16) -; SSE2-NEXT: [[CONV_6:%.*]] = lshr i64 [[A_COERCE1]], 48 -; SSE2-NEXT: [[A_SROA_9_8_EXTRACT_SHIFT:%.*]] = and i64 [[A_SROA_3_0_EXTRACT_SHIFT]], 65535 ; SSE2-NEXT: [[B_SROA_9_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 48 +; SSE2-NEXT: [[A_SROA_9_8_EXTRACT_SHIFT:%.*]] = and i64 [[A_SROA_3_0_EXTRACT_SHIFT]], 65535 ; SSE2-NEXT: [[CONV2_2:%.*]] = and i64 [[B_SROA_3_0_EXTRACT_SHIFT]], 65535 ; SSE2-NEXT: [[TMP31:%.*]] = add nuw nsw <2 x i64> [[TMP7]], splat (i64 1) ; SSE2-NEXT: [[TMP14:%.*]] = add nuw nsw <2 x i64> [[TMP31]], [[TMP8]] @@ -810,11 +810,11 @@ define { i64, i64 } @avgr_8_u16(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0, ; SSE4-NEXT: [[TMP9:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0 ; SSE4-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP1]], i64 1 ; SSE4-NEXT: [[TMP11:%.*]] = lshr <2 x i32> [[TMP10]], splat (i32 16) +; SSE4-NEXT: [[A_SROA_9_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 48 ; SSE4-NEXT: [[B_SROA_4_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 48 ; SSE4-NEXT: [[TMP12:%.*]] = lshr <2 x i32> [[TMP4]], splat (i32 16) -; SSE4-NEXT: [[A_SROA_9_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 48 -; SSE4-NEXT: [[CONV_2:%.*]] = and i64 [[A_SROA_3_0_EXTRACT_SHIFT]], 65535 ; SSE4-NEXT: [[B_SROA_9_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 48 +; SSE4-NEXT: [[CONV_2:%.*]] = and i64 [[A_SROA_3_0_EXTRACT_SHIFT]], 65535 ; SSE4-NEXT: [[CONV2_2:%.*]] = and i64 [[B_SROA_3_0_EXTRACT_SHIFT]], 65535 ; SSE4-NEXT: [[TMP13:%.*]] = add nuw nsw <2 x i64> [[TMP7]], splat (i64 1) ; SSE4-NEXT: [[TMP14:%.*]] = add nuw nsw <2 x i64> [[TMP13]], [[TMP8]] @@ -856,28 +856,26 @@ define { i64, i64 } @avgr_8_u16(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0, ; ; AVX2-LABEL: @avgr_8_u16( ; AVX2-NEXT: entry: -; AVX2-NEXT: [[TMP0:%.*]] = trunc i64 [[A_COERCE0:%.*]] to i32 -; AVX2-NEXT: [[A_SROA_3_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 32 -; AVX2-NEXT: [[A_SROA_4_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 48 ; AVX2-NEXT: [[TMP1:%.*]] = trunc i64 [[A_COERCE1:%.*]] to i32 -; AVX2-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE0:%.*]], i64 0 -; AVX2-NEXT: [[TMP20:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[A_COERCE1]], i64 1 +; AVX2-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE1]], i64 0 +; AVX2-NEXT: [[TMP6:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[A_COERCE2:%.*]], i64 1 +; AVX2-NEXT: [[TMP5:%.*]] = lshr <2 x i64> [[TMP6]], +; AVX2-NEXT: [[A_SROA_4_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 48 +; AVX2-NEXT: [[TMP18:%.*]] = trunc i64 [[A_COERCE2]] to i32 +; AVX2-NEXT: [[TMP20:%.*]] = insertelement <2 x i64> [[TMP6]], i64 [[B_COERCE0:%.*]], i64 0 ; AVX2-NEXT: [[TMP21:%.*]] = lshr <2 x i64> [[TMP20]], ; AVX2-NEXT: [[TMP3:%.*]] = insertelement <2 x i64> [[TMP20]], i64 [[B_COERCE1:%.*]], i64 1 ; AVX2-NEXT: [[TMP4:%.*]] = trunc <2 x i64> [[TMP3]] to <2 x i32> -; AVX2-NEXT: [[B_SROA_3_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 32 +; AVX2-NEXT: [[TMP9:%.*]] = lshr <2 x i64> [[TMP3]], ; AVX2-NEXT: [[B_SROA_8_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 32 -; AVX2-NEXT: [[TMP6:%.*]] = insertelement <2 x i64> [[TMP20]], i64 [[A_COERCE0]], i64 0 ; AVX2-NEXT: [[TMP7:%.*]] = and <2 x i64> [[TMP6]], splat (i64 65535) ; AVX2-NEXT: [[TMP8:%.*]] = and <2 x i64> [[TMP3]], splat (i64 65535) -; AVX2-NEXT: [[TMP9:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0 -; AVX2-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP1]], i64 1 +; AVX2-NEXT: [[TMP19:%.*]] = insertelement <2 x i32> poison, i32 [[TMP1]], i64 0 +; AVX2-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP18]], i64 1 ; AVX2-NEXT: [[TMP11:%.*]] = lshr <2 x i32> [[TMP10]], splat (i32 16) ; AVX2-NEXT: [[TMP12:%.*]] = lshr <2 x i32> [[TMP4]], splat (i32 16) -; AVX2-NEXT: [[A_SROA_9_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 48 -; AVX2-NEXT: [[CONV_2:%.*]] = and i64 [[A_SROA_3_0_EXTRACT_SHIFT]], 65535 -; AVX2-NEXT: [[B_SROA_9_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 48 -; AVX2-NEXT: [[CONV2_2:%.*]] = and i64 [[B_SROA_3_0_EXTRACT_SHIFT]], 65535 +; AVX2-NEXT: [[TMP36:%.*]] = and <2 x i64> [[TMP5]], +; AVX2-NEXT: [[TMP39:%.*]] = and <2 x i64> [[TMP9]], ; AVX2-NEXT: [[TMP13:%.*]] = add nuw nsw <2 x i64> [[TMP7]], splat (i64 1) ; AVX2-NEXT: [[TMP14:%.*]] = add nuw nsw <2 x i64> [[TMP13]], [[TMP8]] ; AVX2-NEXT: [[TMP15:%.*]] = lshr <2 x i64> [[TMP14]], splat (i64 1) @@ -890,12 +888,8 @@ define { i64, i64 } @avgr_8_u16(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0, ; AVX2-NEXT: [[TMP35:%.*]] = insertelement <2 x i64> poison, i64 [[ADD_3]], i64 0 ; AVX2-NEXT: [[TMP25:%.*]] = insertelement <2 x i64> [[TMP35]], i64 [[CONV2_6]], i64 1 ; AVX2-NEXT: [[TMP38:%.*]] = add nuw nsw <2 x i64> [[TMP25]], [[TMP37]] -; AVX2-NEXT: [[ADD_7:%.*]] = add nuw nsw i64 [[A_SROA_9_8_EXTRACT_SHIFT]], 1 -; AVX2-NEXT: [[ADD_2:%.*]] = add nuw nsw i64 [[CONV_2]], 1 -; AVX2-NEXT: [[ADD3_7:%.*]] = add nuw nsw i64 [[ADD_7]], [[B_SROA_9_8_EXTRACT_SHIFT]] -; AVX2-NEXT: [[ADD3_2:%.*]] = add nuw nsw i64 [[ADD_2]], [[CONV2_2]] -; AVX2-NEXT: [[TMP39:%.*]] = insertelement <2 x i64> poison, i64 [[ADD3_2]], i64 0 -; AVX2-NEXT: [[TMP40:%.*]] = insertelement <2 x i64> [[TMP39]], i64 [[ADD3_7]], i64 1 +; AVX2-NEXT: [[TMP42:%.*]] = add nuw nsw <2 x i64> [[TMP36]], splat (i64 1) +; AVX2-NEXT: [[TMP40:%.*]] = add nuw nsw <2 x i64> [[TMP42]], [[TMP39]] ; AVX2-NEXT: [[TMP41:%.*]] = shl nuw <2 x i64> [[TMP40]], ; AVX2-NEXT: [[TMP31:%.*]] = and <2 x i64> [[TMP41]], ; AVX2-NEXT: [[TMP32:%.*]] = shl nuw <2 x i64> [[TMP38]], @@ -917,28 +911,23 @@ define { i64, i64 } @avgr_8_u16(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0, ; AVX512-NEXT: [[TMP0:%.*]] = trunc i64 [[A_COERCE0:%.*]] to i32 ; AVX512-NEXT: [[TMP1:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE0]], i64 0 ; AVX512-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[B_COERCE1:%.*]], i64 1 -; AVX512-NEXT: [[TMP3:%.*]] = lshr <2 x i64> [[TMP2]], -; AVX512-NEXT: [[TMP4:%.*]] = trunc i64 [[A_COERCE1:%.*]] to i32 -; AVX512-NEXT: [[TMP31:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE0:%.*]], i64 0 -; AVX512-NEXT: [[TMP32:%.*]] = insertelement <2 x i64> [[TMP31]], i64 [[A_COERCE1]], i64 1 +; AVX512-NEXT: [[TMP7:%.*]] = lshr <2 x i64> [[TMP2]], +; AVX512-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[B_COERCE2:%.*]], i64 1 +; AVX512-NEXT: [[TMP3:%.*]] = lshr <2 x i64> [[TMP10]], +; AVX512-NEXT: [[TMP4:%.*]] = trunc i64 [[B_COERCE1]] to i32 +; AVX512-NEXT: [[TMP32:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[B_COERCE0:%.*]], i64 0 ; AVX512-NEXT: [[TMP49:%.*]] = lshr <2 x i64> [[TMP32]], -; AVX512-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[B_COERCE0]], i64 0 +; AVX512-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[B_COERCE0]], i64 0 ; AVX512-NEXT: [[TMP6:%.*]] = trunc <2 x i64> [[TMP5]] to <2 x i32> -; AVX512-NEXT: [[TMP7:%.*]] = insertelement <2 x i64> [[TMP32]], i64 [[A_COERCE0]], i64 0 -; AVX512-NEXT: [[TMP8:%.*]] = and <2 x i64> [[TMP7]], splat (i64 65535) +; AVX512-NEXT: [[TMP14:%.*]] = lshr <2 x i64> [[TMP5]], +; AVX512-NEXT: [[TMP8:%.*]] = and <2 x i64> [[TMP2]], splat (i64 65535) ; AVX512-NEXT: [[TMP9:%.*]] = and <2 x i64> [[TMP5]], splat (i64 65535) -; AVX512-NEXT: [[TMP10:%.*]] = lshr <2 x i64> [[TMP7]], ; AVX512-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0 ; AVX512-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP11]], i32 [[TMP4]], i64 1 ; AVX512-NEXT: [[TMP13:%.*]] = lshr <2 x i32> [[TMP12]], splat (i32 16) -; AVX512-NEXT: [[TMP14:%.*]] = lshr <2 x i64> [[TMP5]], ; AVX512-NEXT: [[TMP15:%.*]] = lshr <2 x i32> [[TMP6]], splat (i32 16) -; AVX512-NEXT: [[TMP16:%.*]] = and <2 x i64> [[TMP10]], -; AVX512-NEXT: [[TMP17:%.*]] = lshr <2 x i64> [[TMP10]], -; AVX512-NEXT: [[TMP18:%.*]] = shufflevector <2 x i64> [[TMP16]], <2 x i64> [[TMP17]], <2 x i32> -; AVX512-NEXT: [[TMP19:%.*]] = and <2 x i64> [[TMP14]], -; AVX512-NEXT: [[TMP20:%.*]] = lshr <2 x i64> [[TMP14]], -; AVX512-NEXT: [[TMP21:%.*]] = shufflevector <2 x i64> [[TMP19]], <2 x i64> [[TMP20]], <2 x i32> +; AVX512-NEXT: [[TMP18:%.*]] = and <2 x i64> [[TMP7]], +; AVX512-NEXT: [[TMP21:%.*]] = and <2 x i64> [[TMP14]], ; AVX512-NEXT: [[TMP22:%.*]] = add nuw nsw <2 x i64> [[TMP8]], splat (i64 1) ; AVX512-NEXT: [[TMP23:%.*]] = add nuw nsw <2 x i64> [[TMP22]], [[TMP9]] ; AVX512-NEXT: [[TMP24:%.*]] = lshr <2 x i64> [[TMP23]], splat (i64 1) diff --git a/llvm/test/Transforms/SLPVectorizer/X86/complex-fma-combine.ll b/llvm/test/Transforms/SLPVectorizer/X86/complex-fma-combine.ll index 5c40ea7d79e89..9cc139e8a02a5 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/complex-fma-combine.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/complex-fma-combine.ll @@ -6,13 +6,9 @@ define <2 x float> @fma_f32(ptr %A, float %B) { ; CHECK-SAME: ptr [[A:%.*]], float [[B:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A]], align 4 -; CHECK-NEXT: [[RETVAL_SROA_0_0_VEC_EXTRACT_I:%.*]] = extractelement <2 x float> [[TMP0]], i64 0 -; CHECK-NEXT: [[RETVAL_SROA_0_4_VEC_EXTRACT_I:%.*]] = extractelement <2 x float> [[TMP0]], i64 1 -; CHECK-NEXT: [[MUL_RL_I_I:%.*]] = fmul float [[RETVAL_SROA_0_0_VEC_EXTRACT_I]], 2.000000e+00 -; CHECK-NEXT: [[MUL_IL_I_I:%.*]] = fmul float [[RETVAL_SROA_0_4_VEC_EXTRACT_I]], 2.000000e+00 -; CHECK-NEXT: [[ADD_R_I_I:%.*]] = fadd float [[B]], [[MUL_RL_I_I]] -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[ADD_R_I_I]], i64 0 -; CHECK-NEXT: [[RETVAL_SROA_0_0_VEC_INSERT_I6:%.*]] = insertelement <2 x float> [[TMP1]], float [[MUL_IL_I_I]], i64 1 +; CHECK-NEXT: [[TMP1:%.*]] = fmul <2 x float> [[TMP0]], splat (float 2.000000e+00) +; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> , float [[B]], i32 0 +; CHECK-NEXT: [[RETVAL_SROA_0_0_VEC_INSERT_I6:%.*]] = fadd <2 x float> [[TMP2]], [[TMP1]] ; CHECK-NEXT: ret <2 x float> [[RETVAL_SROA_0_0_VEC_INSERT_I6]] ; entry: diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gathered-loads-non-full-reg.ll b/llvm/test/Transforms/SLPVectorizer/X86/gathered-loads-non-full-reg.ll index 74a4fb3ff2b00..6416d5a6062c7 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/gathered-loads-non-full-reg.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/gathered-loads-non-full-reg.ll @@ -11,7 +11,9 @@ define void @test(ptr noalias %0) { ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP0]], i64 32 ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP0]], i64 128 ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[TMP0]], i64 200 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP0]], i64 208 ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[TMP0]], i64 232 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP0]], i64 288 ; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP0]], i64 320 ; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP0]], i64 304 ; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP0]], i64 424 @@ -23,47 +25,46 @@ define void @test(ptr noalias %0) { ; CHECK-NEXT: [[TMP28:%.*]] = load double, ptr [[TMP2]], align 8 ; CHECK-NEXT: [[TMP29:%.*]] = fadd double [[TMP28]], [[TMP27]] ; CHECK-NEXT: [[DOTNEG969:%.*]] = load double, ptr [[TMP3]], align 8 +; CHECK-NEXT: [[TMP19:%.*]] = load double, ptr [[TMP4]], align 8 +; CHECK-NEXT: [[TMP24:%.*]] = load double, ptr [[TMP5]], align 8 +; CHECK-NEXT: [[TMP26:%.*]] = load double, ptr [[TMP6]], align 8 +; CHECK-NEXT: [[TMP22:%.*]] = fadd double [[TMP26]], [[TMP24]] +; CHECK-NEXT: [[TMP23:%.*]] = load double, ptr [[TMP7]], align 8 ; CHECK-NEXT: [[TMP17:%.*]] = load double, ptr [[TMP8]], align 8 ; CHECK-NEXT: [[TMP25:%.*]] = load double, ptr [[TMP9]], align 8 ; CHECK-NEXT: [[TMP31:%.*]] = load double, ptr [[TMP10]], align 8 ; CHECK-NEXT: [[TMP21:%.*]] = load double, ptr [[TMP11]], align 8 ; CHECK-NEXT: [[TMP20:%.*]] = load double, ptr [[TMP12]], align 8 ; CHECK-NEXT: [[TMP36:%.*]] = fadd double [[TMP20]], [[TMP21]] -; CHECK-NEXT: [[TMP32:%.*]] = fsub double 0.000000e+00, [[TMP25]] -; CHECK-NEXT: [[TMP33:%.*]] = fmul double [[TMP32]], 0.000000e+00 -; CHECK-NEXT: [[TMP34:%.*]] = fadd double [[TMP33]], 0.000000e+00 +; CHECK-NEXT: [[TMP34:%.*]] = fmul double [[TMP22]], [[DOTNEG969]] ; CHECK-NEXT: [[TMP35:%.*]] = fmul double [[TMP34]], 0.000000e+00 -; CHECK-NEXT: [[TMP37:%.*]] = fmul double [[TMP36]], [[TMP31]] +; CHECK-NEXT: [[TMP32:%.*]] = fsub double [[TMP29]], [[TMP19]] +; CHECK-NEXT: [[TMP37:%.*]] = fmul double [[TMP32]], [[TMP23]] ; CHECK-NEXT: [[TMP38:%.*]] = fmul double [[TMP37]], 0.000000e+00 -; CHECK-NEXT: [[TMP42:%.*]] = fadd double [[TMP38]], 0.000000e+00 ; CHECK-NEXT: [[TMP30:%.*]] = load double, ptr [[TMP0]], align 8 ; CHECK-NEXT: [[TMP40:%.*]] = load double, ptr [[TMP13]], align 8 +; CHECK-NEXT: [[TMP47:%.*]] = fmul double [[TMP40]], [[TMP35]] ; CHECK-NEXT: [[TMP60:%.*]] = load double, ptr [[TMP14]], align 8 -; CHECK-NEXT: [[TMP41:%.*]] = fmul double [[TMP35]], 0.000000e+00 -; CHECK-NEXT: [[TMP43:%.*]] = fmul double [[TMP60]], [[TMP42]] -; CHECK-NEXT: store double [[TMP41]], ptr getelementptr inbounds (i8, ptr @solid_, i64 384), align 8 -; CHECK-NEXT: store double [[TMP43]], ptr getelementptr inbounds (i8, ptr @solid_, i64 392), align 8 -; CHECK-NEXT: [[TMP44:%.*]] = load <2 x double>, ptr [[TMP4]], align 8 -; CHECK-NEXT: [[TMP46:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr align 8 [[TMP6]], <8 x i1> , <8 x double> poison) -; CHECK-NEXT: [[TMP50:%.*]] = shufflevector <8 x double> [[TMP46]], <8 x double> poison, <2 x i32> -; CHECK-NEXT: [[TMP39:%.*]] = insertelement <2 x double> [[TMP50]], double [[TMP29]], i32 0 -; CHECK-NEXT: [[TMP55:%.*]] = fsub <2 x double> [[TMP39]], [[TMP44]] -; CHECK-NEXT: [[TMP57:%.*]] = fadd <2 x double> [[TMP39]], [[TMP44]] -; CHECK-NEXT: [[TMP62:%.*]] = shufflevector <2 x double> [[TMP55]], <2 x double> [[TMP57]], <2 x i32> -; CHECK-NEXT: [[TMP51:%.*]] = insertelement <2 x double> [[TMP50]], double [[DOTNEG969]], i32 1 -; CHECK-NEXT: [[TMP52:%.*]] = fmul <2 x double> [[TMP62]], [[TMP51]] -; CHECK-NEXT: [[TMP45:%.*]] = fmul <2 x double> [[TMP52]], zeroinitializer -; CHECK-NEXT: [[TMP63:%.*]] = insertelement <2 x double> poison, double [[TMP60]], i32 0 -; CHECK-NEXT: [[TMP56:%.*]] = insertelement <2 x double> [[TMP63]], double [[TMP40]], i32 1 +; CHECK-NEXT: [[TMP39:%.*]] = fadd double [[TMP47]], 0.000000e+00 +; CHECK-NEXT: [[TMP61:%.*]] = fmul double [[TMP60]], [[TMP38]] +; CHECK-NEXT: store double [[TMP39]], ptr getelementptr inbounds (i8, ptr @solid_, i64 408), align 8 +; CHECK-NEXT: [[TMP41:%.*]] = fmul double [[TMP36]], [[TMP31]] +; CHECK-NEXT: [[TMP42:%.*]] = fsub double 0.000000e+00, [[TMP25]] +; CHECK-NEXT: [[TMP49:%.*]] = insertelement <2 x double> poison, double [[TMP42]], i32 0 +; CHECK-NEXT: [[TMP44:%.*]] = insertelement <2 x double> [[TMP49]], double [[TMP41]], i32 1 +; CHECK-NEXT: [[TMP50:%.*]] = fmul <2 x double> [[TMP44]], zeroinitializer +; CHECK-NEXT: [[TMP46:%.*]] = fadd <2 x double> [[TMP50]], zeroinitializer +; CHECK-NEXT: [[TMP45:%.*]] = fmul <2 x double> [[TMP46]], +; CHECK-NEXT: [[TMP56:%.*]] = insertelement <2 x double> , double [[TMP60]], i32 1 ; CHECK-NEXT: [[TMP48:%.*]] = fmul <2 x double> [[TMP56]], [[TMP45]] -; CHECK-NEXT: [[TMP49:%.*]] = fadd <2 x double> [[TMP48]], -; CHECK-NEXT: store <2 x double> [[TMP49]], ptr getelementptr inbounds (i8, ptr @solid_, i64 400), align 8 +; CHECK-NEXT: store <2 x double> [[TMP48]], ptr getelementptr inbounds (i8, ptr @solid_, i64 384), align 8 +; CHECK-NEXT: store double [[TMP61]], ptr getelementptr inbounds (i8, ptr @solid_, i64 400), align 8 +; CHECK-NEXT: [[TMP43:%.*]] = extractelement <2 x double> [[TMP48]], i32 1 ; CHECK-NEXT: [[DOTNEG965:%.*]] = fmul double [[TMP43]], [[TMP17]] ; CHECK-NEXT: [[REASS_ADD993:%.*]] = fadd double [[DOTNEG965]], 0.000000e+00 ; CHECK-NEXT: [[TMP58:%.*]] = fadd double [[TMP30]], [[REASS_ADD993]] ; CHECK-NEXT: [[TMP59:%.*]] = fsub double 0.000000e+00, [[TMP58]] ; CHECK-NEXT: store double [[TMP59]], ptr getelementptr inbounds (i8, ptr @solid_, i64 296), align 8 -; CHECK-NEXT: [[TMP61:%.*]] = extractelement <2 x double> [[TMP48]], i32 0 ; CHECK-NEXT: [[DOTNEG970:%.*]] = fmul double [[TMP61]], 0.000000e+00 ; CHECK-NEXT: [[REASS_ADD996:%.*]] = fadd double [[DOTNEG970]], 0.000000e+00 ; CHECK-NEXT: [[TMP53:%.*]] = fadd double [[TMP60]], [[REASS_ADD996]] diff --git a/llvm/test/Transforms/SLPVectorizer/buildvector-nodes-dependency.ll b/llvm/test/Transforms/SLPVectorizer/buildvector-nodes-dependency.ll index 6a44433ff5887..a2876b9c06bc5 100644 --- a/llvm/test/Transforms/SLPVectorizer/buildvector-nodes-dependency.ll +++ b/llvm/test/Transforms/SLPVectorizer/buildvector-nodes-dependency.ll @@ -10,16 +10,14 @@ define double @test() { ; X86: cond.true: ; X86-NEXT: [[TMP1:%.*]] = insertelement <2 x double> , double [[TMP0]], i32 1 ; X86-NEXT: [[TMP2:%.*]] = fmul <2 x double> zeroinitializer, [[TMP1]] +; X86-NEXT: [[TMP3:%.*]] = fmul <2 x double> zeroinitializer, [[TMP1]] ; X86-NEXT: [[TMP15:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> ; X86-NEXT: [[TMP16:%.*]] = fmul <2 x double> [[TMP15]], zeroinitializer ; X86-NEXT: [[TMP17:%.*]] = fmul <2 x double> [[TMP15]], zeroinitializer -; X86-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> [[TMP1]], <2 x i32> ; X86-NEXT: [[TMP4:%.*]] = fmul <2 x double> [[TMP3]], zeroinitializer ; X86-NEXT: [[TMP5:%.*]] = fsub <2 x double> [[TMP4]], zeroinitializer -; X86-NEXT: [[TMP6:%.*]] = fmul <2 x double> [[TMP4]], zeroinitializer -; X86-NEXT: [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <2 x i32> -; X86-NEXT: [[TMP8:%.*]] = fadd <2 x double> zeroinitializer, [[TMP7]] -; X86-NEXT: [[TMP9:%.*]] = fmul <2 x double> zeroinitializer, [[TMP7]] +; X86-NEXT: [[TMP8:%.*]] = fadd <2 x double> zeroinitializer, [[TMP5]] +; X86-NEXT: [[TMP9:%.*]] = fmul <2 x double> zeroinitializer, [[TMP5]] ; X86-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> [[TMP9]], <2 x i32> ; X86-NEXT: [[TMP11:%.*]] = fsub <2 x double> [[TMP10]], [[TMP2]] ; X86-NEXT: [[TMP12:%.*]] = fadd <2 x double> [[TMP10]], [[TMP2]] @@ -44,16 +42,14 @@ define double @test() { ; AARCH64: cond.true: ; AARCH64-NEXT: [[TMP1:%.*]] = insertelement <2 x double> , double [[TMP0]], i32 1 ; AARCH64-NEXT: [[TMP2:%.*]] = fmul <2 x double> zeroinitializer, [[TMP1]] +; AARCH64-NEXT: [[TMP6:%.*]] = fmul <2 x double> zeroinitializer, [[TMP1]] ; AARCH64-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> ; AARCH64-NEXT: [[TMP4:%.*]] = fmul <2 x double> [[TMP3]], zeroinitializer ; AARCH64-NEXT: [[TMP5:%.*]] = fmul <2 x double> [[TMP3]], zeroinitializer -; AARCH64-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> [[TMP1]], <2 x i32> ; AARCH64-NEXT: [[TMP7:%.*]] = fmul <2 x double> [[TMP6]], zeroinitializer ; AARCH64-NEXT: [[TMP8:%.*]] = fsub <2 x double> [[TMP7]], zeroinitializer -; AARCH64-NEXT: [[TMP9:%.*]] = fmul <2 x double> [[TMP7]], zeroinitializer -; AARCH64-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> [[TMP9]], <2 x i32> -; AARCH64-NEXT: [[TMP11:%.*]] = fadd <2 x double> zeroinitializer, [[TMP10]] -; AARCH64-NEXT: [[TMP12:%.*]] = fmul <2 x double> zeroinitializer, [[TMP10]] +; AARCH64-NEXT: [[TMP11:%.*]] = fadd <2 x double> zeroinitializer, [[TMP8]] +; AARCH64-NEXT: [[TMP12:%.*]] = fmul <2 x double> zeroinitializer, [[TMP8]] ; AARCH64-NEXT: [[TMP13:%.*]] = shufflevector <2 x double> [[TMP11]], <2 x double> [[TMP12]], <2 x i32> ; AARCH64-NEXT: [[TMP14:%.*]] = fsub <2 x double> [[TMP13]], [[TMP2]] ; AARCH64-NEXT: [[TMP15:%.*]] = fadd <2 x double> [[TMP13]], [[TMP2]]