diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h index b2f664a9c9c0d..684c22fdab2e4 100644 --- a/llvm/include/llvm/Analysis/ValueTracking.h +++ b/llvm/include/llvm/Analysis/ValueTracking.h @@ -639,8 +639,7 @@ inline bool isValidAssumeForContext(const Instruction *I, } /// Returns true, if no instruction between \p Assume and \p CtxI may free -/// memory and the function is marked as NoSync. The latter ensures the current -/// function cannot arrange for another thread to free on its behalf. +/// (including through synchronization). LLVM_ABI bool willNotFreeBetween(const Instruction *Assume, const Instruction *CtxI); diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 7b1550a10f134..8ff5b750203cc 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -704,22 +704,20 @@ bool llvm::isValidAssumeForContext(const Instruction *Inv, bool llvm::willNotFreeBetween(const Instruction *Assume, const Instruction *CtxI) { // Helper to check if there are any calls in the range that may free memory. - auto hasNoFreeCalls = [](auto Range) { + auto hasNoFreeInRange = [](auto Range) { for (const auto &[Idx, I] : enumerate(Range)) { if (Idx > MaxInstrsToCheckForFree) return false; - if (const auto *CB = dyn_cast(&I)) + + if (auto *CB = dyn_cast(&I)) { if (!CB->hasFnAttr(Attribute::NoFree)) return false; + } else if (I.maySynchronize()) + return false; } return true; }; - // Make sure the current function cannot arrange for another thread to free on - // its behalf. - if (!CtxI->getFunction()->hasNoSync()) - return false; - // Handle cross-block case: CtxI in a successor of Assume's block. const BasicBlock *CtxBB = CtxI->getParent(); const BasicBlock *AssumeBB = Assume->getParent(); @@ -728,7 +726,7 @@ bool llvm::willNotFreeBetween(const Instruction *Assume, if (CtxBB->getSinglePredecessor() != AssumeBB) return false; - if (!hasNoFreeCalls(make_range(CtxBB->begin(), CtxIter))) + if (!hasNoFreeInRange(make_range(CtxBB->begin(), CtxIter))) return false; CtxIter = AssumeBB->end(); @@ -740,7 +738,7 @@ bool llvm::willNotFreeBetween(const Instruction *Assume, // Check if there are any calls between Assume and CtxIter that may free // memory. - return hasNoFreeCalls(make_range(Assume->getIterator(), CtxIter)); + return hasNoFreeInRange(make_range(Assume->getIterator(), CtxIter)); } // TODO: cmpExcludesZero misses many cases where `RHS` is non-constant but diff --git a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll index fe758c3ed21e3..894960a74dd34 100644 --- a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll +++ b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll @@ -1307,30 +1307,13 @@ define void @deref_assumption_in_header_constant_trip_count_nofree_via_context_b ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ] +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]] ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4 -; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer -; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0 -; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]] -; CHECK: [[PRED_LOAD_IF]]: -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]] -; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i64 0 -; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]] -; CHECK: [[PRED_LOAD_CONTINUE]]: -; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_LOAD_IF]] ] -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1 -; CHECK-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]] -; CHECK: [[PRED_LOAD_IF1]]: -; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1 -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP9]] -; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 -; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP11]], i64 1 -; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]] -; CHECK: [[PRED_LOAD_CONTINUE2]]: -; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i32> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ] -; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[TMP13]], <2 x i32> [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP2:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4 +; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[WIDE_LOAD1]] ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] ; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP14]], align 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 @@ -1456,4 +1439,150 @@ exit: ret void } +define void @atomicVectorizingConditionallyTest(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) { +; CHECK-LABEL: define void @atomicVectorizingConditionallyTest( +; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4000) ] +; CHECK-NEXT: [[ATOMIC:%.*]] = load atomic ptr, ptr [[A]] seq_cst, align 8 +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0 +; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]] +; CHECK: [[PRED_LOAD_IF]]: +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]] +; CHECK: [[PRED_LOAD_CONTINUE]]: +; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1 +; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]] +; CHECK: [[PRED_LOAD_IF1]]: +; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP10]], i64 1 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]] +; CHECK: [[PRED_LOAD_CONTINUE2]]: +; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ [[TMP6]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], %[[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[TMP12]], <2 x i32> [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] +; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP13]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 +; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 4000) ] + %atomic = load atomic ptr, ptr %a seq_cst, align 8 ; Atomic Instruction + br label %loop.header + +loop.header: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + %gep.a = getelementptr i32, ptr %a, i64 %iv + %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv + %l.b = load i32, ptr %gep.b, align 4 + %c.1 = icmp sge i32 %l.b, 0 + br i1 %c.1, label %loop.latch, label %loop.then + +loop.then: + %l.a = load i32, ptr %gep.a, align 4 + br label %loop.latch + +loop.latch: + %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop.header ] + %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv + store i32 %merge, ptr %gep.c, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 1000 + br i1 %ec, label %exit, label %loop.header + +exit: + ret void +} + +define void @FenceVectorizingConditionallyTest(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) { +; CHECK-LABEL: define void @FenceVectorizingConditionallyTest( +; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4000) ] +; CHECK-NEXT: fence seq_cst +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0 +; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]] +; CHECK: [[PRED_LOAD_IF]]: +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]] +; CHECK: [[PRED_LOAD_CONTINUE]]: +; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP1]], i64 1 +; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]] +; CHECK: [[PRED_LOAD_IF1]]: +; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[TMP10]], i64 1 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]] +; CHECK: [[PRED_LOAD_CONTINUE2]]: +; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x i32> [ [[TMP6]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], %[[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[TMP12]], <2 x i32> [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] +; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP13]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 +; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 4000) ] + fence seq_cst ; Fence Instruction + br label %loop.header + +loop.header: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + %gep.a = getelementptr i32, ptr %a, i64 %iv + %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv + %l.b = load i32, ptr %gep.b, align 4 + %c.1 = icmp sge i32 %l.b, 0 + br i1 %c.1, label %loop.latch, label %loop.then + +loop.then: + %l.a = load i32, ptr %gep.a, align 4 + br label %loop.latch + +loop.latch: + %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop.header ] + %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv + store i32 %merge, ptr %gep.c, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 1000 + br i1 %ec, label %exit, label %loop.header + +exit: + ret void +} + diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll index 13d17b28709a3..9d6be3c9cc5e5 100644 --- a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll +++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll @@ -1090,3 +1090,237 @@ found: exit: ret i32 0 } + +define i64 @VectorizingWithoutNoSyncAttributeTest(ptr noundef nonnull readonly align 8 captures(none) dereferenceable(24) %v, i32 noundef %n) { +; CHECK-LABEL: define i64 @VectorizingWithoutNoSyncAttributeTest( +; CHECK-SAME: ptr noundef nonnull readonly align 8 captures(none) dereferenceable(24) [[V:%.*]], i32 noundef [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[L_V:%.*]] = load ptr, ptr [[V]], align 8 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_V]], i64 4) ] +; CHECK-NEXT: [[PI_L_V:%.*]] = ptrtoint ptr [[L_V]] to i64 +; CHECK-NEXT: [[GEP_V:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 8 +; CHECK-NEXT: [[L_GEP_V:%.*]] = load ptr, ptr [[GEP_V]], align 8 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_GEP_V]], i64 4) ] +; CHECK-NEXT: [[PI_L_GEP_V:%.*]] = ptrtoint ptr [[L_GEP_V]] to i64 +; CHECK-NEXT: [[SUB:%.*]] = sub i64 [[PI_L_GEP_V]], [[PI_L_V]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[L_V]], i64 [[SUB]]) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_V]], i64 4) ] +; CHECK-NEXT: [[C_1:%.*]] = icmp eq ptr [[L_V]], [[L_GEP_V]] +; CHECK-NEXT: br i1 [[C_1]], label %[[EXIT:.*]], label %[[LOOP_PREHEADER:.*]] +; CHECK: [[LOOP_PREHEADER]]: +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[PI_L_GEP_V]], -4 +; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[PI_L_V]] +; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 2 +; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[L_V]], i64 [[TMP4]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[INDEX]], 2 +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[L_V]], i64 [[TMP6]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[NEXT_GEP]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP7]] +; CHECK-NEXT: [[C_2:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[C_2]], label %[[LOOP_EARLY_EXIT:.*]], label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: br i1 [[TMP10]], label %[[LOOP_COMP:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP14:![0-9]+]] +; CHECK: [[LOOP_COMP]]: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], label %[[LOOP_COMP1:.*]], label %[[SCALAR_PH]] +; CHECK: [[LOOP_EARLY_EXIT]]: +; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP7]], i1 false) +; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP13:%.*]] = shl i64 [[TMP12]], 2 +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[L_V]], i64 [[TMP13]] +; CHECK-NEXT: br label %[[LOOP_EARLY_EXIT1:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[LOOP_COMP]] ], [ [[L_V]], %[[LOOP_PREHEADER]] ] +; CHECK-NEXT: br label %[[LOOP_HEADER1:.*]] +; CHECK: [[LOOP_HEADER1]]: +; CHECK-NEXT: [[MERGE:%.*]] = phi ptr [ [[GEP_MERGE:%.*]], %[[LOOP_LATCH1:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ] +; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[MERGE]], align 4 +; CHECK-NEXT: [[C_4:%.*]] = icmp slt i32 [[VAL]], [[N]] +; CHECK-NEXT: br i1 [[C_4]], label %[[LOOP_EARLY_EXIT1]], label %[[LOOP_LATCH1]] +; CHECK: [[LOOP_EARLY_EXIT1]]: +; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[MERGE]], %[[LOOP_HEADER1]] ], [ [[TMP14]], %[[LOOP_EARLY_EXIT]] ] +; CHECK-NEXT: [[PI_IV:%.*]] = ptrtoint ptr [[IV]] to i64 +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[LOOP_LATCH1]]: +; CHECK-NEXT: [[GEP_MERGE]] = getelementptr inbounds nuw i8, ptr [[MERGE]], i64 4 +; CHECK-NEXT: [[C_3:%.*]] = icmp eq ptr [[GEP_MERGE]], [[L_GEP_V]] +; CHECK-NEXT: br i1 [[C_3]], label %[[LOOP_COMP1]], label %[[LOOP_HEADER1]], !llvm.loop [[LOOP15:![0-9]+]] +; CHECK: [[LOOP_COMP1]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[MER:%.*]] = phi i64 [ [[PI_L_GEP_V]], %[[ENTRY]] ], [ [[PI_IV]], %[[LOOP_EARLY_EXIT1]] ], [ [[PI_L_GEP_V]], %[[LOOP_COMP1]] ] +; CHECK-NEXT: ret i64 [[MER]] +; +entry: + %l.v = load ptr, ptr %v, align 8 + call void @llvm.assume(i1 true) [ "align"(ptr %l.v, i64 4) ] + %pi.l.v = ptrtoint ptr %l.v to i64 + %gep.v = getelementptr inbounds nuw i8, ptr %v, i64 8 + %l.gep.v = load ptr, ptr %gep.v, align 8 + call void @llvm.assume(i1 true) [ "align"(ptr %l.gep.v, i64 4) ] + %pi.l.gep.v = ptrtoint ptr %l.gep.v to i64 + %sub = sub i64 %pi.l.gep.v, %pi.l.v + call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %l.v, i64 %sub) ] + call void @llvm.assume(i1 true) [ "align"(ptr %l.v, i64 4) ] + %c.1 = icmp eq ptr %l.v, %l.gep.v + br i1 %c.1, label %exit, label %loop.preheader + +loop.preheader: + br label %loop.header + +loop.header: + %iv = phi ptr [ %gep.merge, %loop.latch ], [ %l.v, %loop.preheader ] + %val = load i32, ptr %iv, align 4 + %c.2 = icmp slt i32 %val, %n + br i1 %c.2, label %loop.early.exit, label %loop.latch + +loop.early.exit: + %iv.1 = phi ptr [ %iv, %loop.header ] + %pi.iv = ptrtoint ptr %iv.1 to i64 + br label %exit + +loop.latch: + %gep.merge = getelementptr inbounds nuw i8, ptr %iv, i64 4 + %c.3 = icmp eq ptr %gep.merge, %l.gep.v + br i1 %c.3, label %loop.comp, label %loop.header + +loop.comp: + br label %exit + +exit: + %mer = phi i64 [ %pi.l.gep.v, %entry ], [ %pi.iv, %loop.early.exit ], [ %pi.l.gep.v, %loop.comp ] + ret i64 %mer +} + +define i64 @volatileNotVectorizingTest(ptr noundef nonnull readonly align 8 captures(none) dereferenceable(24) %v, i32 noundef %n) { +; CHECK-LABEL: define i64 @volatileNotVectorizingTest( +; CHECK-SAME: ptr noundef nonnull readonly align 8 captures(none) dereferenceable(24) [[V:%.*]], i32 noundef [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[L_V:%.*]] = load ptr, ptr [[V]], align 8 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_V]], i64 4) ] +; CHECK-NEXT: [[PI_L_V:%.*]] = ptrtoint ptr [[L_V]] to i64 +; CHECK-NEXT: [[GEP_V:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 8 +; CHECK-NEXT: [[L_GEP_V:%.*]] = load ptr, ptr [[GEP_V]], align 8 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_GEP_V]], i64 4) ] +; CHECK-NEXT: [[PI_L_GEP_V:%.*]] = ptrtoint ptr [[L_GEP_V]] to i64 +; CHECK-NEXT: [[SUB:%.*]] = sub i64 [[PI_L_GEP_V]], [[PI_L_V]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[L_V]], i64 [[SUB]]) ] +; CHECK-NEXT: [[VOLATILE:%.*]] = load volatile ptr, ptr [[V]], align 8 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_V]], i64 4) ] +; CHECK-NEXT: [[C_1:%.*]] = icmp eq ptr [[L_V]], [[L_GEP_V]] +; CHECK-NEXT: br i1 [[C_1]], label %[[EXIT:.*]], label %[[LOOP_PREHEADER:.*]] +; CHECK: [[LOOP_PREHEADER]]: +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[PI_L_GEP_V]], -4 +; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[PI_L_V]] +; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 2 +; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[L_V]], i64 [[TMP4]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ] +; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[INDEX]], 2 +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[L_V]], i64 [[TMP6]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[NEXT_GEP]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = icmp slt <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]] +; CHECK: [[VECTOR_BODY_INTERIM]]: +; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP16:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], label %[[LOOP_COMP:.*]], label %[[SCALAR_PH]] +; CHECK: [[VECTOR_EARLY_EXIT]]: +; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP7]], i1 false) +; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP13:%.*]] = shl i64 [[TMP12]], 2 +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[L_V]], i64 [[TMP13]] +; CHECK-NEXT: br label %[[LOOP_EARLY_EXIT:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[L_V]], %[[LOOP_PREHEADER]] ] +; CHECK-NEXT: br label %[[LOOP_HEADER1:.*]] +; CHECK: [[LOOP_HEADER1]]: +; CHECK-NEXT: [[MERGE:%.*]] = phi ptr [ [[GEP_MERGE:%.*]], %[[LOOP_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ] +; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[MERGE]], align 4 +; CHECK-NEXT: [[C_2:%.*]] = icmp slt i32 [[VAL]], [[N]] +; CHECK-NEXT: br i1 [[C_2]], label %[[LOOP_EARLY_EXIT]], label %[[LOOP_LATCH]] +; CHECK: [[LOOP_EARLY_EXIT]]: +; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[MERGE]], %[[LOOP_HEADER1]] ], [ [[TMP14]], %[[VECTOR_EARLY_EXIT]] ] +; CHECK-NEXT: [[PI_IV:%.*]] = ptrtoint ptr [[IV]] to i64 +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[GEP_MERGE]] = getelementptr inbounds nuw i8, ptr [[MERGE]], i64 4 +; CHECK-NEXT: [[C_3:%.*]] = icmp eq ptr [[GEP_MERGE]], [[L_GEP_V]] +; CHECK-NEXT: br i1 [[C_3]], label %[[LOOP_COMP]], label %[[LOOP_HEADER1]], !llvm.loop [[LOOP17:![0-9]+]] +; CHECK: [[LOOP_COMP]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[MER:%.*]] = phi i64 [ [[PI_L_GEP_V]], %[[ENTRY]] ], [ [[PI_IV]], %[[LOOP_EARLY_EXIT]] ], [ [[PI_L_GEP_V]], %[[LOOP_COMP]] ] +; CHECK-NEXT: ret i64 [[MER]] +; +entry: + %l.v = load ptr, ptr %v, align 8 + call void @llvm.assume(i1 true) [ "align"(ptr %l.v, i64 4) ] + %pi.l.v = ptrtoint ptr %l.v to i64 + %gep.v = getelementptr inbounds nuw i8, ptr %v, i64 8 + %l.gep.v = load ptr, ptr %gep.v, align 8 + call void @llvm.assume(i1 true) [ "align"(ptr %l.gep.v, i64 4) ] + %pi.l.gep.v = ptrtoint ptr %l.gep.v to i64 + %sub = sub i64 %pi.l.gep.v, %pi.l.v + call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %l.v, i64 %sub) ] + %volatile = load volatile ptr, ptr %v, align 8 ; Volatile Instruction + call void @llvm.assume(i1 true) [ "align"(ptr %l.v, i64 4) ] + %c.1 = icmp eq ptr %l.v, %l.gep.v + br i1 %c.1, label %exit, label %loop.preheader + +loop.preheader: + br label %loop.header + +loop.header: + %iv = phi ptr [ %gep.merge, %loop.latch ], [ %l.v, %loop.preheader ] + %val = load i32, ptr %iv, align 4 + %c.2 = icmp slt i32 %val, %n + br i1 %c.2, label %loop.early.exit, label %loop.latch + +loop.early.exit: + %iv.1 = phi ptr [ %iv, %loop.header ] + %pi.iv = ptrtoint ptr %iv.1 to i64 + br label %exit + +loop.latch: + %gep.merge = getelementptr inbounds nuw i8, ptr %iv, i64 4 + %c.3 = icmp eq ptr %gep.merge, %l.gep.v + br i1 %c.3, label %loop.comp, label %loop.header + +loop.comp: + br label %exit + +exit: + %mer = phi i64 [ %pi.l.gep.v, %entry ], [ %pi.iv, %loop.early.exit ], [ %pi.l.gep.v, %loop.comp ] + ret i64 %mer +} diff --git a/llvm/test/Transforms/SimplifyCFG/speculate-derefable-load.ll b/llvm/test/Transforms/SimplifyCFG/speculate-derefable-load.ll index b8c999d700aa7..693da976d1752 100644 --- a/llvm/test/Transforms/SimplifyCFG/speculate-derefable-load.ll +++ b/llvm/test/Transforms/SimplifyCFG/speculate-derefable-load.ll @@ -4,14 +4,10 @@ define i64 @align_deref_align(i1 %c, ptr %p) { ; CHECK-LABEL: define i64 @align_deref_align( ; CHECK-SAME: i1 [[C:%.*]], ptr [[P:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[P]], i64 8), "align"(ptr [[P]], i64 8) ] -; CHECK-NEXT: br i1 [[C]], label %[[IF:.*]], label %[[EXIT:.*]] -; CHECK: [[IF]]: ; CHECK-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8 -; CHECK-NEXT: br label %[[EXIT]] -; CHECK: [[EXIT]]: -; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[V]], %[[IF]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: [[RES:%.*]] = select i1 [[C]], i64 [[V]], i64 0 ; CHECK-NEXT: ret i64 [[RES]] ; entry: @@ -52,17 +48,12 @@ exit: define i64 @assume_deref_align2(i1 %c1, i32 %x, ptr %p) { ; CHECK-LABEL: define i64 @assume_deref_align2( ; CHECK-SAME: i1 [[C1:%.*]], i32 [[X:%.*]], ptr [[P:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[P]], i64 8), "align"(ptr [[P]], i64 8) ] -; CHECK-NEXT: br i1 [[C1]], label %[[IF1:.*]], label %[[EXIT:.*]] -; CHECK: [[IF1]]: ; CHECK-NEXT: [[C2:%.*]] = icmp ugt i32 [[X]], 10 -; CHECK-NEXT: br i1 [[C2]], label %[[IF2:.*]], label %[[EXIT]] -; CHECK: [[IF2]]: ; CHECK-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8 -; CHECK-NEXT: br label %[[EXIT]] -; CHECK: [[EXIT]]: -; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[V]], %[[IF2]] ], [ 1, %[[IF1]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[C2]], i64 [[V]], i64 1 +; CHECK-NEXT: [[RES:%.*]] = select i1 [[C1]], i64 [[SPEC_SELECT]], i64 0 ; CHECK-NEXT: ret i64 [[RES]] ; entry: