diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index e1e24a99d0474..75ef4bead046c 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -3446,9 +3446,18 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { KnownBits Known = computeKnownBits(RK.WasOn, /*CtxI=*/nullptr); unsigned TZ = std::min(Known.countMinTrailingZeros(), Value::MaxAlignmentExponent); - if ((1ULL << TZ) < RK.ArgValue) - continue; - return CallBase::removeOperandBundle(II, OBU.getTagID()); + if ((1ULL << TZ) >= RK.ArgValue) + return CallBase::removeOperandBundle(II, OBU.getTagID()); + + auto *LI = dyn_cast(OBU.Inputs[0]); + if (LI && + isValidAssumeForContext(II, LI, &DT, /*AllowEphemerals=*/true)) { + LI->setMetadata(LLVMContext::MD_align, + MDNode::get(II->getContext(), + ValueAsMetadata::getConstant( + Builder.getInt64(RK.ArgValue)))); + return CallBase::removeOperandBundle(II, OBU.getTagID()); + } } } diff --git a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp index 0f8cc6ca6ed21..67f96d09a7c23 100644 --- a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp +++ b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp @@ -31,6 +31,7 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" @@ -1600,6 +1601,14 @@ bool EarlyCSE::processNode(DomTreeNode *Node) { if (InVal.IsLoad) if (auto *I = dyn_cast(Op)) combineMetadataForCSE(I, &Inst, false); + if (auto *AlignMD = Inst.getMetadata(LLVMContext::MD_align)) { + auto *A = mdconst::extract(AlignMD->getOperand(0)); + if (Op->getPointerAlignment(SQ.DL).value() % A->getZExtValue() != 0) { + IRBuilder B(&Inst); + B.CreateAlignmentAssumption(SQ.DL, Op, A); + } + } + if (!Inst.use_empty()) Inst.replaceAllUsesWith(Op); salvageKnowledge(&Inst, &AC); diff --git a/llvm/test/Transforms/InstCombine/assume-align.ll b/llvm/test/Transforms/InstCombine/assume-align.ll index 4185b10eeca95..83821812067ab 100644 --- a/llvm/test/Transforms/InstCombine/assume-align.ll +++ b/llvm/test/Transforms/InstCombine/assume-align.ll @@ -123,11 +123,9 @@ define i8 @assume_align_non_pow2(ptr %p) { ret i8 %v } -; TODO: Can fold alignment assumption into !align metadata on load. define ptr @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata(ptr %p) { ; CHECK-LABEL: @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata( -; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8 -; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 8) ] +; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0:![0-9]+]] ; CHECK-NEXT: ret ptr [[P2]] ; %p2 = load ptr, ptr %p @@ -135,6 +133,16 @@ define ptr @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata(ptr %p) ret ptr %p2 } +define ptr @fold_assume_align_i32_pow2_of_loaded_pointer_into_align_metadata(ptr %p) { +; CHECK-LABEL: @fold_assume_align_i32_pow2_of_loaded_pointer_into_align_metadata( +; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0]] +; CHECK-NEXT: ret ptr [[P2]] +; + %p2 = load ptr, ptr %p + call void @llvm.assume(i1 true) [ "align"(ptr %p2, i32 8) ] + ret ptr %p2 +} + define ptr @dont_fold_assume_align_pow2_of_loaded_pointer_into_align_metadata_due_to_call(ptr %p) { ; CHECK-LABEL: @dont_fold_assume_align_pow2_of_loaded_pointer_into_align_metadata_due_to_call( ; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8 @@ -187,7 +195,7 @@ define ptr @redundant_assume_align_1(ptr %p) { define ptr @redundant_assume_align_8_via_align_metadata(ptr %p) { ; CHECK-LABEL: @redundant_assume_align_8_via_align_metadata( -; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0:![0-9]+]] +; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0]] ; CHECK-NEXT: call void @foo(ptr [[P2]]) ; CHECK-NEXT: ret ptr [[P2]] ; @@ -199,8 +207,7 @@ define ptr @redundant_assume_align_8_via_align_metadata(ptr %p) { define ptr @assume_align_16_via_align_metadata(ptr %p) { ; CHECK-LABEL: @assume_align_16_via_align_metadata( -; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0]] -; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i32 16) ] +; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META1:![0-9]+]] ; CHECK-NEXT: call void @foo(ptr [[P2]]) ; CHECK-NEXT: ret ptr [[P2]] ; @@ -247,6 +254,16 @@ define ptr @redundant_assume_align_8_via_asume(ptr %p) { ret ptr %p } +@g2 = external constant i128, align 8 + +define void @redundant_assume_align_global_eq() { +; CHECK-LABEL: @redundant_assume_align_global_eq( +; CHECK-NEXT: ret void +; + call void @llvm.assume(i1 true) [ "align"(ptr @g2, i64 8) ] + ret void +} + define ptr @assume_align_1(ptr %p) { ; CHECK-LABEL: @assume_align_1( ; CHECK-NEXT: call void @foo(ptr [[P:%.*]]) @@ -273,4 +290,5 @@ define ptr @assume_load_pointer_result(ptr %p, i64 %align) { ;. ; CHECK: [[META0]] = !{i64 8} +; CHECK: [[META1]] = !{i64 16} ;. diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll index 4196625e6bd21..0ab19c0a797b8 100644 --- a/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll +++ b/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll @@ -8,8 +8,7 @@ declare void @llvm.assume(i1 noundef) define i32 @earlycse_entry(ptr %p) { ; CHECK-LABEL: define i32 @earlycse_entry( ; CHECK-SAME: ptr captures(none) [[P:%.*]]) local_unnamed_addr { -; CHECK-NEXT: [[L_I:%.*]] = load ptr, ptr [[P]], align 8 -; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_I]], i64 4) ] +; CHECK-NEXT: [[L_I:%.*]] = load ptr, ptr [[P]], align 8, !align [[META0:![0-9]+]] ; CHECK-NEXT: [[L_ASSUME_ALIGNED_I_I:%.*]] = load i32, ptr [[L_I]], align 4 ; CHECK-NEXT: [[R_I_I:%.*]] = tail call i32 @swap(i32 [[L_ASSUME_ALIGNED_I_I]]) ; CHECK-NEXT: [[L_2_I:%.*]] = load ptr, ptr [[P]], align 8 @@ -31,8 +30,7 @@ define i32 @earlycse_entry(ptr %p) { define i32 @earlycse_fn1(ptr %p) { ; CHECK-LABEL: define i32 @earlycse_fn1( ; CHECK-SAME: ptr captures(none) [[P:%.*]]) local_unnamed_addr { -; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[P]], align 8 -; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L]], i64 4) ] +; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[P]], align 8, !align [[META0]] ; CHECK-NEXT: [[L_ASSUME_ALIGNED_I:%.*]] = load i32, ptr [[L]], align 4 ; CHECK-NEXT: [[R_I:%.*]] = tail call i32 @swap(i32 [[L_ASSUME_ALIGNED_I]]) ; CHECK-NEXT: [[L_2:%.*]] = load ptr, ptr [[P]], align 8 @@ -67,8 +65,7 @@ declare i32 @swap(i32) define void @sroa_align_entry(ptr %p) { ; CHECK-LABEL: define void @sroa_align_entry( ; CHECK-SAME: ptr readonly captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] { -; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P]], i64 8) ] -; CHECK-NEXT: [[DOT0_COPYLOAD_I_I_I:%.*]] = load i64, ptr [[P]], align 8 +; CHECK-NEXT: [[DOT0_COPYLOAD_I_I_I:%.*]] = load i64, ptr [[P]], align 1 ; CHECK-NEXT: [[TMP2:%.*]] = inttoptr i64 [[DOT0_COPYLOAD_I_I_I]] to ptr ; CHECK-NEXT: store i32 0, ptr [[TMP2]], align 4 ; CHECK-NEXT: ret void @@ -83,8 +80,7 @@ define void @sroa_align_entry(ptr %p) { define ptr @sroa_fn1(ptr %p) { ; CHECK-LABEL: define ptr @sroa_fn1( ; CHECK-SAME: ptr readonly captures(none) [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] { -; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[P]], align 8 -; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L]], i64 8) ] +; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[P]], align 8, !align [[META1:![0-9]+]] ; CHECK-NEXT: [[L_FN3_I_I:%.*]] = load i64, ptr [[L]], align 8 ; CHECK-NEXT: [[I_I:%.*]] = inttoptr i64 [[L_FN3_I_I]] to ptr ; CHECK-NEXT: ret ptr [[I_I]] @@ -118,3 +114,7 @@ define i64 @sroa_fn3(ptr %0) { %l.fn3 = load i64, ptr %0, align 1 ret i64 %l.fn3 } +;. +; CHECK: [[META0]] = !{i64 4} +; CHECK: [[META1]] = !{i64 8} +;.