Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions llvm/include/llvm/IR/Metadata.h
Original file line number Diff line number Diff line change
Expand Up @@ -844,6 +844,13 @@ struct AAMDNodes {
/// together. Different from `merge`, where different locations should
/// overlap each other, `concat` puts non-overlapping locations together.
AAMDNodes concat(const AAMDNodes &Other) const;

/// Create a new AAMDNode for accessing \p AccessSize bytes of this AAMDNode.
/// If his AAMDNode has !tbaa.struct and \p AccessSize matches the size of the
/// field at offset 0, get the TBAA tag describing the accessed field.
AAMDNodes adjustForAccess(unsigned AccessSize);
AAMDNodes adjustForAccess(size_t Offset, Type *AccessTy,
const DataLayout &DL);
};

// Specialize DenseMapInfo for AAMDNodes.
Expand Down
28 changes: 28 additions & 0 deletions llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -817,3 +817,31 @@ MDNode *AAMDNodes::extendToTBAA(MDNode *MD, ssize_t Len) {
ConstantAsMetadata::get(ConstantInt::get(PreviousSize->getType(), Len));
return MDNode::get(MD->getContext(), NextNodes);
}

AAMDNodes AAMDNodes::adjustForAccess(unsigned AccessSize) {
AAMDNodes New = *this;
MDNode *M = New.TBAAStruct;
if (M && M->getOperand(0) && mdconst::hasa<ConstantInt>(M->getOperand(0)) &&
mdconst::extract<ConstantInt>(M->getOperand(0))->isZero() &&
M->getOperand(1) && mdconst::hasa<ConstantInt>(M->getOperand(1)) &&
mdconst::extract<ConstantInt>(M->getOperand(1))->getValue() ==
AccessSize &&
M->getOperand(2) && isa<MDNode>(M->getOperand(2))) {
New.TBAAStruct = nullptr;
New.TBAA = cast<MDNode>(M->getOperand(2));
}
return New;
}

AAMDNodes AAMDNodes::adjustForAccess(size_t Offset, Type *AccessTy,
const DataLayout &DL) {

AAMDNodes New = shift(Offset);
if (!DL.typeSizeEqualsStoreSize(AccessTy))
return New;
TypeSize Size = DL.getTypeStoreSize(AccessTy);
if (Size.isScalable())
return New;

return New.adjustForAccess(Size.getKnownMinValue());
}
15 changes: 1 addition & 14 deletions llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -172,20 +172,7 @@ Instruction *InstCombinerImpl::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {

// If the memcpy has metadata describing the members, see if we can get the
// TBAA tag describing our copy.
AAMDNodes AACopyMD = MI->getAAMetadata();

if (MDNode *M = AACopyMD.TBAAStruct) {
AACopyMD.TBAAStruct = nullptr;
if (M->getNumOperands() == 3 && M->getOperand(0) &&
mdconst::hasa<ConstantInt>(M->getOperand(0)) &&
mdconst::extract<ConstantInt>(M->getOperand(0))->isZero() &&
M->getOperand(1) &&
mdconst::hasa<ConstantInt>(M->getOperand(1)) &&
mdconst::extract<ConstantInt>(M->getOperand(1))->getValue() ==
Size &&
M->getOperand(2) && isa<MDNode>(M->getOperand(2)))
AACopyMD.TBAA = cast<MDNode>(M->getOperand(2));
}
AAMDNodes AACopyMD = MI->getAAMetadata().adjustForAccess(Size);

Value *Src = MI->getArgOperand(1);
Value *Dest = MI->getArgOperand(0);
Expand Down
48 changes: 34 additions & 14 deletions llvm/lib/Transforms/Scalar/SROA.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2914,7 +2914,8 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {

// Do this after copyMetadataForLoad() to preserve the TBAA shift.
if (AATags)
NewLI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
NewLI->setAAMetadata(AATags.adjustForAccess(
NewBeginOffset - BeginOffset, NewLI->getType(), DL));

// Try to preserve nonnull metadata
V = NewLI;
Expand All @@ -2936,7 +2937,9 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
IRB.CreateAlignedLoad(TargetTy, getNewAllocaSlicePtr(IRB, LTy),
getSliceAlign(), LI.isVolatile(), LI.getName());
if (AATags)
NewLI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
NewLI->setAAMetadata(AATags.adjustForAccess(
NewBeginOffset - BeginOffset, NewLI->getType(), DL));

if (LI.isVolatile())
NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
NewLI->copyMetadata(LI, {LLVMContext::MD_mem_parallel_loop_access,
Expand Down Expand Up @@ -3011,7 +3014,8 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
LLVMContext::MD_access_group});
if (AATags)
Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
Store->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
V->getType(), DL));
Pass.DeadInsts.push_back(&SI);

// NOTE: Careful to use OrigV rather than V.
Expand All @@ -3038,7 +3042,8 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
LLVMContext::MD_access_group});
if (AATags)
Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
Store->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
V->getType(), DL));

migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI,
Store, Store->getPointerOperand(),
Expand Down Expand Up @@ -3097,8 +3102,10 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
}
NewSI->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
LLVMContext::MD_access_group});
if (AATags)
NewSI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
if (AATags) {
NewSI->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
V->getType(), DL));
}
if (SI.isVolatile())
NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID());
if (NewSI->isAtomic())
Expand Down Expand Up @@ -3280,8 +3287,10 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
IRB.CreateAlignedStore(V, NewPtr, NewAI.getAlign(), II.isVolatile());
New->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
LLVMContext::MD_access_group});
if (AATags)
New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
if (AATags) {
New->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
V->getType(), DL));
}

migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &II,
New, New->getPointerOperand(), V, DL);
Expand Down Expand Up @@ -3486,7 +3495,8 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
Load->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
LLVMContext::MD_access_group});
if (AATags)
Load->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
Load->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
Load->getType(), DL));
Src = Load;
}

Expand All @@ -3507,8 +3517,10 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
IRB.CreateAlignedStore(Src, DstPtr, DstAlign, II.isVolatile()));
Store->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
LLVMContext::MD_access_group});
if (AATags)
Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
if (AATags) {
Store->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
Src->getType(), DL));
}

APInt Offset(DL.getIndexTypeSizeInBits(DstPtr->getType()), 0);
if (IsDest) {
Expand Down Expand Up @@ -3836,7 +3848,8 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0);
if (AATags &&
GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset))
Load->setAAMetadata(AATags.shift(Offset.getZExtValue()));
Load->setAAMetadata(
AATags.adjustForAccess(Offset.getZExtValue(), Load->getType(), DL));

Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert");
LLVM_DEBUG(dbgs() << " to: " << *Load << "\n");
Expand Down Expand Up @@ -3887,8 +3900,10 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
APInt Offset(
DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0);
GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset);
if (AATags)
Store->setAAMetadata(AATags.shift(Offset.getZExtValue()));
if (AATags) {
Store->setAAMetadata(AATags.adjustForAccess(
Offset.getZExtValue(), ExtractValue->getType(), DL));
}

// migrateDebugInfo requires the base Alloca. Walk to it from this gep.
// If we cannot (because there's an intervening non-const or unbounded
Expand Down Expand Up @@ -4542,6 +4557,7 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {

Value *StoreBasePtr = SI->getPointerOperand();
IRB.SetInsertPoint(SI);
AAMDNodes AATags = SI->getAAMetadata();

LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n");

Expand All @@ -4561,6 +4577,10 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access,
LLVMContext::MD_access_group,
LLVMContext::MD_DIAssignID});

if (AATags)
PStore->setAAMetadata(
AATags.adjustForAccess(PartOffset, PLoad->getType(), DL));
LLVM_DEBUG(dbgs() << " +" << PartOffset << ":" << *PStore << "\n");
}

Expand Down
5 changes: 3 additions & 2 deletions llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ define ptr @test2() {
define void @test3_multiple_fields(ptr nocapture %a, ptr nocapture %b) {
; CHECK-LABEL: @test3_multiple_fields(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr [[B:%.*]], align 4
; CHECK-NEXT: store i64 [[TMP0]], ptr [[A:%.*]], align 4
; CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr [[B:%.*]], align 4, !tbaa.struct [[TBAA_STRUCT3:![0-9]+]]
; CHECK-NEXT: store i64 [[TMP0]], ptr [[A:%.*]], align 4, !tbaa.struct [[TBAA_STRUCT3]]
; CHECK-NEXT: ret void
;
entry:
Expand Down Expand Up @@ -86,4 +86,5 @@ entry:
; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
; CHECK: [[META1]] = !{!"float", [[META2:![0-9]+]]}
; CHECK: [[META2]] = !{!"Simple C/C++ TBAA"}
; CHECK: [[TBAA_STRUCT3]] = !{i64 0, i64 4, [[TBAA0]], i64 4, i64 4, [[TBAA0]]}
;.
22 changes: 10 additions & 12 deletions llvm/test/Transforms/SROA/tbaa-struct2.ll
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ declare double @subcall(double %g, i32 %m)
define double @bar(ptr %wishart) {
; CHECK-LABEL: @bar(
; CHECK-NEXT: [[TMP_SROA_3:%.*]] = alloca [4 x i8], align 4
; CHECK-NEXT: [[TMP_SROA_0_0_COPYLOAD:%.*]] = load double, ptr [[WISHART:%.*]], align 8, !tbaa.struct [[TBAA_STRUCT0:![0-9]+]]
; CHECK-NEXT: [[TMP_SROA_0_0_COPYLOAD:%.*]] = load double, ptr [[WISHART:%.*]], align 8, !tbaa [[TBAA0:![0-9]+]]
; CHECK-NEXT: [[TMP_SROA_2_0_WISHART_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[WISHART]], i64 8
; CHECK-NEXT: [[TMP_SROA_2_0_COPYLOAD:%.*]] = load i32, ptr [[TMP_SROA_2_0_WISHART_SROA_IDX]], align 8, !tbaa.struct [[TBAA_STRUCT7:![0-9]+]]
; CHECK-NEXT: [[TMP_SROA_2_0_COPYLOAD:%.*]] = load i32, ptr [[TMP_SROA_2_0_WISHART_SROA_IDX]], align 8, !tbaa [[TBAA4:![0-9]+]]
; CHECK-NEXT: [[TMP_SROA_3_0_WISHART_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[WISHART]], i64 12
; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP_SROA_3]], ptr align 4 [[TMP_SROA_3_0_WISHART_SROA_IDX]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT8:![0-9]+]]
; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[TMP_SROA_3]], ptr align 4 [[TMP_SROA_3_0_WISHART_SROA_IDX]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT6:![0-9]+]]
; CHECK-NEXT: [[CALL:%.*]] = call double @subcall(double [[TMP_SROA_0_0_COPYLOAD]], i32 [[TMP_SROA_2_0_COPYLOAD]])
; CHECK-NEXT: ret double [[CALL]]
;
Expand All @@ -38,15 +38,13 @@ define double @bar(ptr %wishart) {
;.
; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
;.
; CHECK: [[TBAA_STRUCT0]] = !{i64 0, i64 8, !1, i64 8, i64 4, !5}
; CHECK: [[META1:![0-9]+]] = !{!2, !2, i64 0}
; CHECK: [[META2:![0-9]+]] = !{!"double", !3, i64 0}
; CHECK: [[META3:![0-9]+]] = !{!"omnipotent char", !4, i64 0}
; CHECK: [[META4:![0-9]+]] = !{!"Simple C++ TBAA"}
; CHECK: [[META5:![0-9]+]] = !{!6, !6, i64 0}
; CHECK: [[META6:![0-9]+]] = !{!"int", !3, i64 0}
; CHECK: [[TBAA_STRUCT7]] = !{i64 0, i64 4, !5}
; CHECK: [[TBAA_STRUCT8]] = !{}
; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
; CHECK: [[META1]] = !{!"double", [[META2:![0-9]+]], i64 0}
; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
; CHECK: [[TBAA4]] = !{[[META5:![0-9]+]], [[META5]], i64 0}
; CHECK: [[META5]] = !{!"int", [[META2]], i64 0}
; CHECK: [[TBAA_STRUCT6]] = !{}
;.
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; CHECK-MODIFY-CFG: {{.*}}
Expand Down
28 changes: 14 additions & 14 deletions llvm/test/Transforms/SROA/tbaa-struct3.ll
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ define void @load_store_transfer_split_struct_tbaa_2_float(ptr dereferenceable(2
; CHECK-NEXT: entry:
; CHECK-NEXT: [[TMP0:%.*]] = bitcast float [[A]] to i32
; CHECK-NEXT: [[TMP1:%.*]] = bitcast float [[B]] to i32
; CHECK-NEXT: store i32 [[TMP0]], ptr [[RES]], align 4
; CHECK-NEXT: store i32 [[TMP0]], ptr [[RES]], align 4, !tbaa [[TBAA0:![0-9]+]]
; CHECK-NEXT: [[RES_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[RES]], i64 4
; CHECK-NEXT: store i32 [[TMP1]], ptr [[RES_SROA_IDX]], align 4
; CHECK-NEXT: store i32 [[TMP1]], ptr [[RES_SROA_IDX]], align 4, !tbaa [[TBAA0]]
; CHECK-NEXT: [[P:%.*]] = load ptr, ptr [[RES]], align 8
; CHECK-NEXT: ret void
;
Expand All @@ -29,9 +29,9 @@ define void @memcpy_transfer(ptr dereferenceable(24) %res, float %a, float %b) {
; CHECK-SAME: ptr dereferenceable(24) [[RES:%.*]], float [[A:%.*]], float [[B:%.*]]) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[L_PTR:%.*]] = load ptr, ptr [[RES]], align 8
; CHECK-NEXT: store float [[A]], ptr [[L_PTR]], align 1, !tbaa.struct [[TBAA_STRUCT0:![0-9]+]]
; CHECK-NEXT: store float [[A]], ptr [[L_PTR]], align 1, !tbaa [[TBAA0]]
; CHECK-NEXT: [[TMP_SROA_2_0_L_PTR_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[L_PTR]], i64 4
; CHECK-NEXT: store float [[B]], ptr [[TMP_SROA_2_0_L_PTR_SROA_IDX]], align 1, !tbaa.struct [[TBAA_STRUCT5:![0-9]+]]
; CHECK-NEXT: store float [[B]], ptr [[TMP_SROA_2_0_L_PTR_SROA_IDX]], align 1, !tbaa [[TBAA0]]
; CHECK-NEXT: ret void
;
entry:
Expand All @@ -49,11 +49,11 @@ define void @memcpy_transfer_tbaa_field_and_size_do_not_align(ptr dereferenceabl
; CHECK-SAME: ptr dereferenceable(24) [[RES:%.*]], float [[A:%.*]], float [[B:%.*]]) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[L_PTR:%.*]] = load ptr, ptr [[RES]], align 8
; CHECK-NEXT: store float [[A]], ptr [[L_PTR]], align 1, !tbaa.struct [[TBAA_STRUCT0]]
; CHECK-NEXT: store float [[A]], ptr [[L_PTR]], align 1, !tbaa [[TBAA0]]
; CHECK-NEXT: [[TMP_SROA_2_0_L_PTR_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[L_PTR]], i64 4
; CHECK-NEXT: [[TMP0:%.*]] = bitcast float [[B]] to i32
; CHECK-NEXT: [[TMP_SROA_2_0_EXTRACT_TRUNC:%.*]] = trunc i32 [[TMP0]] to i16
; CHECK-NEXT: store i16 [[TMP_SROA_2_0_EXTRACT_TRUNC]], ptr [[TMP_SROA_2_0_L_PTR_SROA_IDX]], align 1, !tbaa.struct [[TBAA_STRUCT5]]
; CHECK-NEXT: store i16 [[TMP_SROA_2_0_EXTRACT_TRUNC]], ptr [[TMP_SROA_2_0_L_PTR_SROA_IDX]], align 1, !tbaa.struct [[TBAA_STRUCT4:![0-9]+]]
; CHECK-NEXT: ret void
;
entry:
Expand All @@ -74,8 +74,8 @@ define void @load_store_transfer_split_struct_tbaa_2_i31(ptr dereferenceable(24)
; CHECK-NEXT: store i31 [[A]], ptr [[TMP]], align 4
; CHECK-NEXT: [[TMP_4_TMP_4_SROA_IDX:%.*]] = getelementptr inbounds i8, ptr [[TMP]], i64 4
; CHECK-NEXT: store i31 [[B]], ptr [[TMP_4_TMP_4_SROA_IDX]], align 4
; CHECK-NEXT: [[TMP_0_L1:%.*]] = load i62, ptr [[TMP]], align 4, !tbaa.struct [[TBAA_STRUCT0]]
; CHECK-NEXT: store i62 [[TMP_0_L1]], ptr [[RES]], align 4, !tbaa.struct [[TBAA_STRUCT0]]
; CHECK-NEXT: [[TMP_0_L1:%.*]] = load i62, ptr [[TMP]], align 4, !tbaa.struct [[TBAA_STRUCT5:![0-9]+]]
; CHECK-NEXT: store i62 [[TMP_0_L1]], ptr [[RES]], align 4, !tbaa.struct [[TBAA_STRUCT5]]
; CHECK-NEXT: ret void
;
entry:
Expand All @@ -98,10 +98,10 @@ declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias
!3 = !{!"omnipotent char", !4, i64 0}
!4 = !{!"Simple C++ TBAA"}
;.
; CHECK: [[TBAA_STRUCT0]] = !{i64 0, i64 4, [[META1:![0-9]+]], i64 4, i64 4, [[META1]]}
; CHECK: [[META1]] = !{[[META2:![0-9]+]], [[META2]], i64 0}
; CHECK: [[META2]] = !{!"float", [[META3:![0-9]+]], i64 0}
; CHECK: [[META3]] = !{!"omnipotent char", [[META4:![0-9]+]], i64 0}
; CHECK: [[META4]] = !{!"Simple C++ TBAA"}
; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 4, [[META1]]}
; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
; CHECK: [[META1]] = !{!"float", [[META2:![0-9]+]], i64 0}
; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
; CHECK: [[TBAA_STRUCT4]] = !{i64 0, i64 4, [[TBAA0]]}
; CHECK: [[TBAA_STRUCT5]] = !{i64 0, i64 4, [[TBAA0]], i64 4, i64 4, [[TBAA0]]}
;.