diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h index 812b5a9f72a3ad..4140387a1f3410 100644 --- a/llvm/include/llvm/Analysis/AliasAnalysis.h +++ b/llvm/include/llvm/Analysis/AliasAnalysis.h @@ -244,12 +244,23 @@ class AAQueryInfo { public: using LocPair = std::pair; struct CacheEntry { + /// Cache entry is neither an assumption nor does it use a (non-definitive) + /// assumption. + static constexpr int Definitive = -2; + /// Cache entry is not an assumption itself, but may be using an assumption + /// from higher up the stack. + static constexpr int AssumptionBased = -1; + AliasResult Result; - /// Number of times a NoAlias assumption has been used. - /// 0 for assumptions that have not been used, -1 for definitive results. + /// Number of times a NoAlias assumption has been used, 0 for assumptions + /// that have not been used. Can also take one of the Definitive or + /// AssumptionBased values documented above. int NumAssumptionUses; + /// Whether this is a definitive (non-assumption) result. - bool isDefinitive() const { return NumAssumptionUses < 0; } + bool isDefinitive() const { return NumAssumptionUses == Definitive; } + /// Whether this is an assumption that has not been proven yet. + bool isAssumption() const { return NumAssumptionUses >= 0; } }; // Alias analysis result aggregration using which this query is performed. diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp index 161a3034e48299..e474899fb548ec 100644 --- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp +++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp @@ -1692,9 +1692,12 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size, if (!Pair.second) { auto &Entry = Pair.first->second; if (!Entry.isDefinitive()) { - // Remember that we used an assumption. - ++Entry.NumAssumptionUses; + // Remember that we used an assumption. This may either be a direct use + // of an assumption, or a use of an entry that may itself be based on an + // assumption. ++AAQI.NumAssumptionUses; + if (Entry.isAssumption()) + ++Entry.NumAssumptionUses; } // Cache contains sorted {V1,V2} pairs but we should return original order. auto Result = Entry.Result; @@ -1722,7 +1725,6 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size, Entry.Result = Result; // Cache contains sorted {V1,V2} pairs. Entry.Result.swap(Swapped); - Entry.NumAssumptionUses = -1; // If the assumption has been disproven, remove any results that may have // been based on this assumption. Do this after the Entry updates above to @@ -1734,8 +1736,26 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size, // The result may still be based on assumptions higher up in the chain. // Remember it, so it can be purged from the cache later. if (OrigNumAssumptionUses != AAQI.NumAssumptionUses && - Result != AliasResult::MayAlias) + Result != AliasResult::MayAlias) { AAQI.AssumptionBasedResults.push_back(Locs); + Entry.NumAssumptionUses = AAQueryInfo::CacheEntry::AssumptionBased; + } else { + Entry.NumAssumptionUses = AAQueryInfo::CacheEntry::Definitive; + } + + // Depth is incremented before this function is called, so Depth==1 indicates + // a root query. + if (AAQI.Depth == 1) { + // Any remaining assumption based results must be based on proven + // assumptions, so convert them to definitive results. + for (const auto &Loc : AAQI.AssumptionBasedResults) { + auto It = AAQI.AliasCache.find(Loc); + if (It != AAQI.AliasCache.end()) + It->second.NumAssumptionUses = AAQueryInfo::CacheEntry::Definitive; + } + AAQI.AssumptionBasedResults.clear(); + AAQI.NumAssumptionUses = 0; + } return Result; } diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr98978.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr98978.ll new file mode 100644 index 00000000000000..429bf13b2b87a2 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/pr98978.ll @@ -0,0 +1,106 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -S -passes=slp-vectorizer < %s | FileCheck %s + +target triple = "x86_64-redhat-linux-gnu" + +; The load+store sequence inside bb10 should not get vectorized. Previously, +; we incorrectly determined that the pointers do not alias, because a cache +; entry based indirectly on a disproven NoAlias assumption was not cleared +; from the BatchAA cache. +define void @test(ptr %p1, i64 %arg1, i64 %arg2) { +; CHECK-LABEL: define void @test( +; CHECK-SAME: ptr [[P1:%.*]], i64 [[ARG1:%.*]], i64 [[ARG2:%.*]]) { +; CHECK-NEXT: [[_PREHEADER48_PREHEADER_1:.*]]: +; CHECK-NEXT: br label %[[_LOOPEXIT49_1:.*]] +; CHECK: [[_LOOPEXIT49_1]]: +; CHECK-NEXT: [[I:%.*]] = phi ptr [ [[I21:%.*]], %[[BB20:.*]] ], [ [[P1]], %[[_PREHEADER48_PREHEADER_1]] ] +; CHECK-NEXT: br i1 false, label %[[BB22:.*]], label %[[DOTPREHEADER48_PREHEADER_1:.*]] +; CHECK: [[DEAD:.*]]: +; CHECK-NEXT: br label %[[DOTPREHEADER48_PREHEADER_1]] +; CHECK: [[_PREHEADER48_PREHEADER_2:.*:]] +; CHECK-NEXT: [[I5:%.*]] = phi ptr [ [[I]], %[[DEAD]] ], [ [[I]], %[[_LOOPEXIT49_1]] ] +; CHECK-NEXT: br label %[[DOTLOOPEXIT49_1:.*]] +; CHECK: [[DEAD1:.*]]: +; CHECK-NEXT: br i1 false, label %[[DOTLOOPEXIT49_1]], label %[[BB20]] +; CHECK: [[_LOOPEXIT49_2:.*:]] +; CHECK-NEXT: [[I6:%.*]] = phi ptr [ [[I5]], %[[DEAD1]] ], [ [[I5]], %[[DOTPREHEADER48_PREHEADER_1]] ] +; CHECK-NEXT: [[I7:%.*]] = getelementptr i8, ptr [[I6]], i64 [[ARG1]] +; CHECK-NEXT: br label %[[BB10:.*]] +; CHECK: [[DEAD2:.*]]: +; CHECK-NEXT: br label %[[BB10]] +; CHECK: [[BB10]]: +; CHECK-NEXT: [[I11:%.*]] = phi ptr [ [[I7]], %[[DOTLOOPEXIT49_1]] ], [ null, %[[DEAD2]] ] +; CHECK-NEXT: [[I16:%.*]] = getelementptr i8, ptr [[I11]], i64 8 +; CHECK-NEXT: [[I17:%.*]] = load i64, ptr [[I16]], align 1 +; CHECK-NEXT: store i64 [[I17]], ptr [[I6]], align 1 +; CHECK-NEXT: [[I18:%.*]] = getelementptr i8, ptr [[I6]], i64 8 +; CHECK-NEXT: [[I19:%.*]] = load i64, ptr [[I11]], align 1 +; CHECK-NEXT: store i64 [[I19]], ptr [[I18]], align 1 +; CHECK-NEXT: br label %[[BB20]] +; CHECK: [[BB20]]: +; CHECK-NEXT: [[I21]] = phi ptr [ [[I5]], %[[DEAD1]] ], [ [[I6]], %[[BB10]] ] +; CHECK-NEXT: br label %[[_LOOPEXIT49_1]] +; CHECK: [[BB22]]: +; CHECK-NEXT: [[I23:%.*]] = getelementptr i8, ptr [[I]], i64 [[ARG2]] +; CHECK-NEXT: [[I25:%.*]] = getelementptr i8, ptr [[I23]], i64 8 +; CHECK-NEXT: br label %[[BB26:.*]] +; CHECK: [[BB26]]: +; CHECK-NEXT: [[I27:%.*]] = phi ptr [ null, %[[BB26]] ], [ [[I25]], %[[BB22]] ] +; CHECK-NEXT: store i64 0, ptr [[I27]], align 1 +; CHECK-NEXT: [[I28:%.*]] = getelementptr i8, ptr [[I27]], i64 8 +; CHECK-NEXT: [[I29:%.*]] = load i64, ptr [[I23]], align 1 +; CHECK-NEXT: store i64 0, ptr [[I28]], align 1 +; CHECK-NEXT: br label %[[BB26]] +; +entry: + br label %loop1 + +loop1: ; preds = %bb20, %entry + %i = phi ptr [ %i21, %bb20 ], [ %p1, %entry ] + br i1 false, label %bb22, label %.preheader48.preheader.1 + +dead: ; No predecessors! + br label %.preheader48.preheader.1 + +.preheader48.preheader.1: ; preds = %dead, %loop1 + %i5 = phi ptr [ %i, %dead ], [ %i, %loop1 ] + br label %.loopexit49.1 + +dead1: ; No predecessors! + br i1 false, label %.loopexit49.1, label %bb20 + +.loopexit49.1: ; preds = %dead1, %.preheader48.preheader.1 + %i6 = phi ptr [ %i5, %dead1 ], [ %i5, %.preheader48.preheader.1 ] + %i7 = getelementptr i8, ptr %i6, i64 %arg1 + br label %bb10 + +dead2: ; No predecessors! + br label %bb10 + +bb10: ; preds = %dead2, %.loopexit49.1 + %i11 = phi ptr [ %i7, %.loopexit49.1 ], [ null, %dead2 ] + %i16 = getelementptr i8, ptr %i11, i64 8 + %i17 = load i64, ptr %i16, align 1 + store i64 %i17, ptr %i6, align 1 + %i18 = getelementptr i8, ptr %i6, i64 8 + %i19 = load i64, ptr %i11, align 1 + store i64 %i19, ptr %i18, align 1 + br label %bb20 + +bb20: ; preds = %bb10, %dead1 + %i21 = phi ptr [ %i5, %dead1 ], [ %i6, %bb10 ] + br label %loop1 + +bb22: ; preds = %loop1 + %i23 = getelementptr i8, ptr %i, i64 %arg2 + %i25 = getelementptr i8, ptr %i23, i64 8 + br label %bb26 + +bb26: ; preds = %bb26, %bb22 + %i27 = phi ptr [ null, %bb26 ], [ %i25, %bb22 ] + store i64 0, ptr %i27, align 1 + %i28 = getelementptr i8, ptr %i27, i64 8 + %i29 = load i64, ptr %i23, align 1 + store i64 0, ptr %i28, align 1 + br label %bb26 +}