[LAA] Support pointer phis in loop by analyzing each incoming pointer.

SCEV does not look through non-header PHIs inside the loop. Such phis
can be analyzed by adding separate accesses for each incoming pointer
value.

This results in 2 more loops vectorized in SPEC2000/186.crafty and
avoids regressions when sinking instructions before vectorizing.

Reviewed By: Meinersbur

Differential Revision: https://reviews.llvm.org/D101286

3 files changed, 48 insertions, 8 deletions

diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 2a6987848ca5..cd086328ec68 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1938,7 +1938,18 @@ void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI,
       if (blockNeedsPredication(ST->getParent(), TheLoop, DT))
         Loc.AATags.TBAA = nullptr;
 
-      Accesses.addStore(Loc);
+      // SCEV does not look through non-header PHIs inside the loop. Such phis
+      // can be analyzed by adding separate accesses for each incoming pointer
+      // value.
+      auto *PN = dyn_cast<PHINode>(Loc.Ptr);
+      if (PN && TheLoop->contains(PN->getParent()) &&
+          PN->getParent() != TheLoop->getHeader()) {
+        for (const Use &Inc : PN->incoming_values()) {
+          MemoryLocation NewLoc = Loc.getWithNewPtr(Inc);
+          Accesses.addStore(NewLoc);
+        }
+      } else
+        Accesses.addStore(Loc);
     }
   }
 
@@ -1982,7 +1993,17 @@ void LoopAccessInfo::analyzeLoop(AAResults *AA, LoopInfo *LI,
     if (blockNeedsPredication(LD->getParent(), TheLoop, DT))
       Loc.AATags.TBAA = nullptr;
 
-    Accesses.addLoad(Loc, IsReadOnlyPtr);
+    // SCEV does not look through non-header PHIs inside the loop. Such phis can
+    // be analyzed by adding separate accesses for each incoming pointer value.
+    auto *PN = dyn_cast<PHINode>(Loc.Ptr);
+    if (PN && TheLoop->contains(PN->getParent()) &&
+        PN->getParent() != TheLoop->getHeader()) {
+      for (const Use &Inc : PN->incoming_values()) {
+        MemoryLocation NewLoc = Loc.getWithNewPtr(Inc);
+        Accesses.addLoad(NewLoc, IsReadOnlyPtr);
+      }
+    } else
+      Accesses.addLoad(Loc, IsReadOnlyPtr);
   }
 
   // If we write (or read-write) to a single destination and there are no
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/pointer-phis.ll b/llvm/test/Analysis/LoopAccessAnalysis/pointer-phis.ll
index c803825896e9..dcdf6dd6a1fc 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/pointer-phis.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/pointer-phis.ll
@@ -6,7 +6,7 @@
 define i32 @load_with_pointer_phi_no_runtime_checks(%s1* %data) {
 ; CHECK-LABEL: load_with_pointer_phi_no_runtime_checks
 ; CHECK-NEXT:  loop.header:
-; CHECK-NEXT:    Report: cannot identify array bounds
+; CHECK-NEXT:    Memory dependences are safe
 ;
 entry:
   br label %loop.header
@@ -41,7 +41,7 @@ exit:                                             ; preds = %loop.latch
 define i32 @store_with_pointer_phi_no_runtime_checks(%s1* %data) {
 ; CHECK-LABEL: 'store_with_pointer_phi_no_runtime_checks'
 ; CHECK-NEXT:  loop.header:
-; CHECK-NEXT:    Report: cannot identify array bounds
+; CHECK-NEXT:    Memory dependences are safe
 ;
 entry:
   br label %loop.header
@@ -76,7 +76,23 @@ exit:                                             ; preds = %loop.latch
 define i32 @store_with_pointer_phi_runtime_checks(double* %A, double* %B, double* %C) {
 ; CHECK-LABEL: 'store_with_pointer_phi_runtime_checks'
 ; CHECK-NEXT:  loop.header:
-; CHECK-NEXT:    Report: cannot identify array bounds
+; CHECK-NEXT:    Memory dependences are safe with run-time checks
+; CHECK:         Run-time memory checks:
+; CHECK-NEXT:    Check 0:
+; CHECK-NEXT:      Comparing group ([[GROUP_C:.+]]):
+; CHECK-NEXT:        %gep.2 = getelementptr inbounds double, double* %C, i64 %iv
+; CHECK-NEXT:      Against group ([[GROUP_B:.+]]):
+; CHECK-NEXT:        %gep.1 = getelementptr inbounds double, double* %B, i64 %iv
+; CHECK-NEXT:    Check 1:
+; CHECK-NEXT:      Comparing group ([[GROUP_C]]):
+; CHECK-NEXT:        %gep.2 = getelementptr inbounds double, double* %C, i64 %iv
+; CHECK-NEXT:      Against group ([[GROUP_A:.+]]):
+; CHECK-NEXT:        %arrayidx = getelementptr inbounds double, double* %A, i64 %iv
+; CHECK-NEXT:    Check 2:
+; CHECK-NEXT:      Comparing group ([[GROUP_B]]):
+; CHECK-NEXT:        %gep.1 = getelementptr inbounds double, double* %B, i64 %iv
+; CHECK-NEXT:      Against group ([[GROUP_A]]):
+; CHECK-NEXT:        %arrayidx = getelementptr inbounds double, double* %A, i64 %iv
 ;
 entry:
   br label %loop.header
diff --git a/llvm/test/Transforms/LoopVectorize/vectorize-pointer-phis.ll b/llvm/test/Transforms/LoopVectorize/vectorize-pointer-phis.ll
index 53437e3d8c57..e3168d5b5d35 100644
--- a/llvm/test/Transforms/LoopVectorize/vectorize-pointer-phis.ll
+++ b/llvm/test/Transforms/LoopVectorize/vectorize-pointer-phis.ll
@@ -4,7 +4,8 @@
 
 define i32 @load_with_pointer_phi_no_runtime_checks(%s1* %data) {
 ; CHECK-LABEL: @load_with_pointer_phi_no_runtime_checks
-; CHECK-NOT: vector.body
+; CHECK-NOT: memcheck
+; CHECK:     vector.body:
 ;
 entry:
   br label %loop.header
@@ -38,7 +39,8 @@ exit:                                             ; preds = %loop.latch
 
 define i32 @store_with_pointer_phi_no_runtime_checks(%s1* %data) {
 ; CHECK-LABEL: @store_with_pointer_phi_no_runtime_checks
-; CHECK-NOT: vector.body
+; CHECK-NOT: memcheck
+; CHECK:     vector.body
 ;
 entry:
   br label %loop.header
@@ -72,7 +74,8 @@ exit:                                             ; preds = %loop.latch
 
 define i32 @store_with_pointer_phi_runtime_checks(double* %A, double* %B, double* %C) {
 ; CHECK-LABEL: @store_with_pointer_phi_runtime_checks
-; CHECK-NOT: vector.body
+; CHECK:     memcheck
+; CHECK:     vector.body
 ;
 entry:
   br label %loop.header