[LSV] Vectorize up to side-effecting instructions.

[jlebar]

Summary

Previously if we had a chain that contained a side-effecting
instruction, we wouldn't vectorize it at all. Now we'll vectorize
everything that comes before the side-effecting instruction.

[asbirlea]

lib/Transforms/Vectorize/LoadStoreVectorizer.cpp

866 ↗

(On Diff #64565)

Remove comment?

[asbirlea]

LGTM.

[jlebar]

Fix comment.

llvm/trunk/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp

for (Instruction &I : make_range(getBoundaryInstrs(Chain))) {
     ++InstrIdx;
     if (isa<LoadInst>(I) || isa<StoreInst>(I)) {
       if (!is_contained(Chain, &I))
         MemoryInstrs.push_back({&I, InstrIdx});
       else
         ChainInstrs.push_back({&I, InstrIdx});
     } else if (I.mayHaveSideEffects()) {
       DEBUG(dbgs() << "LSV: Found side-effecting operation: " << I << '\n');
-      return 0;
+      break;
     }
   }
 
-  assert(Chain.size() == ChainInstrs.size() &&
-         "All instrs in Chain must be within range getBoundaryInstrs(Chain).");
-
   // Loop until we find an instruction in ChainInstrs that we can't vectorize.
   unsigned ChainInstrIdx, ChainInstrsLen;
   for (ChainInstrIdx = 0, ChainInstrsLen = ChainInstrs.size();
        ChainInstrIdx < ChainInstrsLen; ++ChainInstrIdx) {
     Value *ChainInstr = ChainInstrs[ChainInstrIdx].first;
     unsigned ChainInstrLoc = ChainInstrs[ChainInstrIdx].second;
     bool AliasFound = false;
     for (auto EntryMem : MemoryInstrs) {
for (auto EntryMem : MemoryInstrs) {
 
       Instruction *M0 = cast<Instruction>(MemInstr);
       Instruction *M1 = cast<Instruction>(ChainInstr);
 
       if (!AA.isNoAlias(MemoryLocation::get(M0), MemoryLocation::get(M1))) {
         DEBUG({
           Value *Ptr0 = getPointerOperand(M0);
           Value *Ptr1 = getPointerOperand(M1);
-
           dbgs() << "LSV: Found alias:\n"
                     "  Aliasing instruction and pointer:\n"
                  << "  " << *MemInstr << '\n'
                  << "  " << *Ptr0 << '\n'
                  << "  Aliased instruction and pointer:\n"
                  << "  " << *ChainInstr << '\n'
                  << "  " << *Ptr1 << '\n';
         });
bool Vectorizer::vectorizeStoreChain(
 
   if (!isPowerOf2_32(Sz) || VF < 2 || ChainSize < 2) {
     InstructionsProcessed->insert(Chain.begin(), Chain.end());
     return false;
   }
 
   ArrayRef<Value *> NewChain = getVectorizablePrefix(Chain);
   if (NewChain.empty()) {
-    // There exists a side effect instruction, no vectorization possible.
+    // No vectorization possible.
     InstructionsProcessed->insert(Chain.begin(), Chain.end());
     return false;
   }
   if (NewChain.size() == 1) {
     // Failed after the first instruction. Discard it and try the smaller chain.
     InstructionsProcessed->insert(NewChain.front());
     return false;
   }
bool Vectorizer::vectorizeLoadChain(
 
   if (!isPowerOf2_32(Sz) || VF < 2 || ChainSize < 2) {
     InstructionsProcessed->insert(Chain.begin(), Chain.end());
     return false;
   }
 
   ArrayRef<Value *> NewChain = getVectorizablePrefix(Chain);
   if (NewChain.empty()) {
-    // There exists a side effect instruction, no vectorization possible.
+    // No vectorization possible.
     InstructionsProcessed->insert(Chain.begin(), Chain.end());
     return false;
   }
   if (NewChain.size() == 1) {
     // Failed after the first instruction. Discard it and try the smaller chain.
     InstructionsProcessed->insert(NewChain.front());
     return false;
   }

llvm/trunk/test/Transforms/LoadStoreVectorizer/NVPTX/lit.local.cfg

+if not 'NVPTX' in config.root.targets:
+    config.unsupported = True
+

llvm/trunk/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll

+; RUN: opt -mtriple=nvptx64-nvidia-cuda -load-store-vectorizer -S -o - %s | FileCheck %s
+
+; If we have a chain of loads or stores with a side-effecting operation in the
+; middle, we should still be able to merge the loads/stores that appear
+; before/after the side-effecting op.  We just can't merge *across* the
+; side-effecting op.
+
+declare void @fn() #0
+
+; CHECK-LABEL: @merge_stores
+; CHECK: store <2 x i32> <i32 100, i32 101>
+; CHECK: call void @fn()
+; CHECK: store <2 x i32> <i32 102, i32 103>
+define void @merge_stores(i32* %out) #0 {
+  %out.gep.1 = getelementptr i32, i32* %out, i32 1
+  %out.gep.2 = getelementptr i32, i32* %out, i32 2
+  %out.gep.3 = getelementptr i32, i32* %out, i32 3
+
+  store i32 101, i32* %out.gep.1
+  store i32 100, i32* %out
+  call void @fn()
+  store i32 102, i32* %out.gep.2
+  store i32 103, i32* %out.gep.3
+  ret void
+}
+
+; CHECK-LABEL: @merge_loads
+; CHECK: load <2 x i32>
+; CHECK: call void @fn()
+; CHECK: load <2 x i32>
+define i32 @merge_loads(i32* %in) #0 {
+  %in.gep.1 = getelementptr i32, i32* %in, i32 1
+  %in.gep.2 = getelementptr i32, i32* %in, i32 2
+  %in.gep.3 = getelementptr i32, i32* %in, i32 3
+
+  %v1 = load i32, i32* %in
+  %v2 = load i32, i32* %in.gep.1
+  call void @fn()
+  %v3 = load i32, i32* %in.gep.2
+  %v4 = load i32, i32* %in.gep.3
+
+  %sum1 = add i32 %v1, %v2
+  %sum2 = add i32 %sum1, %v3
+  %sum3 = add i32 %sum2, %v4
+  ret i32 %v4
+}
+
+attributes #0 = { nounwind }