Actually support volatile memcpys in NVPTX lowering

[jingyue]

LGTM

More of a feature request, llvm.memcpy now takes alignments as arguments. We should emit larger loads/stores (e.g. st.u32 or even vector store) when alignments permit. Worth a TODO?

[eliben]

In D11091#202486, @jingyue wrote:

LGTM

More of a feature request, llvm.memcpy now takes alignments as arguments. We should emit larger loads/stores (e.g. st.u32 or even vector store) when alignments permit. Worth a TODO?

Yes, will do.

llvm/trunk/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp

 };
 } // namespace
 
 char NVPTXLowerAggrCopies::ID = 0;
 
 // Lower MemTransferInst or load-store pair to loop
 static void convertTransferToLoop(
     Instruction *splitAt, Value *srcAddr, Value *dstAddr, Value *len,
-    //unsigned numLoads,
     bool srcVolatile, bool dstVolatile, LLVMContext &Context, Function &F) {
   Type *indType = len->getType();
 
   BasicBlock *origBB = splitAt->getParent();
   BasicBlock *newBB = splitAt->getParent()->splitBasicBlock(splitAt, "split");
   BasicBlock *loopBB = BasicBlock::Create(Context, "loadstoreloop", &F, newBB);
 
   origBB->getTerminator()->setSuccessor(0, loopBB);
static void convertTransferToLoop(
 
   IRBuilder<> loop(loopBB);
   // The loop index (ind) is a phi node.
   PHINode *ind = loop.CreatePHI(indType, 0);
   // Incoming value for ind is 0
   ind->addIncoming(ConstantInt::get(indType, 0), origBB);
 
   // load from srcAddr+ind
+  // TODO: we can leverage the align parameter of llvm.memcpy for more efficient
+  // word-sized loads and stores.
   Value *val = loop.CreateLoad(loop.CreateGEP(loop.getInt8Ty(), srcAddr, ind),
                                srcVolatile);
   // store at dstAddr+ind
   loop.CreateStore(val, loop.CreateGEP(loop.getInt8Ty(), dstAddr, ind),
                    dstVolatile);
 
   // The value for ind coming from backedge is (ind + 1)
   Value *newind = loop.CreateAdd(ind, ConstantInt::get(indType, 1));
for (LoadInst *load : aggrLoads) {
     convertTransferToLoop(store, srcAddr, dstAddr, len, load->isVolatile(),
                           store->isVolatile(), Context, F);
 
     store->eraseFromParent();
     load->eraseFromParent();
   }
 
   for (MemTransferInst *cpy : aggrMemcpys) {
-    Value *len = cpy->getLength();
-    // llvm 2.7 version of memcpy does not have volatile
-    // operand yet. So always making it non-volatile
-    // optimistically, so that we don't see unnecessary
-    // st.volatile in ptx
-    convertTransferToLoop(cpy, cpy->getSource(), cpy->getDest(), len, false,
-                          false, Context, F);
+    convertTransferToLoop(/* splitAt */ cpy,
+                          /* srcAddr */ cpy->getSource(),
+                          /* dstAddr */ cpy->getDest(),
+                          /* len */ cpy->getLength(),
+                          /* srcVolatile */ cpy->isVolatile(),
+                          /* dstVolatile */ cpy->isVolatile(),
+                          /* Context */ Context,
+                          /* Function F */ F);
     cpy->eraseFromParent();
   }
 
   for (MemSetInst *memsetinst : aggrMemsets) {
     Value *len = memsetinst->getLength();
     Value *val = memsetinst->getValue();
     convertMemSetToLoop(memsetinst, memsetinst->getDest(), len, val, Context,
                         F);

llvm/trunk/test/CodeGen/NVPTX/lower-aggr-copies.ll

 ; CHECK: LBB[[LABEL:[_0-9]+]]:
 ; CHECK:      ld.u8 %rs[[REG:[0-9]+]]
 ; CHECK:      st.u8 [%r{{[0-9]+}}], %rs[[REG]]
 ; CHECK:      add.s64 %rd[[COUNTER:[0-9]+]], %rd[[COUNTER]], 1
 ; CHECK-NEXT: setp.lt.u64 %p[[PRED:[0-9]+]], %rd[[COUNTER]], %rd
 ; CHECK-NEXT: @%p[[PRED]] bra LBB[[LABEL]]
 }
 
+define i8* @memcpy_volatile_caller(i8* %dst, i8* %src, i64 %n) #0 {
+entry:
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %n, i32 1, i1 true)
+  ret i8* %dst
+; CHECK-LABEL: .visible .func (.param .b32 func_retval0) memcpy_volatile_caller
+; CHECK: LBB[[LABEL:[_0-9]+]]:
+; CHECK:      ld.volatile.u8 %rs[[REG:[0-9]+]]
+; CHECK:      st.volatile.u8 [%r{{[0-9]+}}], %rs[[REG]]
+; CHECK:      add.s64 %rd[[COUNTER:[0-9]+]], %rd[[COUNTER]], 1
+; CHECK-NEXT: setp.lt.u64 %p[[PRED:[0-9]+]], %rd[[COUNTER]], %rd
+; CHECK-NEXT: @%p[[PRED]] bra LBB[[LABEL]]
+}
+
 define i8* @memset_caller(i8* %dst, i32 %c, i64 %n) #0 {
 entry:
   %0 = trunc i32 %c to i8
   tail call void @llvm.memset.p0i8.i64(i8* %dst, i8 %0, i64 %n, i32 1, i1 false)
   ret i8* %dst
 ; CHECK-LABEL: .visible .func (.param .b32 func_retval0) memset_caller(
 ; CHECK:      ld.param.u8 %rs[[REG:[0-9]+]]
 ; CHECK:      LBB[[LABEL:[_0-9]+]]:
 ; CHECK:      st.u8 [%r{{[0-9]+}}], %rs[[REG]]
 ; CHECK:      add.s64 %rd[[COUNTER:[0-9]+]], %rd[[COUNTER]], 1
 ; CHECK-NEXT: setp.lt.u64 %p[[PRED:[0-9]+]], %rd[[COUNTER]], %rd
 ; CHECK-NEXT: @%p[[PRED]] bra LBB[[LABEL]]
 }