[MemCpyOpt] Teach memcpyopt to handle loads from the constant memory.

[hliao]

Summary

• Loads from the constant memory (either explicit one or as the source of memory transfer intrinsics) won't alias any stores.

[efriedma]

Makes sense.

llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp

1571

Is it possible to write this using getModRefInfo()? Something like if (isModSet(AA->getModRefInfo(M, MemoryLocation::getForSource(M)))).

llvm/test/Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll

21

Please don't use undef here, or in the other tests. It might lead to confusing results in the future.

[hliao]

Check isModSet instead.

[hliao]

llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp

1571

Good point!

[efriedma]

LGTM

llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp

1176–1179

I think the original code was trying to do something slightly different... but I guess there isn't any test coverage? Not completely sure how the difference could matter, anyway; maybe we could do something funny with pointer alignment in certain edge cases.

[asbirlea]

lgtm.

[hliao]

llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp

1176–1179

not regression with all targets enabled, I will check any regression due to this change.

llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp

if (LI->isSimple() && LI->hasOneUse() &&
             P = nullptr;
         }
 
         // If a valid insertion position is found, then we can promote
         // the load/store pair to a memcpy.
         if (P) {
           // If we load from memory that may alias the memory we store to,
           // memmove must be used to preserve semantic. If not, memcpy can
-          // be used.
+          // be used. Also, if we load from constant memory, memcpy can be used
+          // as the constant memory won't be modified.
           bool UseMemMove = false;
-          if (!AA->isNoAlias(MemoryLocation::get(SI), LoadLoc))
+          if (!AA->isNoAlias(MemoryLocation::get(SI), LoadLoc) &&
+              !AA->pointsToConstantMemory(LoadLoc))
             UseMemMove = true;
 
           uint64_t Size = DL.getTypeStoreSize(T);
 
           IRBuilder<> Builder(P);
           Instruction *M;
           if (UseMemMove)
             M = Builder.CreateMemMove(
if (EnableMemorySSA) {
     MemDepResult SourceDep =
         MD->getPointerDependencyFrom(MemoryLocation::getForSource(MDep), false,
                                      M->getIterator(), M->getParent());
     if (!SourceDep.isClobber() || SourceDep.getInst() != MDep)
       return false;
   }
 
   // If the dest of the second might alias the source of the first, then the
-  // source and dest might overlap.  We still want to eliminate the intermediate
-  // value, but we have to generate a memmove instead of memcpy.
+  // source and dest might overlap. In addition, if the source of the first
+  // points to constant memory, they won't overlap by definition. Otherwise, we
+  // still want to eliminate the intermediate value, but we have to generate a
+  // memmove instead of memcpy.
   bool UseMemMove = false;
   if (!AA->isNoAlias(MemoryLocation::getForDest(M),
-                     MemoryLocation::getForSource(MDep)))
+                     MemoryLocation::getForSource(MDep)) &&
+      !AA->pointsToConstantMemory(MemoryLocation::getForSource(MDep)))
     UseMemMove = true;
 
   // If all checks passed, then we can transform M.
   LLVM_DEBUG(dbgs() << "MemCpyOptPass: Forwarding memcpy->memcpy src:\n"
                     << *MDep << '\n' << *M << '\n');
 
   // TODO: Is this worth it if we're creating a less aligned memcpy? For
   // example we could be moving from movaps -> movq on x86.

 bool MemCpyOptPass::processMemSetMemCpyDependence(MemCpyInst *MemCpy,
                                                   MemSetInst *MemSet) {
   // We can only transform memset/memcpy with the same destination.
   if (!AA->isMustAlias(MemSet->getDest(), MemCpy->getDest()))
     return false;
 
   // Check that src and dst of the memcpy aren't the same. While memcpy
   // operands cannot partially overlap, exact equality is allowed.
-  if (!AA->isNoAlias(MemoryLocation(MemCpy->getSource(),
-                                    LocationSize::precise(1)),
-                     MemoryLocation(MemCpy->getDest(),
-                                    LocationSize::precise(1))))
+  if (!AA->isNoAlias(
+          MemoryLocation(MemCpy->getSource(), LocationSize::precise(1)),
+          MemoryLocation(MemCpy->getDest(), LocationSize::precise(1))) &&
+      !AA->pointsToConstantMemory(MemoryLocation::getForSource(MemCpy)))
     return false;
 
   if (EnableMemorySSA) {
     // We know that dst up to src_size is not written. We now need to make sure
     // that dst up to dst_size is not accessed. (If we did not move the memset,
     // checking for reads would be sufficient.)
     if (accessedBetween(*AA, MemoryLocation::getForDest(MemSet),
                         MSSA->getMemoryAccess(MemSet),
bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
   }
 
   return false;
 }
 
 /// Transforms memmove calls to memcpy calls when the src/dst are guaranteed
 /// not to alias.
 bool MemCpyOptPass::processMemMove(MemMoveInst *M) {
-  // See if the pointers alias.
+  // See if the pointers alias or the source points to the constant memory.
   if (!AA->isNoAlias(MemoryLocation::getForDest(M),
-                     MemoryLocation::getForSource(M)))
+                     MemoryLocation::getForSource(M)) &&
+      !AA->pointsToConstantMemory(MemoryLocation::getForSource(M)))
     return false;
 
   LLVM_DEBUG(dbgs() << "MemCpyOptPass: Optimizing memmove -> memcpy: " << *M
                     << "\n");
 
   // If not, then we know we can transform this.
   Type *ArgTys[3] = { M->getRawDest()->getType(),
                       M->getRawSource()->getType(),

llvm/test/Transforms/MemCpyOpt/load-store-to-memcpy.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -basic-aa -scoped-noalias-aa -memcpyopt -S %s -enable-memcpyopt-memoryssa=0 | FileCheck %s
 ; RUN: opt -basic-aa -scoped-noalias-aa -memcpyopt -S %s -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s
 
 %T = type { i8, i32 }
 
+; A global constant of %T
+@C = external constant %T
+
 ; Ensure load-store forwarding of an aggregate is interpreted as
 ; a memmove when the source and dest may alias
 define void @test_memmove(%T* align 8 %a, %T* align 16 %b) {
 ; CHECK-LABEL: @test_memmove(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast %T* [[B:%.*]] to i8*
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %T* [[A:%.*]] to i8*
 ; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* align 16 [[TMP1]], i8* align 8 [[TMP2]], i64 8, i1 false)
 ; CHECK-NEXT:    ret void

 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 [[TMP1]], i8* align 8 [[TMP2]], i64 8, i1 false)
 ; CHECK-NEXT:    ret void
 ;
   %val = load %T, %T* %a, align 8
   store %T %val, %T* %b, align 16
   ret void
 }
 
+define void @test_memcpy_constant(%T* %d) {
+; CHECK-LABEL: @test_memcpy_constant(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast %T* [[D:%.*]] to i8*
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 [[TMP1]], i8* align 8 getelementptr inbounds ([[T:%.*]], %T* @C, i32 0, i32 0), i64 8, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %val = load %T, %T* @C, align 8
+  store %T %val, %T* %d, align 16
+  ret void
+}
+
 ; memcpy(%d, %a) should not be generated since store2 may-aliases load %a.
 define void @f(%T* %a, %T* %b, %T* %c, %T* %d) {
 ; CHECK-LABEL: @f(
 ; CHECK-NEXT:    [[VAL:%.*]] = load [[T:%.*]], %T* [[A:%.*]], align 4, !alias.scope !0
 ; CHECK-NEXT:    store [[T]] { i8 23, i32 23 }, %T* [[B:%.*]], align 4, !alias.scope !3
 ; CHECK-NEXT:    store [[T]] { i8 44, i32 44 }, %T* [[C:%.*]], align 4, !alias.scope !6, !noalias !3
 ; CHECK-NEXT:    store [[T]] [[VAL]], %T* [[D:%.*]], align 4, !alias.scope !9, !noalias !12
 ; CHECK-NEXT:    ret void

llvm/test/Transforms/MemCpyOpt/memcpy.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basic-aa -memcpyopt -dse -S -enable-memcpyopt-memoryssa=0 | FileCheck %s
 ; RUN: opt < %s -basic-aa -memcpyopt -dse -S -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i686-apple-darwin9"
 
 %0 = type { x86_fp80, x86_fp80 }
 %1 = type { i32, i32 }
 
+@C = external constant [0 x i8]
+
 declare void @llvm.memcpy.p1i8.p0i8.i64(i8 addrspace(1)* nocapture, i8* nocapture, i64, i1) nounwind
 declare void @llvm.memcpy.p0i8.p1i8.i64(i8* nocapture, i8 addrspace(1)* nocapture, i64, i1) nounwind
 declare void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* nocapture, i8 addrspace(1)* nocapture, i64, i1) nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
 declare void @llvm.memcpy.inline.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i1) nounwind
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
 
 define void @test1(%0* sret(%0)  %agg.result, x86_fp80 %z.0, x86_fp80 %z.1) nounwind  {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP2:%.*]] = alloca [[TMP0:%.*]], align 16
 ; CHECK-NEXT:    [[MEMTMP:%.*]] = alloca [[TMP0]], align 16
 ; CHECK-NEXT:    [[TMP5:%.*]] = fsub x86_fp80 0xK80000000000000000000, [[Z_1:%.*]]
-; CHECK-NEXT:    call void @ccoshl(%0* sret([[TMP0]]) [[TMP2]], x86_fp80 [[TMP5]], x86_fp80 [[Z_0:%.*]]) #[[ATTR0:[0-9]+]]
+; CHECK-NEXT:    call void @ccoshl(%0* sret([[TMP0]]) [[TMP2]], x86_fp80 [[TMP5]], x86_fp80 [[Z_0:%.*]]) #[[ATTR2:[0-9]+]]
 ; CHECK-NEXT:    [[TMP219:%.*]] = bitcast %0* [[TMP2]] to i8*
 ; CHECK-NEXT:    [[MEMTMP20:%.*]] = bitcast %0* [[MEMTMP]] to i8*
 ; CHECK-NEXT:    [[AGG_RESULT21:%.*]] = bitcast %0* [[AGG_RESULT:%.*]] to i8*
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 [[AGG_RESULT21]], i8* align 16 [[TMP219]], i32 32, i1 false)
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %tmp2 = alloca %0
;
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %R, i8* align 16 %P, i32 32, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %Q, i8* align 16 %R, i32 32, i1 false)
   ret void
 
 }
 
 ; The intermediate alloca and one of the memcpy's should be eliminated, the
 ; other should be related with a memcpy.
+define void @test2_constant(i8* %Q) nounwind  {
+; CHECK-LABEL: @test2_constant(
+; CHECK-NEXT:    [[P:%.*]] = getelementptr inbounds [0 x i8], [0 x i8]* @C, i64 0, i64 undef
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 [[Q:%.*]], i8* align 16 [[P]], i32 32, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %memtmp = alloca %0, align 16
+  %R = bitcast %0* %memtmp to i8*
+  %P = getelementptr inbounds [0 x i8], [0 x i8]* @C, i64 0, i64 undef
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %R, i8* align 16 %P, i32 32, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %Q, i8* align 16 %R, i32 32, i1 false)
+  ret void
+
+}
+
+; The intermediate alloca and one of the memcpy's should be eliminated, the
+; other should be related with a memcpy.
 define void @test2_memcpy(i8* noalias %P, i8* noalias %Q) nounwind  {
 ; CHECK-LABEL: @test2_memcpy(
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 [[Q:%.*]], i8* align 16 [[P:%.*]], i32 32, i1 false)
 ; CHECK-NEXT:    ret void
 ;
   %memtmp = alloca %0, align 16
   %R = bitcast %0* %memtmp to i8*
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 %R, i8* align 16 %P, i32 32, i1 false)

 
 ; PR9794 - Should forward memcpy into byval argument even though the memcpy
 ; isn't itself 8 byte aligned.
 %struct.p = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }
 
 define i32 @test7(%struct.p* nocapture align 8 byval(%struct.p) %q) nounwind ssp {
 ; CHECK-LABEL: @test7(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @g(%struct.p* byval([[STRUCT_P:%.*]]) align 8 [[Q:%.*]]) #[[ATTR0]]
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @g(%struct.p* byval([[STRUCT_P:%.*]]) align 8 [[Q:%.*]]) #[[ATTR2]]
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
 entry:
   %agg.tmp = alloca %struct.p, align 4
   %tmp = bitcast %struct.p* %agg.tmp to i8*
   %tmp1 = bitcast %struct.p* %q to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %tmp, i8* align 4 %tmp1, i64 48, i1 false)
   %call = call i32 @g(%struct.p* align 8 byval(%struct.p) %agg.tmp) nounwind

llvm/test/Transforms/MemCpyOpt/memmove.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basic-aa -memcpyopt -S -enable-memcpyopt-memoryssa=0 | FileCheck %s
 ; RUN: opt < %s -basic-aa -memcpyopt -S -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s
 ; These memmoves should get optimized to memcpys.
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 target triple = "x86_64-apple-darwin9.0"
 
+@C = external constant [0 x i8]
+
 declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
 
 define i8* @test1(i8* nocapture %src) nounwind {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[MALLOCCALL:%.*]] = tail call i8* @malloc(i32 trunc (i64 mul nuw (i64 ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i64), i64 13) to i32))
 ; CHECK-NEXT:    [[CALL3:%.*]] = bitcast i8* [[MALLOCCALL]] to [13 x i8]*
 ; CHECK-NEXT:    [[CALL3_SUB:%.*]] = getelementptr inbounds [13 x i8], [13 x i8]* [[CALL3]], i64 0, i64 0

 ; CHECK-NEXT:    tail call void @llvm.memmove.p0i8.p0i8.i64(i8* [[P]], i8* [[ADD_PTR]], i64 17, i1 false)
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %add.ptr = getelementptr i8, i8* %P, i64 16
   tail call void @llvm.memmove.p0i8.p0i8.i64(i8* %P, i8* %add.ptr, i64 17, i1 false)
   ret void
 }
+
+define void @test4(i8* %P) nounwind {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds [0 x i8], [0 x i8]* @C, i64 0, i64 undef
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[P:%.*]], i8* [[ADD_PTR]], i64 17, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %add.ptr = getelementptr inbounds [0 x i8], [0 x i8]* @C, i64 0, i64 undef
+  tail call void @llvm.memmove.p0i8.p0i8.i64(i8* %P, i8* %add.ptr, i64 17, i1 false)
+  ret void
+}

llvm/test/Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -basic-aa -memcpyopt -S %s -enable-memcpyopt-memoryssa=0 | FileCheck %s
 ; RUN: opt -basic-aa -memcpyopt -S %s -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
+@C = external constant [0 x i8]
+
+define void @test_constant(i64 %src_size, i8* %dst, i64 %dst_size, i8 %c) {
+; CHECK-LABEL: @test_constant(
+; CHECK-NEXT:    [[SRC:%.*]] = getelementptr inbounds [0 x i8], [0 x i8]* @C, i64 0, i64 undef
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i64 [[DST_SIZE:%.*]], [[SRC_SIZE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[DST_SIZE]], [[SRC_SIZE]]
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i64 0, i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, i8* [[DST:%.*]], i64 [[SRC_SIZE]]
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 1 [[TMP4]], i8 [[C:%.*]], i64 [[TMP3]], i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[DST]], i8* [[SRC]], i64 [[SRC_SIZE]], i1 false)
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.memset.p0i8.i64(i8* %dst, i8 %c, i64 %dst_size, i1 false)
+  %src = getelementptr inbounds [0 x i8], [0 x i8]* @C, i64 0, i64 undef
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i1 false)
+  ret void
+}
+
 define void @test(i8* %src, i64 %src_size, i8* noalias %dst, i64 %dst_size, i8 %c) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i64 [[DST_SIZE:%.*]], [[SRC_SIZE:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[DST_SIZE]], [[SRC_SIZE]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i64 0, i64 [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, i8* [[DST:%.*]], i64 [[SRC_SIZE]]
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 1 [[TMP4]], i8 [[C:%.*]], i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[DST]], i8* [[SRC:%.*]], i64 [[SRC_SIZE]], i1 false)