[MemCpyOpt] Don't fold memcpy.inline into memmove

[miyuki]

Summary

The llvm.memcpy.inline intrinsic must be expanded into code that
does not contain any function calls because it is intended for
the implementation of low-level functions like memcpy. Currently the
MemCpyOpt might covert llvm.memcpy.inline into llvm.memmove in
certain circumstances. This patch fixes the issue.

Fixes https://github.com/llvm/llvm-project/issues/61791.

[tschuett]

@gchatelet

[nikic]

LGTM

[miyuki]

CI failures look unrelated.

[gchatelet]

Thx a lot for the fix!

llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp

if (writtenBetween(MSSA, BAA, MemoryLocation::getForSource(MDep),
     return false;
 
   // If the dest of the second might alias the source of the first, then the
   // source and dest might overlap. In addition, if the source of the first
   // points to constant memory, they won't overlap by definition. Otherwise, we
   // still want to eliminate the intermediate value, but we have to generate a
   // memmove instead of memcpy.
   bool UseMemMove = false;
-  if (isModSet(BAA.getModRefInfo(M, MemoryLocation::getForSource(MDep))))
+  if (isModSet(BAA.getModRefInfo(M, MemoryLocation::getForSource(MDep)))) {
+    // Don't convert llvm.memcpy.inline into memmove because memmove can be
+    // lowered as a call, and that is not allowed for llvm.memcpy.inline (and
+    // there is no inline version of llvm.memmove)
+    if (isa<MemCpyInlineInst>(M))
+      return false;
     UseMemMove = true;
+  }
 
   // If all checks passed, then we can transform M.
   LLVM_DEBUG(dbgs() << "MemCpyOptPass: Forwarding memcpy->memcpy src:\n"
                     << *MDep << '\n' << *M << '\n');
 
   // TODO: Is this worth it if we're creating a less aligned memcpy? For
   // example we could be moving from movaps -> movq on x86.
   IRBuilder<> Builder(M);

llvm/test/Transforms/MemCpyOpt/memcpy.ll

 ;
   %memtmp = alloca %0, align 16
   call void @llvm.memcpy.inline.p0.p0.i32(ptr align 16 %memtmp, ptr align 16 %P, i32 32, i1 false)
   call void @llvm.memcpy.inline.p0.p0.i32(ptr align 16 %Q, ptr align 16 %memtmp, i32 32, i1 false)
   ret void
 
 }
 
+; Similar to test5_memcpy, but without noalias; check that memcpy.inline is not folded into memmove.
+define void @test6_memcpy(ptr %src, ptr %dest) nounwind {
+; CHECK-LABEL: @test6_memcpy(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca [16 x i8], align 1
+; CHECK-NEXT:    call void @llvm.memcpy.inline.p0.p0.i32(ptr align 1 [[TMP]], ptr align 1 [[DEST:%.*]], i32 16, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.inline.p0.p0.i32(ptr align 1 [[DEST]], ptr align 1 [[TMP]], i32 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %tmp = alloca [16 x i8], align 1
+  call void @llvm.memcpy.inline.p0.p0.i32(ptr align 1 %tmp, ptr align 1 %dest, i32 16, i1 false)
+  call void @llvm.memcpy.inline.p0.p0.i32(ptr align 1 %dest, ptr align 1 %tmp, i32 16, i1 false)
+  ret void
+}
+
 
 @x = external global %0
 
 define void @test3(ptr noalias sret(%0) %agg.result) nounwind  {
 ; CHECK-LABEL: @test3(
 ; CHECK-NEXT:    [[X_0:%.*]] = alloca [[TMP0:%.*]], align 16
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 16 [[AGG_RESULT:%.*]], ptr align 16 @x, i32 32, i1 false)
 ; CHECK-NEXT:    ret void