[LoopIdiomRecognize] Add support for atomic memmove

[EricWF]

Summary

This patch builds on D42889 by adding support for unordered atomic memmove.

[chandlerc]

Just marking that this would need rebasing on a corrected form of the underlying change. That way it doesn't show up in my queue to review.

include/llvm/IR/IRBuilder.h

return CreateMemCpy(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
                         NoAliasTag);
   }
 
   CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src,
                          unsigned SrcAlign, Value *Size,
                          bool isVolatile = false, MDNode *TBAATag = nullptr,
                          MDNode *TBAAStructTag = nullptr,
                          MDNode *ScopeTag = nullptr,
-                         MDNode *NoAliasTag = nullptr);
+                         MDNode *NoAliasTag = nullptr) {
+    return CreateMemCpyOrMemMove(Intrinsic::memcpy, Dst, DstAlign, Src,
+                                 SrcAlign, Size, isVolatile, TBAATag,
+                                 TBAAStructTag, ScopeTag, NoAliasTag);
+  }
 
   // TODO: Old API. Remove this when no longer used.
   CallInst *CreateMemCpy(Value *Dst, Value *Src, uint64_t Size, unsigned Align,
                          bool isVolatile = false, MDNode *TBAATag = nullptr,
                          MDNode *TBAAStructTag = nullptr,
                          MDNode *ScopeTag = nullptr,
                          MDNode *NoAliasTag = nullptr) {
     return CreateMemCpy(Dst, Align, Src, Align, getInt64(Size), isVolatile, TBAATag,
return CreateElementUnorderedAtomicMemCpy(
         Dst, DstAlign, Src, SrcAlign, getInt64(Size), ElementSize, TBAATag,
         TBAAStructTag, ScopeTag, NoAliasTag);
   }
 
   CallInst *CreateElementUnorderedAtomicMemCpy(
       Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
       uint32_t ElementSize, MDNode *TBAATag = nullptr,
       MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
-      MDNode *NoAliasTag = nullptr);
+      MDNode *NoAliasTag = nullptr) {
+    return CreateElementUnorderedAtomicMemCpyOrMemMove(
+        Intrinsic::memcpy_element_unordered_atomic, Dst, DstAlign, Src,
+        SrcAlign, Size, ElementSize, TBAATag, TBAAStructTag, ScopeTag,
+        NoAliasTag);
+  }
 
   /// \brief Create and insert a memmove between the specified
   /// pointers.
   ///
   /// If the pointers aren't i8*, they will be converted.  If a TBAA tag is
   /// specified, it will be added to the instruction. Likewise with alias.scope
   /// and noalias tags.
   CallInst *CreateMemMove(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
                           uint64_t Size, bool isVolatile = false,
                           MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr,
                           MDNode *NoAliasTag = nullptr) {
     return CreateMemMove(Dst, DstAlign, Src, SrcAlign, getInt64(Size), isVolatile,
                          TBAATag, ScopeTag, NoAliasTag);
   }
 
-  CallInst *CreateMemMove(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
-                          Value *Size, bool isVolatile = false, MDNode *TBAATag = nullptr,
+  CallInst *CreateMemMove(Value *Dst, unsigned DstAlign, Value *Src,
+                          unsigned SrcAlign, Value *Size,
+                          bool isVolatile = false, MDNode *TBAATag = nullptr,
                           MDNode *ScopeTag = nullptr,
-                          MDNode *NoAliasTag = nullptr);
+                          MDNode *NoAliasTag = nullptr) {
+    return CreateMemCpyOrMemMove(Intrinsic::memmove, Dst, DstAlign, Src,
+                                 SrcAlign, Size, isVolatile, TBAATag,
+                                 /*TBAAStructTag*/ nullptr, ScopeTag,
+                                 NoAliasTag);
+  }
 
   // TODO: Old API. Remove this when no longer used.
   CallInst *CreateMemMove(Value *Dst, Value *Src, uint64_t Size, unsigned Align,
                           bool isVolatile = false, MDNode *TBAATag = nullptr,
                           MDNode *ScopeTag = nullptr,
                           MDNode *NoAliasTag = nullptr) {
     return CreateMemMove(Dst, Align, Src, Align, getInt64(Size), isVolatile,
                          TBAATag, ScopeTag, NoAliasTag);
   }
   // TODO: Old API. Remove this when no longer used.
   CallInst *CreateMemMove(Value *Dst, Value *Src, Value *Size, unsigned Align,
                           bool isVolatile = false, MDNode *TBAATag = nullptr,
                           MDNode *ScopeTag = nullptr,
                           MDNode *NoAliasTag = nullptr) {
     return CreateMemMove(Dst, Align, Src, Align, Size, isVolatile, TBAATag,
                          ScopeTag, NoAliasTag);
   }
 
+  /// \brief Create and insert an element unordered-atomic memmove between the
+  /// specified pointers.
+  ///
+  /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers,
+  /// respectively.
+  ///
+  /// If the pointers aren't i8*, they will be converted.  If a TBAA tag is
+  /// specified, it will be added to the instruction. Likewise with alias.scope
+  /// and noalias tags.
+  CallInst *CreateElementUnorderedAtomicMemMove(
+      Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
+      uint64_t Size, uint32_t ElementSize, MDNode *TBAATag = nullptr,
+      MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) {
+    return CreateElementUnorderedAtomicMemMove(Dst, DstAlign, Src, SrcAlign,
+                                               getInt64(Size), ElementSize,
+                                               TBAATag, ScopeTag, NoAliasTag);
+  }
+
+  CallInst *CreateElementUnorderedAtomicMemMove(
+      Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
+      uint32_t ElementSize, MDNode *TBAATag = nullptr,
+      MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) {
+    return CreateElementUnorderedAtomicMemCpyOrMemMove(
+        Intrinsic::memmove_element_unordered_atomic, Dst, DstAlign, Src,
+        SrcAlign, Size, ElementSize, TBAATag, /*TBAAStructTag*/ nullptr,
+        ScopeTag, NoAliasTag);
+  }
+
   /// \brief Create a vector fadd reduction intrinsic of the source vector.
   /// The first parameter is a scalar accumulator value for ordered reductions.
   CallInst *CreateFAddReduce(Value *Acc, Value *Src);
 
   /// \brief Create a vector fmul reduction intrinsic of the source vector.
   /// The first parameter is a scalar accumulator value for ordered reductions.
   CallInst *CreateFMulReduce(Value *Acc, Value *Src);
 
public:
   }
 
 private:
   /// \brief Create a call to a masked intrinsic with given Id.
   CallInst *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops,
                                   ArrayRef<Type *> OverloadedTypes,
                                   const Twine &Name = "");
 
+  CallInst *CreateMemCpyOrMemMove(Intrinsic::ID ID, Value *Dst,
+                                  unsigned DstAlign, Value *Src,
+                                  unsigned SrcAlign, Value *Size,
+                                  bool isVolatile, MDNode *TBAATag,
+                                  MDNode *TBAAStructTag, MDNode *ScopeTag,
+                                  MDNode *NoAliasTag);
+
+  CallInst *CreateElementUnorderedAtomicMemCpyOrMemMove(
+      Intrinsic::ID ID, Value *Dst, unsigned DstAlign, Value *Src,
+      unsigned SrcAlign, Value *Size, uint32_t ElementSize, MDNode *TBAATag,
+      MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag);
+
   Value *getCastedInt8PtrValue(Value *Ptr);
 };
 
 /// \brief This provides a uniform API for creating instructions and inserting
 /// them into a basic block: either at the end of a BasicBlock, or at a specific
 /// iterator location in a block.
 ///
 /// Note that the builder does not expose the full generality of LLVM

lib/IR/IRBuilder.cpp

if (ScopeTag)
     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
  
   if (NoAliasTag)
     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 
   return CI;
 }
 
-CallInst *IRBuilderBase::
-CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
-             Value *Size, bool isVolatile, MDNode *TBAATag,
-             MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
-  assert((DstAlign == 0 || isPowerOf2_32(DstAlign)) && "Must be 0 or a power of 2");
-  assert((SrcAlign == 0 || isPowerOf2_32(SrcAlign)) && "Must be 0 or a power of 2");
+CallInst *IRBuilderBase::CreateMemCpyOrMemMove(
+    Intrinsic::ID ID, Value *Dst, unsigned DstAlign, Value *Src,
+    unsigned SrcAlign, Value *Size, bool isVolatile, MDNode *TBAATag,
+  assert((ID == Intrinsic::memcpy || ID == Intrinsic::memmove) &&
+         "invalid intrinsic ID");
+  assert((DstAlign == 0 || isPowerOf2_32(DstAlign)) &&
+         "Must be 0 or a power of 2");
+  assert((SrcAlign == 0 || isPowerOf2_32(SrcAlign)) &&
+         "Must be 0 or a power of 2");
   Dst = getCastedInt8PtrValue(Dst);
   Src = getCastedInt8PtrValue(Src);
 
   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
-  Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
+  Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
   Module *M = BB->getParent()->getParent();
-  Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
-  
+  Value *TheFn = Intrinsic::getDeclaration(M, ID, Tys);
   CallInst *CI = createCallHelper(TheFn, Ops, this);
 
-  auto* MCI = cast<MemCpyInst>(CI);
+  auto *MTI = cast<MemTransferInst>(CI);
   if (DstAlign > 0)
-    MCI->setDestAlignment(DstAlign);
+    MTI->setDestAlignment(DstAlign);
   if (SrcAlign > 0)
-    MCI->setSourceAlignment(SrcAlign);
+    MTI->setSourceAlignment(SrcAlign);
 
   // Set the TBAA info if present.
   if (TBAATag)
     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 
   // Set the TBAA Struct info if present.
   if (TBAAStructTag)
     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
- 
   if (ScopeTag)
     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
- 
   if (NoAliasTag)
     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 
-  return CI;  
+  return CI;
 }
 
-CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
-    Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
-    uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
-    MDNode *ScopeTag, MDNode *NoAliasTag) {
+CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpyOrMemMove(
+    Intrinsic::ID ID, Value *Dst, unsigned DstAlign, Value *Src,
+    unsigned SrcAlign, Value *Size, uint32_t ElementSize, MDNode *TBAATag,
+    MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
+  assert((ID == Intrinsic::memcpy_element_unordered_atomic ||
+          ID == Intrinsic::memmove_element_unordered_atomic) &&
+         "invalid intrinsic ID");
   assert(DstAlign >= ElementSize &&
          "Pointer alignment must be at least element size");
   assert(SrcAlign >= ElementSize &&
          "Pointer alignment must be at least element size");
   Dst = getCastedInt8PtrValue(Dst);
   Src = getCastedInt8PtrValue(Src);
 
   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
   Module *M = BB->getParent()->getParent();
-  Value *TheFn = Intrinsic::getDeclaration(
-      M, Intrinsic::memcpy_element_unordered_atomic, Tys);
+  Value *TheFn = Intrinsic::getDeclaration(M, ID, Tys);
 
   CallInst *CI = createCallHelper(TheFn, Ops, this);
 
   // Set the alignment of the pointer args.
-  auto *AMCI = cast<AtomicMemCpyInst>(CI);
-  AMCI->setDestAlignment(DstAlign);
-  AMCI->setSourceAlignment(SrcAlign);
+  auto *AMTI = cast<AtomicMemTransferInst>(CI);
+  AMTI->setDestAlignment(DstAlign);
+  AMTI->setSourceAlignment(SrcAlign);
 
   // Set the TBAA info if present.
   if (TBAATag)
     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 
   // Set the TBAA Struct info if present.
   if (TBAAStructTag)
     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
 
   if (ScopeTag)
     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 
   if (NoAliasTag)
     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 
   return CI;
 }
 
-CallInst *IRBuilderBase::
-CreateMemMove(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
-              Value *Size, bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
-              MDNode *NoAliasTag) {
-  assert((DstAlign == 0 || isPowerOf2_32(DstAlign)) && "Must be 0 or a power of 2");
-  assert((SrcAlign == 0 || isPowerOf2_32(SrcAlign)) && "Must be 0 or a power of 2");
-  Dst = getCastedInt8PtrValue(Dst);
-  Src = getCastedInt8PtrValue(Src);
-
-  Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
-  Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
-  Module *M = BB->getParent()->getParent();
-  Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
-  
-  CallInst *CI = createCallHelper(TheFn, Ops, this);
-
-  auto *MMI = cast<MemMoveInst>(CI);
-  if (DstAlign > 0)
-    MMI->setDestAlignment(DstAlign);
-  if (SrcAlign > 0)
-    MMI->setSourceAlignment(SrcAlign);
-
-  // Set the TBAA info if present.
-  if (TBAATag)
-    CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
- 
-  if (ScopeTag)
-    CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
- 
-  if (NoAliasTag)
-    CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
- 
-  return CI;  
-}
-
 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
                                     Value *Src) {
   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
   Value *Ops[] = {Src};
   Type *Tys[] = { Src->getType()->getVectorElementType(), Src->getType() };
   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
   return createCallHelper(Decl, Ops, Builder);
 }

lib/Transforms/Scalar/LoopIdiomRecognize.cpp

private:
   bool HasMemmove;
 
   /// Return code for isLegalStore()
   enum LegalStoreKind {
     None = 0,
     Memset,
     MemsetPattern,
     MemcpyOrMemmove,
-    UnorderedAtomicMemcpy,
+    UnorderedAtomicMemcpyOrMemmove,
     DontUse // Dummy retval never to be used. Allows catching errors in retval
             // handling.
   };
 
   /// \name Countable Loop Idiom Handling
   /// @{
 
   bool runOnCountableLoop();
const SCEVAddRecExpr *LoadEv =
         dyn_cast<SCEVAddRecExpr>(SE->getSCEV(LI->getPointerOperand()));
     if (!LoadEv || LoadEv->getLoop() != CurLoop || !LoadEv->isAffine())
       return LegalStoreKind::None;
 
     // The store and load must share the same stride.
     if (StoreEv->getOperand(1) != LoadEv->getOperand(1))
       return LegalStoreKind::None;
 
-    if (UnorderedAtomic || LI->isAtomic())
-      return HasMemcpy ? LegalStoreKind::UnorderedAtomicMemcpy
-                       : LegalStoreKind::None;
-
     // Success.  This store can be converted into a memcpy or memmove.
-    return LegalStoreKind::MemcpyOrMemmove;
+    return (UnorderedAtomic || LI->isAtomic())
+               ? LegalStoreKind::UnorderedAtomicMemcpyOrMemmove
+               : LegalStoreKind::MemcpyOrMemmove;
   }
   // This store can't be transformed into a memset/memcpy.
   return LegalStoreKind::None;
 }
 
 void LoopIdiomRecognize::collectStores(BasicBlock *BB) {
   StoreRefsForMemset.clear();
   StoreRefsForMemsetPattern.clear();
case LegalStoreKind::Memset: {
       StoreRefsForMemset[Ptr].push_back(SI);
     } break;
     case LegalStoreKind::MemsetPattern: {
       // Find the base pointer.
       Value *Ptr = GetUnderlyingObject(SI->getPointerOperand(), *DL);
       StoreRefsForMemsetPattern[Ptr].push_back(SI);
     } break;
     case LegalStoreKind::MemcpyOrMemmove:
-    case LegalStoreKind::UnorderedAtomicMemcpy:
+    case LegalStoreKind::UnorderedAtomicMemcpyOrMemmove:
       StoreRefsForMemcpyOrMemmove.push_back(SI);
       break;
     default:
       assert(false && "unhandled return value");
       break;
     }
   }
 }
bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(StoreInst *SI,
   Value *NumBytes =
       Expander.expandCodeFor(NumBytesS, IntPtrTy, Preheader->getTerminator());
 
   unsigned Align = std::min(SI->getAlignment(), LI->getAlignment());
   CallInst *NewCall = nullptr;
   // Check whether to generate an unordered atomic memcpy:
   //  If the load or store are atomic, then they must necessarily be unordered
   //  by previous checks.
-  bool IsAtomicLoadOrStore = SI->isAtomic() || LI->isAtomic();
-
-  // FIXME: We should build an atomic memmove lowering like we have for
-  // memcpy.
-  assert((!IsAtomicLoadOrStore || !PerformMemmove) &&
-         "cannot memmove atomic load or store");
-
-  if (PerformMemmove)
-    NewCall = Builder.CreateMemMove(StoreBasePtr, Align, LoadBasePtr, Align,
-                                    NumBytes);
-  else if (!IsAtomicLoadOrStore)
-    NewCall = Builder.CreateMemCpy(StoreBasePtr, LoadBasePtr, NumBytes, Align);
-  else {
+  if (SI->isAtomic() || LI->isAtomic()) {
     // We cannot allow unaligned ops for unordered load/store, so reject
     // anything where the alignment isn't at least the element size.
     if (Align < StoreSize)
       return false;
 
     // If the element.atomic memcpy is not lowered into explicit
     // loads/stores later, then it will be lowered into an element-size
     // specific lib call. If the lib call doesn't exist for our store size, then
     // we shouldn't generate the memcpy.
     if (StoreSize > TTI->getAtomicMemIntrinsicMaxElementSize())
       return false;
 
     // Create the call.
     // Note that unordered atomic loads/stores are *required* by the spec to
     // have an alignment but non-atomic loads/stores may not.
+    if (PerformMemmove)
+      NewCall = Builder.CreateElementUnorderedAtomicMemMove(
+          StoreBasePtr, SI->getAlignment(), LoadBasePtr, LI->getAlignment(),
+          NumBytes, StoreSize);
+    else
       NewCall = Builder.CreateElementUnorderedAtomicMemCpy(
           StoreBasePtr, SI->getAlignment(), LoadBasePtr, LI->getAlignment(),
           NumBytes, StoreSize);
-  }
+  } else if (PerformMemmove)
+    NewCall = Builder.CreateMemMove(StoreBasePtr, Align, LoadBasePtr, Align,
+                                    NumBytes);
+  else
+    NewCall = Builder.CreateMemCpy(StoreBasePtr, LoadBasePtr, NumBytes, Align);
+
   NewCall->setDebugLoc(SI->getDebugLoc());
 
 
   DEBUG(dbgs() << "  Formed " << (PerformMemmove ? "memmove: " : "memcpy: ")
                << *NewCall << "\n"
                << "    from load ptr=" << *LoadEv << " at: " << *LI << "\n"
                << "    from store ptr=" << *StoreEv << " at: " << *SI << "\n");
 

test/Transforms/LoopIdiom/X86/unordered-atomic-memcpy.ll

for.body:                                         ; preds = %entry, %for.body
   store atomic i32 1, i32* %arrayidx unordered, align 4
   %indvar.next = add i64 %indvar, 1
   %exitcond = icmp eq i64 %indvar.next, 10000
   br i1 %exitcond, label %for.end, label %for.body
 
 for.end:                                          ; preds = %for.body
   ret void
 }
+
+;; memmove.atomic formation (atomic load & store)
+define void @test_memmove_one(i8* readonly %begin, i8* readnone %end, i8* nocapture %out) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i8* %begin, %end
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %dest.i = phi i8* [ %dest.next, %for.body ], [ %out, %for.body.preheader ]
+  %begin.i = phi i8* [ %begin.next, %for.body ], [ %begin, %for.body.preheader ]
+  %0 = load atomic i8, i8* %begin.i unordered, align 1
+  store atomic i8 %0, i8* %dest.i unordered, align 1
+  %begin.next = getelementptr inbounds i8, i8* %begin.i, i64 1
+  %dest.next = getelementptr inbounds i8, i8* %dest.i, i64 1
+  %cmp = icmp eq i8* %begin.next, %end
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+; CHECK-LABEL: @test_memmove_one(
+; CHECK: call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %out, i8* align 1 %begin,
+; CHECK-NOT: store
+; CHECK: ret void
+}
+
+
+;; memmove.atomic formation (atomic store & normal load)
+define void @test_memmove_two(i8* readonly %begin, i8* readnone %end, i8* nocapture %out) nounwind ssp {
+entry:
+  %cmp1 = icmp eq i8* %begin, %end
+  br i1 %cmp1, label %for.end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %dest.i = phi i8* [ %dest.next, %for.body ], [ %out, %for.body.preheader ]
+  %begin.i = phi i8* [ %begin.next, %for.body ], [ %begin, %for.body.preheader ]
+  %0 = load i8, i8* %begin.i, align 1
+  store atomic i8 %0, i8* %dest.i unordered, align 1
+  %begin.next = getelementptr inbounds i8, i8* %begin.i, i64 1
+  %dest.next = getelementptr inbounds i8, i8* %dest.i, i64 1
+  %cmp = icmp eq i8* %begin.next, %end
+  br i1 %cmp, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+; CHECK-LABEL: @test_memmove_two(
+; CHECK: call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i64(i8* align 1 %out, i8* align 1 %begin,
+; CHECK-NOT: store
+; CHECK: ret void
+}