[MemCpyOpt] Expand two memcpy's with clobber inbetween (PR59116)

[lebedev.ri]

Summary

This is a WIP of a somewhat more principled attempt to solve https://github.com/llvm/llvm-project/issues/59116
Admittedly, i am not familiar neither with this pass, nor with MSSA.

We have something like this:

MDep: memcpy(tmp <- a)
...
... a is potentially modified inbetween, e.g.:
memcpy(a <- b)
...
M:    memcpy(b <- tmp)

Since we know that tmp is last modified by MDep,
what we can do, is expand MDep's memcpy into load+store pair,
and then expand M's memcpy into a store of the MDep's load:

reload = load a
store reload, tmp ; spill
...
store reload, b   ; final store

This pattern can happen e.g. when swapping contents of the a and b,
in which case tmp might go away completely, especially if it is an alloca.

This isn't always an obvious improvement, and in general, creating large
vectors can easily cause problematic compile-time implications, so there is a
profitability heuristic: loading said vector should not require more vectors
than theoretically available on the given target.

Conceptually, does this make sense? Thoughts on the profitability check?
Should we actually ensure that the clobber is a memcpy(a <- b) ?

I suspect it's still too lax, yet we will have compile-time implications regardless.

[lebedev.ri]

poke.
Does this conceptually make sense for this pass?

[nikic]

At a high level, I'd say that this transform would be a better fit for SROA. The profitability is clearer if we can actually eliminate the alloca and spill from the first memcpy, making this a single load and store.

llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp

1195

So we want to use up *all* vector registers for the copy? That's like 64 * 32 = 2048 bytes for AVX-512. That seems *way* too aggressive.

[lebedev.ri]

Indeed, this originated as a SROA change, which is now back in.
I don't have an immediate need for non-SROA variant of this.

llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h

 class MemCpyInst;
 class MemMoveInst;
 class MemorySSA;
 class MemorySSAUpdater;
 class MemSetInst;
 class StoreInst;
 class TargetLibraryInfo;
 class Value;
+class TargetTransformInfo;
 
 class MemCpyOptPass : public PassInfoMixin<MemCpyOptPass> {
+  const TargetTransformInfo *TTI;
   TargetLibraryInfo *TLI = nullptr;
   AAResults *AA = nullptr;
   AssumptionCache *AC = nullptr;
   DominatorTree *DT = nullptr;
   MemorySSA *MSSA = nullptr;
   MemorySSAUpdater *MSSAU = nullptr;
 
 public:
   MemCpyOptPass() = default;
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 
   // Glue for the old PM.
-  bool runImpl(Function &F, TargetLibraryInfo *TLI, AAResults *AA,
-               AssumptionCache *AC, DominatorTree *DT, MemorySSA *MSSA);
+  bool runImpl(Function &F, TargetTransformInfo *TTI, TargetLibraryInfo *TLI,
+               AAResults *AA, AssumptionCache *AC, DominatorTree *DT,
+               MemorySSA *MSSA);
 
 private:
   // Helper functions
   bool processStore(StoreInst *SI, BasicBlock::iterator &BBI);
   bool processMemSet(MemSetInst *SI, BasicBlock::iterator &BBI);
   bool processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI);
   bool processMemMove(MemMoveInst *M);
   bool performCallSlotOptzn(Instruction *cpyLoad, Instruction *cpyStore,
                             Value *cpyDst, Value *cpySrc, TypeSize cpyLen,
                             Align cpyAlign, std::function<CallInst *()> GetC);
-  bool processMemCpyMemCpyDependence(MemCpyInst *M, MemCpyInst *MDep);
+  bool processMemCpyThroughImmediate(MemCpyInst *M, MemCpyInst *MDep,
+                                     BasicBlock::iterator &BBI);
+  bool processMemCpyMemCpyDependence(MemCpyInst *M, MemCpyInst *MDep,
+                                     BasicBlock::iterator &BBI);
   bool processMemSetMemCpyDependence(MemCpyInst *MemCpy, MemSetInst *MemSet);
   bool performMemCpyToMemSetOptzn(MemCpyInst *MemCpy, MemSetInst *MemSet);
   bool processByValArgument(CallBase &CB, unsigned ArgNo);
   Instruction *tryMergingIntoMemset(Instruction *I, Value *StartPtr,
                                     Value *ByteVal);
   bool moveUp(StoreInst *SI, Instruction *P, const LoadInst *LI);
 
   void eraseInstruction(Instruction *I);
   bool iterateOnFunction(Function &F);
 };
 
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_SCALAR_MEMCPYOPTIMIZER_H

llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp

 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/CaptureTracking.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/MemorySSA.h"
 #include "llvm/Analysis/MemorySSAUpdater.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalVariable.h"

 using namespace llvm;
 
 #define DEBUG_TYPE "memcpyopt"
 
 static cl::opt<bool> EnableMemCpyOptWithoutLibcalls(
     "enable-memcpyopt-without-libcalls", cl::Hidden,
     cl::desc("Enable memcpyopt even when libcalls are disabled"));
 
+STATISTIC(NumMemCpyThroughIntermediate,
+          "Number of memcpy of memcpy w/ clobber inbetween expanded into "
+          "loads+stores instructions deleted");
 STATISTIC(NumMemCpyInstr, "Number of memcpy instructions deleted");
 STATISTIC(NumMemSetInfer, "Number of memsets inferred");
 STATISTIC(NumMoveToCpy,   "Number of memmoves converted to memcpy");
 STATISTIC(NumCpyToSet,    "Number of memcpys converted to memset");
 STATISTIC(NumCallSlot,    "Number of call slot optimizations performed");
 
 namespace {
 
public:
 
   bool runOnFunction(Function &F) override;
 
 private:
   // This transformation requires dominator postdominator info
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesCFG();
     AU.addRequired<AssumptionCacheTracker>();
+    AU.addRequired<TargetTransformInfoWrapperPass>();
     AU.addRequired<DominatorTreeWrapperPass>();
     AU.addPreserved<DominatorTreeWrapperPass>();
     AU.addPreserved<GlobalsAAWrapperPass>();
     AU.addRequired<TargetLibraryInfoWrapperPass>();
     AU.addRequired<AAResultsWrapperPass>();
     AU.addPreserved<AAResultsWrapperPass>();
     AU.addRequired<MemorySSAWrapperPass>();
     AU.addPreserved<MemorySSAWrapperPass>();
   }
 };
 
 } // end anonymous namespace
 
 char MemCpyOptLegacyPass::ID = 0;
 
 /// The public interface to this file...
 FunctionPass *llvm::createMemCpyOptPass() { return new MemCpyOptLegacyPass(); }
 
 INITIALIZE_PASS_BEGIN(MemCpyOptLegacyPass, "memcpyopt", "MemCpy Optimization",
                       false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
 INITIALIZE_PASS_END(MemCpyOptLegacyPass, "memcpyopt", "MemCpy Optimization",
                     false, false)
bool MemCpyOptPass::performCallSlotOptzn(Instruction *cpyLoad,
   combineMetadata(C, cpyLoad, KnownIDs, true);
   if (cpyLoad != cpyStore)
     combineMetadata(C, cpyStore, KnownIDs, true);
 
   ++NumCallSlot;
   return true;
 }
 
+// We have something like this:
+// ```
+//   MDep: memcpy(tmp <- a)
+//   ...
+//   ... a is potentially modified inbetween, e.g.:
+//   memcpy(a <- b)
+//   ...
+//   M:    memcpy(b <- tmp)
+// ```
+// Since we know that tmp is last modified by MDep,
+// what we can do, is expand MDep's memcpy into load+store pair,
+// and then expand M's memcpy into a store of the MDep's load:
+// ```
+//   reload = load a
+//   store reload, tmp ; spill
+//   ...
+//   store reload, b   ; final store
+// ```
+// This pattern can happen e.g. when swapping contents of the a and b,
+// in which case tmp might go away completely, especially if it is an alloca.
+//
+// This isn't always an obvious improvement, and in general, creating large
+// vectors can easily cause problematic compile-time implications, so there is a
+// profitability heuristic: loading said vector should not require more vectors
+// than theoretically avaliable on the given target.
+bool MemCpyOptPass::processMemCpyThroughImmediate(MemCpyInst *M,
+                                                  MemCpyInst *MDep,
+                                                  BasicBlock::iterator &BBI) {
+  IRBuilder<> Builder(M->getContext());
+
+  const uint64_t NumBytes = cast<ConstantInt>(M->getLength())->getZExtValue();
+
+  Type *ByteVecTy = FixedVectorType::get(
+      IntegerType::getInt8Ty(Builder.getContext()), NumBytes);
+
+  unsigned NumRegs = TTI->getNumberOfRegisters(
+      TTI->getRegisterClassForType(/*Vector=*/true, ByteVecTy));
+  if (!NumRegs)
+    return false;
+
+  unsigned RegBitWidth =
+      TTI->getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
+          .getFixedSize();
+  const unsigned NeededRegs = divideCeil(8 * NumBytes, RegBitWidth);
+  if (NeededRegs > NumRegs)
+    return false;
+
+  Value *SrcAddr = MDep->getSource();
+  Value *SpillAddr = MDep->getDest();
+  assert(M->getSource() == SpillAddr && "Unexpected memory flow.");
+  Value *TgtAddr = M->getDest();
+
+  Builder.SetInsertPoint(MDep);
+  Instruction *ReloadedVal =
+      Builder.CreateAlignedLoad(ByteVecTy, SrcAddr, MDep->getSourceAlign(),
+                                SrcAddr->getName() + ".reload");
+  ReloadedVal->setAAMetadata(MDep->getAAMetadata());
+  Instruction *Spill =
+      Builder.CreateAlignedStore(ReloadedVal, SpillAddr, MDep->getDestAlign());
+  Spill->setAAMetadata(MDep->getAAMetadata());
+
+  Builder.SetInsertPoint(M);
+  Instruction *Store =
+      Builder.CreateAlignedStore(ReloadedVal, TgtAddr, M->getDestAlign());
+  Store->setAAMetadata(M->getAAMetadata());
+
+  BBI = Store->getIterator();
+
+  eraseInstruction(M);
+  eraseInstruction(MDep);
+
+  ++NumMemCpyThroughIntermediate;
+  return true;
+}
+
 /// We've found that the (upward scanning) memory dependence of memcpy 'M' is
 /// the memcpy 'MDep'. Try to simplify M to copy from MDep's input if we can.
 bool MemCpyOptPass::processMemCpyMemCpyDependence(MemCpyInst *M,
-                                                  MemCpyInst *MDep) {
+                                                  MemCpyInst *MDep,
+                                                  BasicBlock::iterator &BBI) {
   // We can only transforms memcpy's where the dest of one is the source of the
   // other.
   if (M->getSource() != MDep->getDest() || MDep->isVolatile())
     return false;
 
   // If dep instruction is reading from our current input, then it is a noop
   // transfer and substituting the input won't change this instruction.  Just
   // ignore the input and let someone else zap MDep.  This handles cases like:
bool MemCpyOptPass::processMemCpyMemCpyDependence(MemCpyInst *M,
   }
 
   // Verify that the copied-from memory doesn't change in between the two
   // transfers.  For example, in:
   //    memcpy(a <- b)
   //    *b = 42;
   //    memcpy(c <- a)
   // It would be invalid to transform the second memcpy into memcpy(c <- b).
+  // but this could be a part of a swap pattern.
   //
   // TODO: If the code between M and MDep is transparent to the destination "c",
   // then we could still perform the xform by moving M up to the first memcpy.
   // TODO: It would be sufficient to check the MDep source up to the memcpy
   // size of M, rather than MDep.
   if (writtenBetween(MSSA, *AA, MemoryLocation::getForSource(MDep),
                      MSSA->getMemoryAccess(MDep), MSSA->getMemoryAccess(M)))
-    return false;
+    return processMemCpyThroughImmediate(M, MDep, BBI);
 
   // If the dest of the second might alias the source of the first, then the
   // source and dest might overlap. In addition, if the source of the first
   // points to constant memory, they won't overlap by definition. Otherwise, we
   // still want to eliminate the intermediate value, but we have to generate a
   // memmove instead of memcpy.
   bool UseMemMove = false;
   if (isModSet(AA->getModRefInfo(M, MemoryLocation::getForSource(MDep))))
if (Instruction *MI = MD->getMemoryInst()) {
                               << "    memcpy: " << *M << "\n");
             eraseInstruction(M);
             ++NumMemCpyInstr;
             return true;
           }
         }
       }
       if (auto *MDep = dyn_cast<MemCpyInst>(MI))
-        return processMemCpyMemCpyDependence(M, MDep);
+        return processMemCpyMemCpyDependence(M, MDep, BBI);
       if (auto *MDep = dyn_cast<MemSetInst>(MI)) {
         if (performMemCpyToMemSetOptzn(M, MDep)) {
           LLVM_DEBUG(dbgs() << "Converted memcpy to memset\n");
           eraseInstruction(M);
           ++NumCpyToSet;
           return true;
         }
       }
for (BasicBlock::iterator BI = BB.begin(), BE = BB.end(); BI != BE;) {
       }
     }
   }
 
   return MadeChange;
 }
 
 PreservedAnalyses MemCpyOptPass::run(Function &F, FunctionAnalysisManager &AM) {
+  auto &TTI = AM.getResult<TargetIRAnalysis>(F);
   auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
   auto *AA = &AM.getResult<AAManager>(F);
   auto *AC = &AM.getResult<AssumptionAnalysis>(F);
   auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
   auto *MSSA = &AM.getResult<MemorySSAAnalysis>(F);
 
-  bool MadeChange = runImpl(F, &TLI, AA, AC, DT, &MSSA->getMSSA());
+  bool MadeChange = runImpl(F, &TTI, &TLI, AA, AC, DT, &MSSA->getMSSA());
   if (!MadeChange)
     return PreservedAnalyses::all();
 
   PreservedAnalyses PA;
   PA.preserveSet<CFGAnalyses>();
   PA.preserve<MemorySSAAnalysis>();
   return PA;
 }
 
-bool MemCpyOptPass::runImpl(Function &F, TargetLibraryInfo *TLI_,
-                            AliasAnalysis *AA_, AssumptionCache *AC_,
-                            DominatorTree *DT_, MemorySSA *MSSA_) {
+bool MemCpyOptPass::runImpl(Function &F, TargetTransformInfo *TTI_,
+                            TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
+                            AssumptionCache *AC_, DominatorTree *DT_,
+                            MemorySSA *MSSA_) {
   bool MadeChange = false;
+  TTI = TTI_;
   TLI = TLI_;
   AA = AA_;
   AC = AC_;
   DT = DT_;
   MSSA = MSSA_;
   MemorySSAUpdater MSSAU_(MSSA_);
   MSSAU = &MSSAU_;
 
bool MemCpyOptPass::runImpl(Function &F, TargetTransformInfo *TTI_,
   return MadeChange;
 }
 
 /// This is the main transformation entry point for a function.
 bool MemCpyOptLegacyPass::runOnFunction(Function &F) {
   if (skipFunction(F))
     return false;
 
+  auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
   auto *TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
   auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
   auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
   auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
   auto *MSSA = &getAnalysis<MemorySSAWrapperPass>().getMSSA();
 
-  return Impl.runImpl(F, TLI, AA, AC, DT, MSSA);
+  return Impl.runImpl(F, TTI, TLI, AA, AC, DT, MSSA);
 }

llvm/test/Transforms/MemCpyOpt/X86/lit.local.cfg

+if not 'X86' in config.root.targets:
+    config.unsupported = True

llvm/test/Transforms/MemCpyOpt/X86/memory-swap.ll

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -memcpyopt -verify-memoryssa -mtriple=x86_64-- -mattr=-sse,-avx,-avx512f | FileCheck %s --check-prefixes=CHECK-ALL,CHECK-SCALAR
+; RUN: opt < %s -S -memcpyopt -verify-memoryssa -mtriple=x86_64-- -mattr=+sse,-avx,-avx512f | FileCheck %s --check-prefixes=CHECK-ALL,CHECK-SSE,CHECK-SSE1
+; RUN: opt < %s -S -memcpyopt -verify-memoryssa -mtriple=x86_64-- -mattr=+sse,+avx,-avx512f | FileCheck %s --check-prefixes=CHECK-ALL,CHECK-SSE,CHECK-AVX,CHECK-AVX1
+; RUN: opt < %s -S -memcpyopt -verify-memoryssa -mtriple=x86_64-- -mattr=+sse,+avx,+avx512f | FileCheck %s --check-prefixes=CHECK-ALL,CHECK-SSE,CHECK-AVX,CHECK-AVX512F
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @swap-16bytes(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B) {
+; CHECK-SCALAR-LABEL: @swap-16bytes(
+; CHECK-SCALAR-NEXT:    [[INTERMEDIATE:%.*]] = alloca [16 x i8], align 1
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 16, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 16, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 16, i1 false)
+; CHECK-SCALAR-NEXT:    ret void
+;
+; CHECK-SSE-LABEL: @swap-16bytes(
+; CHECK-SSE-NEXT:    [[INTERMEDIATE:%.*]] = alloca [16 x i8], align 1
+; CHECK-SSE-NEXT:    [[SOURCE_A_RELOAD:%.*]] = load <16 x i8>, ptr [[SOURCE_A:%.*]], align 1
+; CHECK-SSE-NEXT:    store <16 x i8> [[SOURCE_A_RELOAD]], ptr [[INTERMEDIATE]], align 1
+; CHECK-SSE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 16, i1 false)
+; CHECK-SSE-NEXT:    store <16 x i8> [[SOURCE_A_RELOAD]], ptr [[SOURCE_B]], align 1
+; CHECK-SSE-NEXT:    ret void
+;
+  %INTERMEDIATE = alloca [16 x i8]
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %INTERMEDIATE, ptr align 1 %SOURCE_A, i64 16, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 16, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_B, ptr align 1 %INTERMEDIATE, i64 16, i1 false)
+  ret void
+}
+
+define void @swap-32bytes(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B) {
+; CHECK-SCALAR-LABEL: @swap-32bytes(
+; CHECK-SCALAR-NEXT:    [[INTERMEDIATE:%.*]] = alloca [32 x i8], align 1
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 32, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 32, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 32, i1 false)
+; CHECK-SCALAR-NEXT:    ret void
+;
+; CHECK-SSE-LABEL: @swap-32bytes(
+; CHECK-SSE-NEXT:    [[INTERMEDIATE:%.*]] = alloca [32 x i8], align 1
+; CHECK-SSE-NEXT:    [[SOURCE_A_RELOAD:%.*]] = load <32 x i8>, ptr [[SOURCE_A:%.*]], align 1
+; CHECK-SSE-NEXT:    store <32 x i8> [[SOURCE_A_RELOAD]], ptr [[INTERMEDIATE]], align 1
+; CHECK-SSE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 32, i1 false)
+; CHECK-SSE-NEXT:    store <32 x i8> [[SOURCE_A_RELOAD]], ptr [[SOURCE_B]], align 1
+; CHECK-SSE-NEXT:    ret void
+;
+  %INTERMEDIATE = alloca [32 x i8]
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %INTERMEDIATE, ptr align 1 %SOURCE_A, i64 32, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 32, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_B, ptr align 1 %INTERMEDIATE, i64 32, i1 false)
+  ret void
+}
+
+define void @swap-64bytes(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B) {
+; CHECK-SCALAR-LABEL: @swap-64bytes(
+; CHECK-SCALAR-NEXT:    [[INTERMEDIATE:%.*]] = alloca [64 x i8], align 1
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 64, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 64, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 64, i1 false)
+; CHECK-SCALAR-NEXT:    ret void
+;
+; CHECK-SSE-LABEL: @swap-64bytes(
+; CHECK-SSE-NEXT:    [[INTERMEDIATE:%.*]] = alloca [64 x i8], align 1
+; CHECK-SSE-NEXT:    [[SOURCE_A_RELOAD:%.*]] = load <64 x i8>, ptr [[SOURCE_A:%.*]], align 1
+; CHECK-SSE-NEXT:    store <64 x i8> [[SOURCE_A_RELOAD]], ptr [[INTERMEDIATE]], align 1
+; CHECK-SSE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 64, i1 false)
+; CHECK-SSE-NEXT:    store <64 x i8> [[SOURCE_A_RELOAD]], ptr [[SOURCE_B]], align 1
+; CHECK-SSE-NEXT:    ret void
+;
+  %INTERMEDIATE = alloca [64 x i8]
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %INTERMEDIATE, ptr align 1 %SOURCE_A, i64 64, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 64, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_B, ptr align 1 %INTERMEDIATE, i64 64, i1 false)
+  ret void
+}
+
+define void @swap-128bytes(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B) {
+; CHECK-SCALAR-LABEL: @swap-128bytes(
+; CHECK-SCALAR-NEXT:    [[INTERMEDIATE:%.*]] = alloca [128 x i8], align 1
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 128, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 128, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 128, i1 false)
+; CHECK-SCALAR-NEXT:    ret void
+;
+; CHECK-SSE-LABEL: @swap-128bytes(
+; CHECK-SSE-NEXT:    [[INTERMEDIATE:%.*]] = alloca [128 x i8], align 1
+; CHECK-SSE-NEXT:    [[SOURCE_A_RELOAD:%.*]] = load <128 x i8>, ptr [[SOURCE_A:%.*]], align 1
+; CHECK-SSE-NEXT:    store <128 x i8> [[SOURCE_A_RELOAD]], ptr [[INTERMEDIATE]], align 1
+; CHECK-SSE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 128, i1 false)
+; CHECK-SSE-NEXT:    store <128 x i8> [[SOURCE_A_RELOAD]], ptr [[SOURCE_B]], align 1
+; CHECK-SSE-NEXT:    ret void
+;
+  %INTERMEDIATE = alloca [128 x i8]
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %INTERMEDIATE, ptr align 1 %SOURCE_A, i64 128, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 128, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_B, ptr align 1 %INTERMEDIATE, i64 128, i1 false)
+  ret void
+}
+
+define void @swap-256bytes(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B) {
+; CHECK-SCALAR-LABEL: @swap-256bytes(
+; CHECK-SCALAR-NEXT:    [[INTERMEDIATE:%.*]] = alloca [256 x i8], align 1
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 256, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 256, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 256, i1 false)
+; CHECK-SCALAR-NEXT:    ret void
+;
+; CHECK-SSE-LABEL: @swap-256bytes(
+; CHECK-SSE-NEXT:    [[INTERMEDIATE:%.*]] = alloca [256 x i8], align 1
+; CHECK-SSE-NEXT:    [[SOURCE_A_RELOAD:%.*]] = load <256 x i8>, ptr [[SOURCE_A:%.*]], align 1
+; CHECK-SSE-NEXT:    store <256 x i8> [[SOURCE_A_RELOAD]], ptr [[INTERMEDIATE]], align 1
+; CHECK-SSE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 256, i1 false)
+; CHECK-SSE-NEXT:    store <256 x i8> [[SOURCE_A_RELOAD]], ptr [[SOURCE_B]], align 1
+; CHECK-SSE-NEXT:    ret void
+;
+  %INTERMEDIATE = alloca [256 x i8]
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %INTERMEDIATE, ptr align 1 %SOURCE_A, i64 256, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 256, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_B, ptr align 1 %INTERMEDIATE, i64 256, i1 false)
+  ret void
+}
+
+define void @swap-512bytes(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B) {
+; CHECK-SCALAR-LABEL: @swap-512bytes(
+; CHECK-SCALAR-NEXT:    [[INTERMEDIATE:%.*]] = alloca [512 x i8], align 1
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 512, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 512, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 512, i1 false)
+; CHECK-SCALAR-NEXT:    ret void
+;
+; CHECK-SSE1-LABEL: @swap-512bytes(
+; CHECK-SSE1-NEXT:    [[INTERMEDIATE:%.*]] = alloca [512 x i8], align 1
+; CHECK-SSE1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 512, i1 false)
+; CHECK-SSE1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 512, i1 false)
+; CHECK-SSE1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 512, i1 false)
+; CHECK-SSE1-NEXT:    ret void
+;
+; CHECK-AVX-LABEL: @swap-512bytes(
+; CHECK-AVX-NEXT:    [[INTERMEDIATE:%.*]] = alloca [512 x i8], align 1
+; CHECK-AVX-NEXT:    [[SOURCE_A_RELOAD:%.*]] = load <512 x i8>, ptr [[SOURCE_A:%.*]], align 1
+; CHECK-AVX-NEXT:    store <512 x i8> [[SOURCE_A_RELOAD]], ptr [[INTERMEDIATE]], align 1
+; CHECK-AVX-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 512, i1 false)
+; CHECK-AVX-NEXT:    store <512 x i8> [[SOURCE_A_RELOAD]], ptr [[SOURCE_B]], align 1
+; CHECK-AVX-NEXT:    ret void
+;
+  %INTERMEDIATE = alloca [512 x i8]
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %INTERMEDIATE, ptr align 1 %SOURCE_A, i64 512, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 512, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_B, ptr align 1 %INTERMEDIATE, i64 512, i1 false)
+  ret void
+}
+
+define void @swap-1024bytes(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B) {
+; CHECK-SCALAR-LABEL: @swap-1024bytes(
+; CHECK-SCALAR-NEXT:    [[INTERMEDIATE:%.*]] = alloca [1024 x i8], align 1
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 1024, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 1024, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 1024, i1 false)
+; CHECK-SCALAR-NEXT:    ret void
+;
+; CHECK-SSE1-LABEL: @swap-1024bytes(
+; CHECK-SSE1-NEXT:    [[INTERMEDIATE:%.*]] = alloca [1024 x i8], align 1
+; CHECK-SSE1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 1024, i1 false)
+; CHECK-SSE1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 1024, i1 false)
+; CHECK-SSE1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 1024, i1 false)
+; CHECK-SSE1-NEXT:    ret void
+;
+; CHECK-AVX1-LABEL: @swap-1024bytes(
+; CHECK-AVX1-NEXT:    [[INTERMEDIATE:%.*]] = alloca [1024 x i8], align 1
+; CHECK-AVX1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 1024, i1 false)
+; CHECK-AVX1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 1024, i1 false)
+; CHECK-AVX1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 1024, i1 false)
+; CHECK-AVX1-NEXT:    ret void
+;
+; CHECK-AVX512F-LABEL: @swap-1024bytes(
+; CHECK-AVX512F-NEXT:    [[INTERMEDIATE:%.*]] = alloca [1024 x i8], align 1
+; CHECK-AVX512F-NEXT:    [[SOURCE_A_RELOAD:%.*]] = load <1024 x i8>, ptr [[SOURCE_A:%.*]], align 1
+; CHECK-AVX512F-NEXT:    store <1024 x i8> [[SOURCE_A_RELOAD]], ptr [[INTERMEDIATE]], align 1
+; CHECK-AVX512F-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 1024, i1 false)
+; CHECK-AVX512F-NEXT:    store <1024 x i8> [[SOURCE_A_RELOAD]], ptr [[SOURCE_B]], align 1
+; CHECK-AVX512F-NEXT:    ret void
+;
+  %INTERMEDIATE = alloca [1024 x i8]
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %INTERMEDIATE, ptr align 1 %SOURCE_A, i64 1024, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 1024, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_B, ptr align 1 %INTERMEDIATE, i64 1024, i1 false)
+  ret void
+}
+
+define void @swap-2048bytes(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B) {
+; CHECK-SCALAR-LABEL: @swap-2048bytes(
+; CHECK-SCALAR-NEXT:    [[INTERMEDIATE:%.*]] = alloca [2048 x i8], align 1
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 2048, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 2048, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 2048, i1 false)
+; CHECK-SCALAR-NEXT:    ret void
+;
+; CHECK-SSE1-LABEL: @swap-2048bytes(
+; CHECK-SSE1-NEXT:    [[INTERMEDIATE:%.*]] = alloca [2048 x i8], align 1
+; CHECK-SSE1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 2048, i1 false)
+; CHECK-SSE1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 2048, i1 false)
+; CHECK-SSE1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 2048, i1 false)
+; CHECK-SSE1-NEXT:    ret void
+;
+; CHECK-AVX1-LABEL: @swap-2048bytes(
+; CHECK-AVX1-NEXT:    [[INTERMEDIATE:%.*]] = alloca [2048 x i8], align 1
+; CHECK-AVX1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 2048, i1 false)
+; CHECK-AVX1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 2048, i1 false)
+; CHECK-AVX1-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 2048, i1 false)
+; CHECK-AVX1-NEXT:    ret void
+;
+; CHECK-AVX512F-LABEL: @swap-2048bytes(
+; CHECK-AVX512F-NEXT:    [[INTERMEDIATE:%.*]] = alloca [2048 x i8], align 1
+; CHECK-AVX512F-NEXT:    [[SOURCE_A_RELOAD:%.*]] = load <2048 x i8>, ptr [[SOURCE_A:%.*]], align 1
+; CHECK-AVX512F-NEXT:    store <2048 x i8> [[SOURCE_A_RELOAD]], ptr [[INTERMEDIATE]], align 1
+; CHECK-AVX512F-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 2048, i1 false)
+; CHECK-AVX512F-NEXT:    store <2048 x i8> [[SOURCE_A_RELOAD]], ptr [[SOURCE_B]], align 1
+; CHECK-AVX512F-NEXT:    ret void
+;
+  %INTERMEDIATE = alloca [2048 x i8]
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %INTERMEDIATE, ptr align 1 %SOURCE_A, i64 2048, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 2048, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_B, ptr align 1 %INTERMEDIATE, i64 2048, i1 false)
+  ret void
+}
+
+define void @length-mismatch-final-is-smaller(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B) {
+; CHECK-SCALAR-LABEL: @length-mismatch-final-is-smaller(
+; CHECK-SCALAR-NEXT:    [[INTERMEDIATE:%.*]] = alloca [16 x i8], align 1
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 16, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 16, i1 false)
+; CHECK-SCALAR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 8, i1 false)
+; CHECK-SCALAR-NEXT:    ret void
+;
+; CHECK-SSE-LABEL: @length-mismatch-final-is-smaller(
+; CHECK-SSE-NEXT:    [[INTERMEDIATE:%.*]] = alloca [16 x i8], align 1
+; CHECK-SSE-NEXT:    [[SOURCE_A_RELOAD:%.*]] = load <8 x i8>, ptr [[SOURCE_A:%.*]], align 1
+; CHECK-SSE-NEXT:    store <8 x i8> [[SOURCE_A_RELOAD]], ptr [[INTERMEDIATE]], align 1
+; CHECK-SSE-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 16, i1 false)
+; CHECK-SSE-NEXT:    store <8 x i8> [[SOURCE_A_RELOAD]], ptr [[SOURCE_B]], align 1
+; CHECK-SSE-NEXT:    ret void
+;
+  %INTERMEDIATE = alloca [16 x i8]
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %INTERMEDIATE, ptr align 1 %SOURCE_A, i64 16, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 16, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_B, ptr align 1 %INTERMEDIATE, i64 8, i1 false)
+  ret void
+}
+
+define void @length-mismatch-final-is-larger(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B) {
+; CHECK-ALL-LABEL: @length-mismatch-final-is-larger(
+; CHECK-ALL-NEXT:    [[INTERMEDIATE:%.*]] = alloca [16 x i8], align 1
+; CHECK-ALL-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 8, i1 false)
+; CHECK-ALL-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 16, i1 false)
+; CHECK-ALL-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 16, i1 false)
+; CHECK-ALL-NEXT:    ret void
+;
+  %INTERMEDIATE = alloca [16 x i8]
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %INTERMEDIATE, ptr align 1 %SOURCE_A, i64 8, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 16, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_B, ptr align 1 %INTERMEDIATE, i64 16, i1 false)
+  ret void
+}
+
+define void @first-length-is-variable(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 %len_a) {
+; CHECK-ALL-LABEL: @first-length-is-variable(
+; CHECK-ALL-NEXT:    [[INTERMEDIATE:%.*]] = alloca [16 x i8], align 1
+; CHECK-ALL-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 [[LEN_A:%.*]], i1 false)
+; CHECK-ALL-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 16, i1 false)
+; CHECK-ALL-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 16, i1 false)
+; CHECK-ALL-NEXT:    ret void
+;
+  %INTERMEDIATE = alloca [16 x i8]
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %INTERMEDIATE, ptr align 1 %SOURCE_A, i64 %len_a, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 16, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_B, ptr align 1 %INTERMEDIATE, i64 16, i1 false)
+  ret void
+}
+
+define void @second-length-is-variable(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 %len_b) {
+; CHECK-ALL-LABEL: @second-length-is-variable(
+; CHECK-ALL-NEXT:    [[INTERMEDIATE:%.*]] = alloca [16 x i8], align 1
+; CHECK-ALL-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 16, i1 false)
+; CHECK-ALL-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 16, i1 false)
+; CHECK-ALL-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 [[LEN_B:%.*]], i1 false)
+; CHECK-ALL-NEXT:    ret void
+;
+  %INTERMEDIATE = alloca [16 x i8]
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %INTERMEDIATE, ptr align 1 %SOURCE_A, i64 16, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 16, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_B, ptr align 1 %INTERMEDIATE, i64 %len_b, i1 false)
+  ret void
+}
+
+define void @lengths-are-variable(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 %len_a, i64 %len_b) {
+; CHECK-ALL-LABEL: @lengths-are-variable(
+; CHECK-ALL-NEXT:    [[INTERMEDIATE:%.*]] = alloca [16 x i8], align 1
+; CHECK-ALL-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[INTERMEDIATE]], ptr align 1 [[SOURCE_A:%.*]], i64 [[LEN_A:%.*]], i1 false)
+; CHECK-ALL-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_A]], ptr align 1 [[SOURCE_B:%.*]], i64 16, i1 false)
+; CHECK-ALL-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[SOURCE_B]], ptr align 1 [[INTERMEDIATE]], i64 [[LEN_B:%.*]], i1 false)
+; CHECK-ALL-NEXT:    ret void
+;
+  %INTERMEDIATE = alloca [16 x i8]
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %INTERMEDIATE, ptr align 1 %SOURCE_A, i64 %len_a, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_A, ptr align 1 %SOURCE_B, i64 16, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 1 %SOURCE_B, ptr align 1 %INTERMEDIATE, i64 %len_b, i1 false)
+  ret void
+}
+
+declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1)

llvm/test/Transforms/MemCpyOpt/lifetime.ll

bb:
   call void @llvm.lifetime.end.p0(i64 16, ptr %tmp)
   ret void
 }
 
 define void @memcpy_memcpy_across_lifetime(ptr noalias %p1, ptr noalias %p2, ptr noalias %p3) {
 ; CHECK-LABEL: @memcpy_memcpy_across_lifetime(
 ; CHECK-NEXT:    [[A:%.*]] = alloca [16 x i8], align 1
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[A]])
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[A]], ptr [[P1:%.*]], i64 16, i1 false)
+; CHECK-NEXT:    [[P1_RELOAD:%.*]] = load <16 x i8>, ptr [[P1:%.*]], align 16
+; CHECK-NEXT:    store <16 x i8> [[P1_RELOAD]], ptr [[A]], align 16
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[P1]], ptr [[P2:%.*]], i64 16, i1 false)
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[P2]], ptr [[A]], i64 16, i1 false)
+; CHECK-NEXT:    store <16 x i8> [[P1_RELOAD]], ptr [[P2]], align 16
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[A]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[P3:%.*]], ptr [[P2]], i64 16, i1 false)
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca [16 x i8]
   call void @llvm.lifetime.start.p0(i64 16, ptr %a)
   call void @llvm.memcpy.p0.p0.i64(ptr %a, ptr %p1, i64 16, i1 false)
   call void @llvm.memcpy.p0.p0.i64(ptr %p1, ptr %p2, i64 16, i1 false)