[MemCpyOpt] Port to MemorySSA. Disabled by default.

[bryant]

Summary

This creates a MemorySSA-backed analogue of MemCpyOpt that is disabled by
default. It can be enabled by passing to opt either its pass-specific flag
(-memcpyopt-mssa) or a force flag (e.g., -O3 -mco-mssa).

In order to facilitate code review and correctness checking, the changes have
been made to resemble as closely as possible the pattern matching and
transformation logic of the legacy pass. In particular, non-local results from
MemorySSAWalker::getClobberingMemoryAccess/getCMA are ignored since the
MemDep version can only see patterns within a single block. The idea is to land
this patch first before adding back non-locality and other more sophisticated
matchers.

[bryant]

FileCheck output of both versions (should, and do) closely match each other.
This can be verified by first applying the patch and then:

bash
# refresh the FileCheck results for pertinent .ll tests
$ for chk in `grep -r -l MCO-MSSA test/`; do python util/update_test_check.py --tool=opt $chk; done

# verify that results are identical
$ for chk in `grep -r -l MCO-MSSA test/`; do python verifier.py $chk; done

# the verifier
$ cat verifier.py
"""
Checks if FileChecks of both labels are identical.
"""

from re import sub
from itertools import ifilter, groupby, izip_longest

def collect_outputs(lines):
    mco, mssa = [], []
    check_lines = ifilter(lambda s: "; CHECK" in s or "; MCO-MSSA" in s, lines)
    for mcoty, line in groupby(check_lines, lambda s: "; CHECK" in s):
        if mcoty:
            mco.append(map(lambda s: sub("^; CHECK\S+\s+", "", s), line))
        else:
            mssa.append(map(lambda s: sub("^; MCO-MSSA\S+\s+", "", s), line))
    return mco, mssa

if __name__ == "__main__":
    from sys import stdin, argv
    from difflib import ndiff

    lines = stdin.readlines() if len(argv) < 2 else open(argv[1]).readlines()
    mco, mssa = collect_outputs(lines)
    for a, b in izip_longest(mco, mssa):
        try:
            assert a == b
        except AssertionError:
            print "Problem in", ("stdin" if len(argv) == 0 else argv[1])
            a_ = [" " * 4 + l.strip() for l in (a or [])]
            b_ = [" " * 4 + l.strip() for l in (b or [])]
            print "\n".join(ndiff(a_, b_))
            continue

[bryant]

Forgot to mention that this depends on:

• The upward AccessList splice defined in D26661 and used by moveUp,
• D26631 (NFC), which sets up the FileChecks for the legacy MemCpyOpt, and
• the instructionClobbersQuery patch at D26659 in order to get the right clobbers for either source of dest memcpy locs.

EDIT: Clickables.

[mcrosier]

[bryant]

Holding off on this until I can get non-local DSE running, since non-local MCO is useless without that.

[jfb]

(random drive-by)

include/llvm/Transforms/Scalar.h

354

bool = false? enum class FTW.

lib/Transforms/Scalar/MemCpyOptimizer.cpp

127

Oxygen comments usually have triple slash at the beginning.

167

Drop StartPtrUser - (even if it matches the surrounding style).

331

Use the enum class here too.

include/llvm-c/Transforms/Scalar.h

 void LLVMAddLoopUnrollPass(LLVMPassManagerRef PM);
 
 /** See llvm::createLoopUnswitchPass function. */
 void LLVMAddLoopUnswitchPass(LLVMPassManagerRef PM);
 
 /** See llvm::createMemCpyOptPass function. */
 void LLVMAddMemCpyOptPass(LLVMPassManagerRef PM);
 
+/** See llvm::createMemCpyOptPass function. */
+void LLVMAddMemCpyMemSSAOptPass(LLVMPassManagerRef PM);
+
 /** See llvm::createPartiallyInlineLibCallsPass function. */
 void LLVMAddPartiallyInlineLibCallsPass(LLVMPassManagerRef PM);
 
 /** See llvm::createLowerSwitchPass function. */
 void LLVMAddLowerSwitchPass(LLVMPassManagerRef PM);
 
 /** See llvm::createPromoteMemoryToRegisterPass function. */
 void LLVMAddPromoteMemoryToRegisterPass(LLVMPassManagerRef PM);

include/llvm/InitializePasses.h

 void initializeMachinePipelinerPass(PassRegistry&);
 void initializeMachinePostDominatorTreePass(PassRegistry&);
 void initializeMachineRegionInfoPassPass(PassRegistry&);
 void initializeMachineSchedulerPass(PassRegistry&);
 void initializeMachineSinkingPass(PassRegistry&);
 void initializeMachineTraceMetricsPass(PassRegistry&);
 void initializeMachineVerifierPassPass(PassRegistry&);
 void initializeMemCpyOptLegacyPassPass(PassRegistry&);
+void initializeMemCpyOptMemSSALegacyPassPass(PassRegistry&);
 void initializeMemDepPrinterPass(PassRegistry&);
 void initializeMemDerefPrinterPass(PassRegistry&);
 void initializeMemoryDependenceWrapperPassPass(PassRegistry&);
 void initializeMemorySSAWrapperPassPass(PassRegistry&);
 void initializeMemorySSAPrinterLegacyPassPass(PassRegistry &);
 void initializeMemorySanitizerPass(PassRegistry&);
 void initializeMergeFunctionsPass(PassRegistry&);
 void initializeMergedLoadStoreMotionLegacyPassPass(PassRegistry &);

include/llvm/Transforms/Scalar.h

 //
 FunctionPass *createMergedLoadStoreMotionPass();
 
 //===----------------------------------------------------------------------===//
 //
 // MemCpyOpt - This pass performs optimizations related to eliminating memcpy
 // calls and/or combining multiple stores into memset's.
 //
-FunctionPass *createMemCpyOptPass();
+FunctionPass *createMemCpyOptPass(bool = false);
 
 //===----------------------------------------------------------------------===//
 //
 // LoopDeletion - This pass performs DCE of non-infinite loops that it
 // can prove are dead.
 //
 Pass *createLoopDeletionPass();
 

include/llvm/Transforms/Scalar/MemCpyOptimizer.h

 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/PassManager.h"
 
 namespace llvm {
 
+class MemorySSA;
+
 class MemCpyOptPass : public PassInfoMixin<MemCpyOptPass> {
+  bool UseMemorySSA;
+  MemorySSA *MSSA = nullptr;
   MemoryDependenceResults *MD = nullptr;
   TargetLibraryInfo *TLI = nullptr;
   std::function<AliasAnalysis &()> LookupAliasAnalysis;
   std::function<AssumptionCache &()> LookupAssumptionCache;
   std::function<DominatorTree &()> LookupDomTree;
 
 public:
-  MemCpyOptPass() {}
+  MemCpyOptPass(bool UseMSSA) : UseMemorySSA(UseMSSA) {}
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
   // Glue for the old PM.
-  bool runImpl(Function &F, MemoryDependenceResults *MD_,
+  bool runImpl(Function &F, MemorySSA *MSSA_, MemoryDependenceResults *MD_,
                TargetLibraryInfo *TLI_,
                std::function<AliasAnalysis &()> LookupAliasAnalysis_,
                std::function<AssumptionCache &()> LookupAssumptionCache_,
                std::function<DominatorTree &()> LookupDomTree_);
 
 private:
   // Helper functions
   bool processStore(StoreInst *SI, BasicBlock::iterator &BBI);
   bool processMemSet(MemSetInst *SI, BasicBlock::iterator &BBI);
   bool processMemCpy(MemCpyInst *M);
+  bool processMemCpyMSSA(MemCpyInst *);
   bool processMemMove(MemMoveInst *M);
   bool performCallSlotOptzn(Instruction *cpy, Value *cpyDst, Value *cpySrc,
                             uint64_t cpyLen, unsigned cpyAlign, CallInst *C);
   bool processMemCpyMemCpyDependence(MemCpyInst *M, MemCpyInst *MDep);
   bool processMemSetMemCpyDependence(MemCpyInst *M, MemSetInst *MDep);
   bool performMemCpyToMemSetOptzn(MemCpyInst *M, MemSetInst *MDep);
   bool processByValArgument(CallSite CS, unsigned ArgNo);
   Instruction *tryMergingIntoMemset(Instruction *I, Value *StartPtr,
                                     Value *ByteVal);
 
+  void eraseInstruction(Instruction *);
+
   bool iterateOnFunction(Function &F);
 };
 }
 
 #endif // LLVM_TRANSFORMS_SCALAR_MEMCPYOPTIMIZER_H

lib/Passes/PassRegistry.def

 FUNCTION_PASS("loweratomic", LowerAtomicPass())
 FUNCTION_PASS("lower-expect", LowerExpectIntrinsicPass())
 FUNCTION_PASS("lower-guard-intrinsic", LowerGuardIntrinsicPass())
 FUNCTION_PASS("guard-widening", GuardWideningPass())
 FUNCTION_PASS("gvn", GVN())
 FUNCTION_PASS("loop-simplify", LoopSimplifyPass())
 FUNCTION_PASS("lowerinvoke", LowerInvokePass())
 FUNCTION_PASS("mem2reg", PromotePass())
-FUNCTION_PASS("memcpyopt", MemCpyOptPass())
+FUNCTION_PASS("memcpyopt", MemCpyOptPass(false))
+FUNCTION_PASS("memcpyopt-mssa", MemCpyOptPass(true))
 FUNCTION_PASS("mldst-motion", MergedLoadStoreMotionPass())
 FUNCTION_PASS("nary-reassociate", NaryReassociatePass())
 FUNCTION_PASS("jump-threading", JumpThreadingPass())
 FUNCTION_PASS("partially-inline-libcalls", PartiallyInlineLibCallsPass())
 FUNCTION_PASS("lcssa", LCSSAPass())
 FUNCTION_PASS("loop-data-prefetch", LoopDataPrefetchPass())
 FUNCTION_PASS("loop-distribute", LoopDistributePass())
 FUNCTION_PASS("loop-vectorize", LoopVectorizePass())

lib/Transforms/Scalar/MemCpyOptimizer.cpp

 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/GetElementPtrTypeIterator.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/MemorySSA.h"
 #include <algorithm>
 using namespace llvm;
 
 #define DEBUG_TYPE "memcpyopt"
 
+static cl::opt<bool> MCOMSSA("mco-mssa", cl::init(false), cl::Hidden,
+                             cl::desc("Force MemCpyOpt to use MemorySSA"));
+
 STATISTIC(NumMemCpyInstr, "Number of memcpy instructions deleted");
 STATISTIC(NumMemSetInfer, "Number of memsets inferred");
 STATISTIC(NumMoveToCpy,   "Number of memmoves converted to memcpy");
 STATISTIC(NumCpyToSet,    "Number of memcpys converted to memset");
 
 static int64_t GetOffsetFromIndex(const GEPOperator *GEP, unsigned Idx,
                                   bool &VariableIdxFound,
                                   const DataLayout &DL) {
static bool IsPointerOffset(Value *Ptr1, Value *Ptr2, int64_t &Offset,
   int64_t Offset1 = GetOffsetFromIndex(GEP1, Idx, VariableIdxFound, DL);
   int64_t Offset2 = GetOffsetFromIndex(GEP2, Idx, VariableIdxFound, DL);
   if (VariableIdxFound) return false;
 
   Offset = Offset2-Offset1;
   return true;
 }
 
+// \brief Locates the nearest MemoryAccess that clobbers \p MemLoc and strictly
+// dominates \p StartAbove
+static MemoryAccess *getCMA(MemorySSA *MSSA, MemoryUseOrDef *StartAbove,
+                            const MemoryLocation &MemLoc) {
+  MemoryAccess *Start = StartAbove->getDefiningAccess();
+  return MSSA->getWalker()->getClobberingMemoryAccess(Start, MemLoc);
+}
+
+// \brief Returns the clobber between [\p EndAt, \p StartAbove) that clobbers \p
+// MemLoc, otherwise returns \p EndAt. For this query to make sense, \p EndAt
+// must dominate \p StartAbove.
+static MemoryAccess *getCMABetween(const MemoryLocation &MemLoc,
+                                   MemoryAccess *EndAt,
+                                   MemoryUseOrDef *StartAbove,
+                                   MemorySSA *MSSA) {
+  assert(MSSA->dominates(EndAt, StartAbove) &&
+         "EndAt must dominate StartAbove.");
+  MemoryAccess *Clob = getCMA(MSSA, StartAbove, MemLoc);
+  return MSSA->dominates(EndAt, Clob) ? Clob : EndAt;
+}
 
 /// Represents a range of memset'd bytes with the ByteVal value.
 /// This allows us to analyze stores like:
 ///   store 0 -> P+1
 ///   store 0 -> P+0
 ///   store 0 -> P+3
 ///   store 0 -> P+2
 /// which sometimes happens with stores to arrays of structs etc.  When we see
 /// the first store, we make a range [1, 2).  The second store extends the range
 /// to [0, 2).  The third makes a new range [2, 3).  The fourth store joins the
 /// two ranges into [0, 3) which is memset'able.
 namespace {
 struct MemsetRange {
   // Start/End - A semi range that describes the span that this range covers.
   // The range is closed at the start and open at the end: [Start, End).
   int64_t Start, End;
 
   /// StartPtr - The getelementptr instruction that points to the start of the
   /// range.
   Value *StartPtr;
 
+  /// StartPtrUser - The insertion point for the resulting memset.
+  Instruction *StartPtrUser;
+
   /// Alignment - The known alignment of the first store.
   unsigned Alignment;
 
   /// TheStores - The actual stores that make up this range.
   SmallVector<Instruction*, 16> TheStores;
 
   bool isProfitableToUseMemset(const DataLayout &DL) const;
 };
void MemsetRanges::addRange(int64_t Start, int64_t Size, Value *Ptr,
   // We now know that I == E, in which case we didn't find anything to merge
   // with, or that Start <= I->End.  If End < I->Start or I == E, then we need
   // to insert a new range.  Handle this now.
   if (I == Ranges.end() || End < I->Start) {
     MemsetRange &R = *Ranges.insert(I, MemsetRange());
     R.Start        = Start;
     R.End          = End;
     R.StartPtr     = Ptr;
+    R.StartPtrUser = Inst;
     R.Alignment    = Alignment;
     R.TheStores.push_back(Inst);
     return;
   }
 
   // This store overlaps with I, add it.
   I->TheStores.push_back(Inst);
 
   // At this point, we may have an interval that completely contains our store.
   // If so, just add it to the interval and return.
   if (I->Start <= Start && I->End >= End)
     return;
 
   // Now we know that Start <= I->End and End >= I->Start so the range overlaps
   // but is not entirely contained within the range.
 
   // See if the range extends the start of the range.  In this case, it couldn't
   // possibly cause it to join the prior range, because otherwise we would have
   // stopped on *it*.
   if (Start < I->Start) {
     I->Start = Start;
     I->StartPtr = Ptr;
+    I->StartPtrUser = Inst;
     I->Alignment = Alignment;
   }
 
   // Now we know that Start <= I->End and Start >= I->Start (so the startpoint
   // is in or right at the end of I), and that End >= I->Start.  Extend I out to
   // End.
   if (End > I->End) {
     I->End = End;
void MemsetRanges::addRange(int64_t Start, int64_t Size, Value *Ptr,
   }
 }
 
 //===----------------------------------------------------------------------===//
 //                         MemCpyOptLegacyPass Pass
 //===----------------------------------------------------------------------===//
 
 namespace {
-  class MemCpyOptLegacyPass : public FunctionPass {
+  template <bool UseMSSA>
+  class MemCpyOptLegacyCommon : public FunctionPass {
     MemCpyOptPass Impl;
   public:
     static char ID; // Pass identification, replacement for typeid
-    MemCpyOptLegacyPass() : FunctionPass(ID) {
+    MemCpyOptLegacyCommon() : FunctionPass(ID), Impl(UseMSSA) {
+      if (UseMSSA)
+        initializeMemCpyOptMemSSALegacyPassPass(
+            *PassRegistry::getPassRegistry());
+      else
         initializeMemCpyOptLegacyPassPass(*PassRegistry::getPassRegistry());
     }
 
     bool runOnFunction(Function &F) override;
 
   private:
     // This transformation requires dominator postdominator info
     void getAnalysisUsage(AnalysisUsage &AU) const override {
       AU.setPreservesCFG();
       AU.addRequired<AssumptionCacheTracker>();
       AU.addRequired<DominatorTreeWrapperPass>();
       AU.addRequired<MemoryDependenceWrapperPass>();
       AU.addRequired<AAResultsWrapperPass>();
       AU.addRequired<TargetLibraryInfoWrapperPass>();
       AU.addPreserved<GlobalsAAWrapperPass>();
       AU.addPreserved<MemoryDependenceWrapperPass>();
+
+      if (UseMSSA) {
+        AU.addRequired<MemorySSAWrapperPass>();
+        AU.addPreserved<MemorySSAWrapperPass>();
+      }
     }
 
     // Helper functions
     bool processStore(StoreInst *SI, BasicBlock::iterator &BBI);
     bool processMemSet(MemSetInst *SI, BasicBlock::iterator &BBI);
     bool processMemCpy(MemCpyInst *M);
     bool processMemMove(MemMoveInst *M);
     bool performCallSlotOptzn(Instruction *cpy, Value *cpyDst, Value *cpySrc,
                               uint64_t cpyLen, unsigned cpyAlign, CallInst *C);
     bool processMemCpyMemCpyDependence(MemCpyInst *M, MemCpyInst *MDep);
     bool processMemSetMemCpyDependence(MemCpyInst *M, MemSetInst *MDep);
     bool performMemCpyToMemSetOptzn(MemCpyInst *M, MemSetInst *MDep);
     bool processByValArgument(CallSite CS, unsigned ArgNo);
     Instruction *tryMergingIntoMemset(Instruction *I, Value *StartPtr,
                                       Value *ByteVal);
 
     bool iterateOnFunction(Function &F);
   };
 
-  char MemCpyOptLegacyPass::ID = 0;
+  template<> char MemCpyOptLegacyCommon<false>::ID = 0;
+  template<> char MemCpyOptLegacyCommon<true>::ID = 0;
 }
 
+using MemCpyOptLegacyPass = MemCpyOptLegacyCommon<false>;
+using MemCpyOptMemSSALegacyPass = MemCpyOptLegacyCommon<true>;
+
 /// The public interface to this file...
-FunctionPass *llvm::createMemCpyOptPass() { return new MemCpyOptLegacyPass(); }
+FunctionPass *llvm::createMemCpyOptPass(bool UseMSSA) {
+  if (UseMSSA || MCOMSSA)
+    return new MemCpyOptMemSSALegacyPass();
+  return new MemCpyOptLegacyPass();
+}
 
 INITIALIZE_PASS_BEGIN(MemCpyOptLegacyPass, "memcpyopt", "MemCpy Optimization",
                       false, false)
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
 INITIALIZE_PASS_END(MemCpyOptLegacyPass, "memcpyopt", "MemCpy Optimization",
                     false, false)
 
+INITIALIZE_PASS_BEGIN(MemCpyOptMemSSALegacyPass, "memcpyopt-mssa",
+                      "MemCpy Optimization (Memory SSA)", false, false)
+INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
+INITIALIZE_PASS_END(MemCpyOptMemSSALegacyPass, "memcpyopt-mssa",
+                    "MemCpy Optimization (Memory SSA)", false, false)
+
+void MemCpyOptPass::eraseInstruction(Instruction *I) {
+  assert(MD);
+  MD->removeInstruction(I);
+  if (UseMemorySSA)
+    if (MemoryAccess *MA = MSSA->getMemoryAccess(I))
+      MSSA->removeMemoryAccess(MA);
+  I->eraseFromParent();
+}
+
+static MemoryUseOrDef *replaceMemoryAccess(MemorySSA &MSSA, Instruction *Old,
+                                           Instruction *New) {
+  MemoryUseOrDef *OldAcc = MSSA.getMemoryAccess(Old);
+  MemoryUseOrDef *NewAcc =
+      MSSA.createMemoryAccessBefore(New, OldAcc->getDefiningAccess(), OldAcc);
+  assert((isa<MemoryUse>(OldAcc) && isa<MemoryUse>(NewAcc)) ||
+         (isa<MemoryDef>(OldAcc) && isa<MemoryDef>(NewAcc)) &&
+             "Must replace with equivalent MSSA access type.");
+  OldAcc->replaceAllUsesWith(NewAcc);
+  return NewAcc;
+}
+
 /// When scanning forward over instructions, we look for some other patterns to
 /// fold away. In particular, this looks for stores to neighboring locations of
 /// memory. If it sees enough consecutive ones, it attempts to merge them
 /// together into a memcpy/memset.
 Instruction *MemCpyOptPass::tryMergingIntoMemset(Instruction *StartInst,
                                                  Value *StartPtr,
                                                  Value *ByteVal) {
   const DataLayout &DL = StartInst->getModule()->getDataLayout();
 
   // Okay, so we now have a single store that can be splatable.  Scan to find
   // all subsequent stores of the same value to offset from the same pointer.
   // Join these together into ranges, so we can decide whether contiguous blocks
   // are stored.
   MemsetRanges Ranges(DL);
 
   BasicBlock::iterator BI(StartInst);
   for (++BI; !isa<TerminatorInst>(BI); ++BI) {
+    // TODO: walk along bb AccessList instead.
     if (!isa<StoreInst>(BI) && !isa<MemSetInst>(BI)) {
       // If the instruction is readnone, ignore it, otherwise bail out.  We
       // don't even allow readonly here because we don't want something like:
       // A[1] = 2; strlen(A); A[2] = 2; -> memcpy(A, ...); strlen(A).
       if (BI->mayWriteToMemory() || BI->mayReadFromMemory())
         break;
       continue;
     }
Instruction *MemCpyOptPass::tryMergingIntoMemset(Instruction *StartInst,
   if (Ranges.empty())
     return nullptr;
 
   // If we had at least one store that could be merged in, add the starting
   // store as well.  We try to avoid this unless there is at least something
   // interesting as a small compile-time optimization.
   Ranges.addInst(0, StartInst);
 
-  // If we create any memsets, we put it right before the first instruction that
-  // isn't part of the memset block.  This ensure that the memset is dominated
-  // by any addressing instruction needed by the start of the block.
-  IRBuilder<> Builder(&*BI);
-
   // Now that we have full information about ranges, loop over the ranges and
   // emit memset's for anything big enough to be worthwhile.
   Instruction *AMemSet = nullptr;
   for (const MemsetRange &Range : Ranges) {
 
     if (Range.TheStores.size() == 1) continue;
 
     // If it is profitable to lower this range to memset, do so now.
     if (!Range.isProfitableToUseMemset(DL))
       continue;
 
     // Otherwise, we do want to transform this!  Create a new memset.
     // Get the starting pointer of the block.
     StartPtr = Range.StartPtr;
 
     // Determine alignment
     unsigned Alignment = Range.Alignment;
     if (Alignment == 0) {
       Type *EltType =
         cast<PointerType>(StartPtr->getType())->getElementType();
       Alignment = DL.getABITypeAlignment(EltType);
     }
 
-    AMemSet =
-      Builder.CreateMemSet(StartPtr, ByteVal, Range.End-Range.Start, Alignment);
+    // Insert memset just before the store in this range that uses
+    // Range.StartPtr.
+    IRBuilder<> Builder(Range.StartPtrUser);
+    AMemSet = Builder.CreateMemSet(StartPtr, ByteVal, Range.End - Range.Start,
+                                   Alignment);
+
+    if (UseMemorySSA) {
+      replaceMemoryAccess(*MSSA, Range.StartPtrUser, AMemSet);
+      MSSA->removeMemoryAccess(MSSA->getMemoryAccess(Range.StartPtrUser));
+    }
 
     DEBUG(dbgs() << "Replace stores:\n";
           for (Instruction *SI : Range.TheStores)
             dbgs() << *SI << '\n';
           dbgs() << "With: " << *AMemSet << '\n');
 
     if (!Range.TheStores.empty())
       AMemSet->setDebugLoc(Range.TheStores[0]->getDebugLoc());
 
     // Zap all the stores.
     for (Instruction *SI : Range.TheStores) {
-      MD->removeInstruction(SI);
-      SI->eraseFromParent();
+      eraseInstruction(SI);
     }
     ++NumMemSetInfer;
   }
 
   return AMemSet;
 }
 
 static unsigned findCommonAlignment(const DataLayout &DL, const StoreInst *SI,
                                      const LoadInst *LI) {
   unsigned StoreAlign = SI->getAlignment();
   if (!StoreAlign)
     StoreAlign = DL.getABITypeAlignment(SI->getOperand(0)->getType());
   unsigned LoadAlign = LI->getAlignment();
   if (!LoadAlign)
     LoadAlign = DL.getABITypeAlignment(LI->getType());
 
   return std::min(StoreAlign, LoadAlign);
 }
 
-// This method try to lift a store instruction before position P.
+// This method try to lift a store instruction before position demarcated by
+// memory instruction P.
 // It will lift the store and its argument + that anything that
 // may alias with these.
 // The method returns true if it was successful.
-static bool moveUp(AliasAnalysis &AA, StoreInst *SI, Instruction *P) {
+static bool moveUp(AliasAnalysis &AA, StoreInst *SI, Instruction *P,
+                   MemorySSA *MSSA = nullptr) {
+  MemoryDef *PAcc =
+      MSSA ? dyn_cast_or_null<MemoryDef>(MSSA->getMemoryAccess(P)) : nullptr;
+  assert((!MSSA || PAcc) && "P must be a memory def-ing instruction.");
   // If the store alias this position, early bail out.
   MemoryLocation StoreLoc = MemoryLocation::get(SI);
   if (AA.getModRefInfo(P, StoreLoc) != MRI_NoModRef)
     return false;
 
   // Keep track of the arguments of all instruction we plan to lift
   // so we can make sure to lift them as well if apropriate.
   DenseSet<Instruction*> Args;
   if (auto *Ptr = dyn_cast<Instruction>(SI->getPointerOperand()))
     if (Ptr->getParent() == SI->getParent())
       Args.insert(Ptr);
 
   // Instruction to lift before P.
-  SmallVector<Instruction*, 8> ToLift;
+  SmallVector<Instruction *, 8> ToLift{SI};
 
   // Memory locations of lifted instructions.
   SmallVector<MemoryLocation, 8> MemLocs;
   MemLocs.push_back(StoreLoc);
 
   // Lifted callsites.
   SmallVector<ImmutableCallSite, 8> CallSites;
 
for (unsigned k = 0, e = C->getNumOperands(); k != e; ++k)
         if (A->getParent() == SI->getParent())
           Args.insert(A);
   }
 
   // We made it, we need to lift
   for (auto *I : reverse(ToLift)) {
     DEBUG(dbgs() << "Lifting " << *I << " before " << *P << "\n");
     I->moveBefore(P);
+    if (MSSA) {
+      if (MemoryUseOrDef *A = MSSA->getMemoryAccess(I)) {
+        MSSA->spliceMemoryAccessAbove(PAcc, A);
+      }
+    }
   }
 
   return true;
 }
 
 bool MemCpyOptPass::processStore(StoreInst *SI, BasicBlock::iterator &BBI) {
   if (!SI->isSimple()) return false;
 
   // Avoid merging nontemporal stores since the resulting
   // memcpy/memset would not be able to preserve the nontemporal hint.
   // In theory we could teach how to propagate the !nontemporal metadata to
   // memset calls. However, that change would force the backend to
   // conservatively expand !nontemporal memset calls back to sequences of
   // store instructions (effectively undoing the merging).
   if (SI->getMetadata(LLVMContext::MD_nontemporal))
     return false;
 
   const DataLayout &DL = SI->getModule()->getDataLayout();
 
   // Load to store forwarding can be interpreted as memcpy.
   if (LoadInst *LI = dyn_cast<LoadInst>(SI->getOperand(0))) {
     if (LI->isSimple() && LI->hasOneUse() &&
+        // TODO: Make non-local if LI doms SI and SI post-doms LI.
         LI->getParent() == SI->getParent()) {
 
+      MemoryUse *LUse =
+          UseMemorySSA ? cast<MemoryUse>(MSSA->getMemoryAccess(LI)) : nullptr;
       auto *T = LI->getType();
       if (T->isAggregateType()) {
         AliasAnalysis &AA = LookupAliasAnalysis();
         MemoryLocation LoadLoc = MemoryLocation::get(LI);
+        Instruction *P = nullptr;
 
+        if (UseMemorySSA) {
+          if (auto *LClob = dyn_cast<MemoryDef>(
+                  getCMA(MSSA, MSSA->getMemoryAccess(SI), LoadLoc)))
+            P = MSSA->dominates(LUse, LClob) ? LClob->getMemoryInst() : SI;
+        } else {
           // We use alias analysis to check if an instruction may store to
           // the memory we load from in between the load and the store. If
           // such an instruction is found, we try to promote there instead
           // of at the store position.
-        Instruction *P = SI;
+          P = SI;
           for (auto &I : make_range(++LI->getIterator(), SI->getIterator())) {
             if (AA.getModRefInfo(&I, LoadLoc) & MRI_Mod) {
               P = &I;
               break;
             }
           }
+        }
 
         // We found an instruction that may write to the loaded memory.
         // We can try to promote at this position instead of the store
         // position if nothing alias the store memory after this and the store
         // destination is not in the range.
         if (P && P != SI) {
-          if (!moveUp(AA, SI, P))
+          if (!moveUp(AA, SI, P, MSSA))
             P = nullptr;
         }
 
         // If a valid insertion position is found, then we can promote
         // the load/store pair to a memcpy.
         if (P) {
           // If we load from memory that may alias the memory we store to,
           // memmove must be used to preserve semantic. If not, memcpy can
if (LI->isSimple() && LI->hasOneUse() &&
           if (UseMemMove)
             M = Builder.CreateMemMove(SI->getPointerOperand(),
                                       LI->getPointerOperand(), Size,
                                       Align, SI->isVolatile());
           else
             M = Builder.CreateMemCpy(SI->getPointerOperand(),
                                      LI->getPointerOperand(), Size,
                                      Align, SI->isVolatile());
+          if (UseMemorySSA)
+            replaceMemoryAccess(*MSSA, SI, M);
 
           DEBUG(dbgs() << "Promoting " << *LI << " to " << *SI
                        << " => " << *M << "\n");
 
-          MD->removeInstruction(SI);
-          SI->eraseFromParent();
-          MD->removeInstruction(LI);
-          LI->eraseFromParent();
+          eraseInstruction(SI);
+          eraseInstruction(LI);
           ++NumMemCpyInstr;
 
           // Make sure we do not invalidate the iterator.
           BBI = M->getIterator();
           return true;
         }
       }
 
       // Detect cases where we're performing call slot forwarding, but
       // happen to be using a load-store pair to implement it, rather than
       // a memcpy.
-      MemDepResult ldep = MD->getDependency(LI);
       CallInst *C = nullptr;
+      if (UseMemorySSA) {
+        if (MemoryUseOrDef *LoadClob = dyn_cast<MemoryUseOrDef>(
+                MSSA->getWalker()->getClobberingMemoryAccess(LUse)))
+          // TODO: This funnels to performCallSlotOptzn, whose transform is
+          // valid as long as LI doms SI and SI post-doms LI.
+          if (LoadClob->getBlock() == SI->getParent())
+            C = dyn_cast_or_null<CallInst>(LoadClob->getMemoryInst());
+      } else {
+        MemDepResult ldep = MD->getDependency(LI);
         if (ldep.isClobber() && !isa<MemCpyInst>(ldep.getInst()))
           C = dyn_cast<CallInst>(ldep.getInst());
+      }
 
       if (C) {
         // Check that nothing touches the dest of the "copy" between
         // the call and the store.
         Value *CpyDest = SI->getPointerOperand()->stripPointerCasts();
         bool CpyDestIsLocal = isa<AllocaInst>(CpyDest);
         AliasAnalysis &AA = LookupAliasAnalysis();
         MemoryLocation StoreLoc = MemoryLocation::get(SI);
if (LI->isSimple() && LI->hasOneUse() &&
 
       if (C) {
         bool changed = performCallSlotOptzn(
             LI, SI->getPointerOperand()->stripPointerCasts(),
             LI->getPointerOperand()->stripPointerCasts(),
             DL.getTypeStoreSize(SI->getOperand(0)->getType()),
             findCommonAlignment(DL, SI, LI), C);
         if (changed) {
-          MD->removeInstruction(SI);
-          SI->eraseFromParent();
-          MD->removeInstruction(LI);
-          LI->eraseFromParent();
+          eraseInstruction(SI);
+          eraseInstruction(LI);
           ++NumMemCpyInstr;
           return true;
         }
       }
     }
   }
 
   // There are two cases that are interesting for this code to handle: memcpy
if (Value *ByteVal = isBytewiseValue(V)) {
     if (T->isAggregateType()) {
       uint64_t Size = DL.getTypeStoreSize(T);
       unsigned Align = SI->getAlignment();
       if (!Align)
         Align = DL.getABITypeAlignment(T);
       IRBuilder<> Builder(SI);
       auto *M = Builder.CreateMemSet(SI->getPointerOperand(), ByteVal,
                                      Size, Align, SI->isVolatile());
+      if (UseMemorySSA)
+        replaceMemoryAccess(*MSSA, SI, M);
 
       DEBUG(dbgs() << "Promoting " << *SI << " to " << *M << "\n");
 
-      MD->removeInstruction(SI);
-      SI->eraseFromParent();
+      eraseInstruction(SI);
       NumMemSetInfer++;
 
       // Make sure we do not invalidate the iterator.
       BBI = M->getIterator();
       return true;
     }
   }
 
bool MemCpyOptPass::performCallSlotOptzn(Instruction *cpy, Value *cpyDest,
   // Update AA metadata
   // FIXME: MD_tbaa_struct and MD_mem_parallel_loop_access should also be
   // handled here, but combineMetadata doesn't support them yet
   unsigned KnownIDs[] = {LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
                          LLVMContext::MD_noalias,
                          LLVMContext::MD_invariant_group};
   combineMetadata(C, cpy, KnownIDs);
 
-  // Remove the memcpy.
-  MD->removeInstruction(cpy);
-  ++NumMemCpyInstr;
-
+  // memcpy removal left to caller.
   return true;
 }
 
 /// We've found that the (upward scanning) memory dependence of memcpy 'M' is
 /// the memcpy 'MDep'. Try to simplify M to copy from MDep's input if we can.
 bool MemCpyOptPass::processMemCpyMemCpyDependence(MemCpyInst *M,
                                                   MemCpyInst *MDep) {
   // We can only transforms memcpy's where the dest of one is the source of the
bool MemCpyOptPass::processMemCpyMemCpyDependence(MemCpyInst *M,
   //    memcpy(c <- a)
   // It would be invalid to transform the second memcpy into memcpy(c <- b).
   //
   // TODO: If the code between M and MDep is transparent to the destination "c",
   // then we could still perform the xform by moving M up to the first memcpy.
   //
   // NOTE: This is conservative, it will stop on any read from the source loc,
   // not just the defining memcpy.
+  if (UseMemorySSA) {
+    MemoryUseOrDef *DepAcc = MSSA->getMemoryAccess(MDep);
+    if (getCMABetween(MemoryLocation::getForSource(MDep), DepAcc,
+                      MSSA->getMemoryAccess(M), MSSA) != DepAcc)
+      return false;
+  } else {
     MemDepResult SourceDep =
         MD->getPointerDependencyFrom(MemoryLocation::getForSource(MDep), false,
                                      M->getIterator(), M->getParent());
     if (!SourceDep.isClobber() || SourceDep.getInst() != MDep)
       return false;
+  }
 
   // If the dest of the second might alias the source of the first, then the
   // source and dest might overlap.  We still want to eliminate the intermediate
   // value, but we have to generate a memmove instead of memcpy.
   bool UseMemMove = false;
   if (!AA.isNoAlias(MemoryLocation::getForDest(M),
                     MemoryLocation::getForSource(MDep)))
     UseMemMove = true;
 
   // If all checks passed, then we can transform M.
 
   // Make sure to use the lesser of the alignment of the source and the dest
   // since we're changing where we're reading from, but don't want to increase
   // the alignment past what can be read from or written to.
   // TODO: Is this worth it if we're creating a less aligned memcpy? For
   // example we could be moving from movaps -> movq on x86.
   unsigned Align = std::min(MDep->getAlignment(), M->getAlignment());
 
   IRBuilder<> Builder(M);
-  if (UseMemMove)
-    Builder.CreateMemMove(M->getRawDest(), MDep->getRawSource(), M->getLength(),
-                          Align, M->isVolatile());
-  else
-    Builder.CreateMemCpy(M->getRawDest(), MDep->getRawSource(), M->getLength(),
-                         Align, M->isVolatile());
+  auto *New =
+      UseMemMove
+          ? Builder.CreateMemMove(M->getRawDest(), MDep->getRawSource(),
+                                  M->getLength(), Align, M->isVolatile())
+          : Builder.CreateMemCpy(M->getRawDest(), MDep->getRawSource(),
+                                 M->getLength(), Align, M->isVolatile());
+  if (UseMemorySSA)
+    replaceMemoryAccess(*MSSA, M, New);
 
   // Remove the instruction we're replacing.
-  MD->removeInstruction(M);
-  M->eraseFromParent();
+  eraseInstruction(M);
   ++NumMemCpyInstr;
   return true;
 }
 
 /// We've found that the (upward scanning) memory dependence of \p MemCpy is
 /// \p MemSet.  Try to simplify \p MemSet to only set the trailing bytes that
 /// weren't copied over by \p MemCpy.
 ///
 /// In other words, transform:
 /// \code
 ///   memset(dst, c, dst_size);
 ///   memcpy(dst, src, src_size);
 /// \endcode
 /// into:
 /// \code
-///   memcpy(dst, src, src_size);
 ///   memset(dst + src_size, c, dst_size <= src_size ? 0 : dst_size - src_size);
+///   memcpy(dst, src, src_size);
 /// \endcode
 bool MemCpyOptPass::processMemSetMemCpyDependence(MemCpyInst *MemCpy,
                                                   MemSetInst *MemSet) {
   // We can only transform memset/memcpy with the same destination.
   if (MemSet->getDest() != MemCpy->getDest())
     return false;
 
   // Check that there are no other dependencies on the memset destination.
-  MemDepResult DstDepInfo =
-      MD->getPointerDependencyFrom(MemoryLocation::getForDest(MemSet), false,
-                                   MemCpy->getIterator(), MemCpy->getParent());
+  if (UseMemorySSA) {
+    MemoryUseOrDef *MSAcc = MSSA->getMemoryAccess(MemSet);
+    MemoryUseOrDef *MemCpyAcc = MSSA->getMemoryAccess(MemCpy);
+    assert(MSSA->dominates(MSAcc, MemCpyAcc));
+
+    // TODO: non-local
+    if (MemCpy->getParent() != MemSet->getParent())
+      return false;
+
+    AliasAnalysis &AA = LookupAliasAnalysis();
+    using It = MemorySSA::AccessList::iterator;
+    for (const auto &Acc : make_range(std::next(It(MSAcc)), It(MemCpyAcc))) {
+      if (AA.getModRefInfo(cast<MemoryUseOrDef>(Acc).getMemoryInst(),
+                           MemoryLocation::getForDest(MemSet)) != MRI_NoModRef)
+        return false;
+    }
+  } else {
+    MemDepResult DstDepInfo = MD->getPointerDependencyFrom(
+        MemoryLocation::getForDest(MemSet), false, MemCpy->getIterator(),
+        MemCpy->getParent());
     if (DstDepInfo.getInst() != MemSet)
       return false;
+  }
 
   // Use the same i8* dest as the memcpy, killing the memset dest if different.
   Value *Dest = MemCpy->getRawDest();
   Value *DestSize = MemSet->getLength();
   Value *SrcSize = MemCpy->getLength();
 
   // By default, create an unaligned memset.
   unsigned Align = 1;
   // If Dest is aligned, and SrcSize is constant, use the minimum alignment
   // of the sum.
   const unsigned DestAlign =
       std::max(MemSet->getAlignment(), MemCpy->getAlignment());
   if (DestAlign > 1)
     if (ConstantInt *SrcSizeC = dyn_cast<ConstantInt>(SrcSize))
       Align = MinAlign(SrcSizeC->getZExtValue(), DestAlign);
 
-  IRBuilder<> Builder(MemCpy);
+  // MSSA replacement requires new memset to be inserted next to old.
+  IRBuilder<> Builder(MemSet);
 
   // If the sizes have different types, zext the smaller one.
   if (DestSize->getType() != SrcSize->getType()) {
     if (DestSize->getType()->getIntegerBitWidth() >
         SrcSize->getType()->getIntegerBitWidth())
       SrcSize = Builder.CreateZExt(SrcSize, DestSize->getType());
     else
       DestSize = Builder.CreateZExt(DestSize, SrcSize->getType());
   }
 
   Value *MemsetLen =
       Builder.CreateSelect(Builder.CreateICmpULE(DestSize, SrcSize),
                            ConstantInt::getNullValue(DestSize->getType()),
                            Builder.CreateSub(DestSize, SrcSize));
-  Builder.CreateMemSet(Builder.CreateGEP(Dest, SrcSize), MemSet->getOperand(1),
-                       MemsetLen, Align);
+  auto *M = Builder.CreateMemSet(Builder.CreateGEP(Dest, SrcSize),
+                                 MemSet->getOperand(1), MemsetLen, Align);
+  if (UseMemorySSA)
+    replaceMemoryAccess(*MSSA, MemSet, M);
 
-  MD->removeInstruction(MemSet);
-  MemSet->eraseFromParent();
+  eraseInstruction(MemSet);
   return true;
 }
 
 /// Transform memcpy to memset when its source was just memset.
 /// In other words, turn:
 /// \code
 ///   memset(dst1, c, dst1_size);
 ///   memcpy(dst2, dst1, dst2_size);
bool MemCpyOptPass::performMemCpyToMemSetOptzn(MemCpyInst *MemCpy,
   ConstantInt *CopySize = cast<ConstantInt>(MemCpy->getLength());
   ConstantInt *MemSetSize = dyn_cast<ConstantInt>(MemSet->getLength());
   // Make sure the memcpy doesn't read any more than what the memset wrote.
   // Don't worry about sizes larger than i64.
   if (!MemSetSize || CopySize->getZExtValue() > MemSetSize->getZExtValue())
     return false;
 
   IRBuilder<> Builder(MemCpy);
+  auto *MemSetNew =
       Builder.CreateMemSet(MemCpy->getRawDest(), MemSet->getOperand(1),
                            CopySize, MemCpy->getAlignment());
+  if (UseMemorySSA)
+    replaceMemoryAccess(*MSSA, MemCpy, MemSetNew);
+  eraseInstruction(MemCpy);
+  ++NumCpyToSet;
   return true;
 }
 
 /// Perform simplification of memcpy's.  If we have memcpy A
 /// which copies X to Y, and memcpy B which copies Y to Z, then we can rewrite
 /// B to be a memcpy from X to Z (or potentially a memmove, depending on
 /// circumstances). This allows later passes to remove the first memcpy
 /// altogether.
 bool MemCpyOptPass::processMemCpy(MemCpyInst *M) {
   // We can only optimize non-volatile memcpy's.
   if (M->isVolatile()) return false;
 
   // If the source and destination of the memcpy are the same, then zap it.
   if (M->getSource() == M->getDest()) {
-    MD->removeInstruction(M);
-    M->eraseFromParent();
+    eraseInstruction(M);
     return false;
   }
 
   // If copying from a constant, try to turn the memcpy into a memset.
   if (GlobalVariable *GV = dyn_cast<GlobalVariable>(M->getSource()))
     if (GV->isConstant() && GV->hasDefinitiveInitializer())
       if (Value *ByteVal = isBytewiseValue(GV->getInitializer())) {
         IRBuilder<> Builder(M);
         Builder.CreateMemSet(M->getRawDest(), ByteVal, M->getLength(),
                              M->getAlignment(), false);
-        MD->removeInstruction(M);
-        M->eraseFromParent();
+        eraseInstruction(M);
         ++NumCpyToSet;
         return true;
       }
 
   MemDepResult DepInfo = MD->getDependency(M);
 
   // Try to turn a partially redundant memset + memcpy into
   // memcpy + smaller memset.  We don't need the memcpy size for this.
bool MemCpyOptPass::processMemCpy(MemCpyInst *M) {
   //      lifetime copies undefined data, and we can therefore eliminate the
   //      memcpy in favor of the data that was already at the destination.
   //   d) memcpy from a just-memset'd source can be turned into memset.
   if (DepInfo.isClobber()) {
     if (CallInst *C = dyn_cast<CallInst>(DepInfo.getInst())) {
       if (performCallSlotOptzn(M, M->getDest(), M->getSource(),
                                CopySize->getZExtValue(), M->getAlignment(),
                                C)) {
-        MD->removeInstruction(M);
-        M->eraseFromParent();
+        eraseInstruction(M);
+        ++NumMemCpyInstr;
         return true;
       }
     }
   }
 
   MemoryLocation SrcLoc = MemoryLocation::getForSource(M);
   MemDepResult SrcDepInfo = MD->getPointerDependencyFrom(
       SrcLoc, true, M->getIterator(), M->getParent());
if (SrcDepInfo.isClobber()) {
     } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
       if (II->getIntrinsicID() == Intrinsic::lifetime_start)
         if (ConstantInt *LTSize = dyn_cast<ConstantInt>(II->getArgOperand(0)))
           if (LTSize->getZExtValue() >= CopySize->getZExtValue())
             hasUndefContents = true;
     }
 
     if (hasUndefContents) {
-      MD->removeInstruction(M);
-      M->eraseFromParent();
+      eraseInstruction(M);
       ++NumMemCpyInstr;
       return true;
     }
   }
 
   if (SrcDepInfo.isClobber())
     if (MemSetInst *MDep = dyn_cast<MemSetInst>(SrcDepInfo.getInst()))
-      if (performMemCpyToMemSetOptzn(M, MDep)) {
-        MD->removeInstruction(M);
-        M->eraseFromParent();
+      if (performMemCpyToMemSetOptzn(M, MDep))
+          return true;
+
+  return false;
+}
+
+bool MemCpyOptPass::processMemCpyMSSA(MemCpyInst *M) {
+  if (M->isVolatile())
+    return false;
+
+  if (M->getSource() == M->getDest()) {
+    eraseInstruction(M);
+    return false;
+  }
+
+  // If copying from a constant, try to turn the memcpy into a memset.
+  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(M->getSource()))
+    if (GV->isConstant() && GV->hasDefinitiveInitializer())
+      if (Value *ByteVal = isBytewiseValue(GV->getInitializer())) {
+        IRBuilder<> Builder(M);
+        auto *MemSet = Builder.CreateMemSet(
+            M->getRawDest(), ByteVal, M->getLength(), M->getAlignment(), false);
+        replaceMemoryAccess(*MSSA, M, MemSet);
+        eraseInstruction(M);
         ++NumCpyToSet;
         return true;
       }
 
+  MemoryAccess *DestClob =
+      getCMA(MSSA, MSSA->getMemoryAccess(M), MemoryLocation::getForDest(M));
+
+  // TODO: This can be made non-local if M post-doms MDep
+  if (DestClob->getBlock() == M->getParent())
+    if (auto *MUD = dyn_cast<MemoryUseOrDef>(DestClob))
+      if (auto *MDep = dyn_cast_or_null<MemSetInst>(MUD->getMemoryInst()))
+        if (processMemSetMemCpyDependence(M, MDep))
+          return true;
+
+  // The optimizations after this point require the memcpy size.
+  ConstantInt *CopySize = dyn_cast<ConstantInt>(M->getLength());
+  if (!CopySize)
+    return false;
+
+  MemoryUseOrDef *MAcc = MSSA->getMemoryAccess(M);
+  MemoryAccess *SrcClob = getCMA(MSSA, MAcc, MemoryLocation::getForSource(M));
+
+  if (auto *MUD = dyn_cast<MemoryUseOrDef>(SrcClob)) {
+    if (auto *C = dyn_cast_or_null<CallInst>(MUD->getMemoryInst())) {
+      // TODO: Can be made non-local if M post-doms C
+      if (C->getParent() == M->getParent() &&
+          performCallSlotOptzn(M, M->getDest(), M->getSource(),
+                               CopySize->getZExtValue(), M->getAlignment(),
+                               C)) {
+        eraseInstruction(M);
+        ++NumMemCpyInstr;
+        return true;
+      }
+    }
+
+    // Non-local permitted since processMemCpyMemCpyDependence modifies M only
+    // and getCMA ensures that MDep doms M.
+    if (auto *MDep = dyn_cast_or_null<MemCpyInst>(MUD->getMemoryInst()))
+      if (processMemCpyMemCpyDependence(M, MDep))
+        return true;
+
+    if (auto *MDep = dyn_cast_or_null<MemSetInst>(MUD->getMemoryInst()))
+      if (performMemCpyToMemSetOptzn(M, MDep))
+        return true;
+
+    const DataLayout &DL = M->getParent()->getModule()->getDataLayout();
+    // TODO: test these.
+    bool hasUndefContents =
+        MSSA->isLiveOnEntryDef(MUD) &&
+        isa<AllocaInst>(GetUnderlyingObject(M->getRawSource(), DL));
+
+    // Both MUD and and lifetime_start (if one exists for M's source) dominate
+    // MAcc. The only way for lifetime_start to imply undef contents is if it
+    // resides between MUD and MAcc.
+    for (MemoryUseOrDef *L =
+             dyn_cast_or_null<MemoryUseOrDef>(MAcc->getDefiningAccess());
+         L && L != MUD;
+         L = dyn_cast_or_null<MemoryUseOrDef>(L->getDefiningAccess())) {
+      if (auto *II = dyn_cast_or_null<IntrinsicInst>(L->getMemoryInst())) {
+        if (II->getIntrinsicID() == Intrinsic::lifetime_start &&
+            II->getArgOperand(1)->stripPointerCasts() == M->getSource()) {
+          if (ConstantInt *LTSize =
+                  dyn_cast<ConstantInt>(II->getArgOperand(0))) {
+            hasUndefContents |=
+                LTSize->getZExtValue() >= CopySize->getZExtValue();
+            break;
+          }
+        }
+      }
+    }
+
+    if (hasUndefContents) {
+      eraseInstruction(M);
+      ++NumMemCpyInstr;
+      return true;
+    }
+  }
+
   return false;
 }
 
 /// Transforms memmove calls to memcpy calls when the src/dst are guaranteed
 /// not to alias.
 bool MemCpyOptPass::processMemMove(MemMoveInst *M) {
   AliasAnalysis &AA = LookupAliasAnalysis();
 

 
 /// This is called on every byval argument in call sites.
 bool MemCpyOptPass::processByValArgument(CallSite CS, unsigned ArgNo) {
   const DataLayout &DL = CS.getCaller()->getParent()->getDataLayout();
   // Find out what feeds this byval argument.
   Value *ByValArg = CS.getArgument(ArgNo);
   Type *ByValTy = cast<PointerType>(ByValArg->getType())->getElementType();
   uint64_t ByValSize = DL.getTypeAllocSize(ByValTy);
+  MemoryLocation ByValLoc(ByValArg, ByValSize);
+  MemCpyInst *MDep = nullptr;
+
+  if (UseMemorySSA) {
+    if (MemoryUseOrDef *MUD = MSSA->getMemoryAccess(CS.getInstruction()))
+      if (auto *ByValClob =
+              dyn_cast<MemoryUseOrDef>(getCMA(MSSA, MUD, ByValLoc)))
+        // TODO: Non-local: It's sufficient for MDep to dominate CS.
+        if (ByValClob->getBlock() == CS.getParent())
+          MDep = dyn_cast_or_null<MemCpyInst>(ByValClob->getMemoryInst());
+  } else {
     MemDepResult DepInfo = MD->getPointerDependencyFrom(
-      MemoryLocation(ByValArg, ByValSize), true,
-      CS.getInstruction()->getIterator(), CS.getInstruction()->getParent());
+        ByValLoc, true, CS.getInstruction()->getIterator(),
+        CS.getInstruction()->getParent());
     if (!DepInfo.isClobber())
       return false;
+    MDep = dyn_cast<MemCpyInst>(DepInfo.getInst());
+  }
 
   // If the byval argument isn't fed by a memcpy, ignore it.  If it is fed by
   // a memcpy, see if we can byval from the source of the memcpy instead of the
   // result.
-  MemCpyInst *MDep = dyn_cast<MemCpyInst>(DepInfo.getInst());
   if (!MDep || MDep->isVolatile() ||
       ByValArg->stripPointerCasts() != MDep->getDest())
     return false;
 
   // The length of the memcpy must be larger or equal to the size of the byval.
   ConstantInt *C1 = dyn_cast<ConstantInt>(MDep->getLength());
   if (!C1 || C1->getValue().getZExtValue() < ByValSize)
     return false;
bool MemCpyOptPass::processByValArgument(CallSite CS, unsigned ArgNo) {
   // the byval call.
   //    memcpy(a <- b)
   //    *b = 42;
   //    foo(*a)
   // It would be invalid to transform the second memcpy into foo(*b).
   //
   // NOTE: This is conservative, it will stop on any read from the source loc,
   // not just the defining memcpy.
+  if (UseMemorySSA) {
+    MemoryUseOrDef *Dep = MSSA->getMemoryAccess(MDep);
+    if (getCMABetween(MemoryLocation::getForSource(MDep), Dep,
+                      MSSA->getMemoryAccess(CS.getInstruction()), MSSA) != Dep)
+      return false;
+  } else {
     MemDepResult SourceDep = MD->getPointerDependencyFrom(
         MemoryLocation::getForSource(MDep), false,
         CS.getInstruction()->getIterator(), MDep->getParent());
     if (!SourceDep.isClobber() || SourceDep.getInst() != MDep)
       return false;
+  }
 
   Value *TmpCast = MDep->getSource();
   if (MDep->getSource()->getType() != ByValArg->getType())
     TmpCast = new BitCastInst(MDep->getSource(), ByValArg->getType(),
                               "tmpcast", CS.getInstruction());
 
   DEBUG(dbgs() << "MemCpyOptPass: Forwarding memcpy to byval:\n"
                << "  " << *MDep << "\n"
for (BasicBlock::iterator BI = BB.begin(), BE = BB.end(); BI != BE;) {
 
       bool RepeatInstruction = false;
 
       if (StoreInst *SI = dyn_cast<StoreInst>(I))
         MadeChange |= processStore(SI, BI);
       else if (MemSetInst *M = dyn_cast<MemSetInst>(I))
         RepeatInstruction = processMemSet(M, BI);
       else if (MemCpyInst *M = dyn_cast<MemCpyInst>(I))
-        RepeatInstruction = processMemCpy(M);
+        RepeatInstruction =
+            UseMemorySSA ? processMemCpyMSSA(M) : processMemCpy(M);
       else if (MemMoveInst *M = dyn_cast<MemMoveInst>(I))
         RepeatInstruction = processMemMove(M);
       else if (auto CS = CallSite(I)) {
         for (unsigned i = 0, e = CS.arg_size(); i != e; ++i)
           if (CS.isByValArgument(i))
             MadeChange |= processByValArgument(CS, i);
       }
 
       // Reprocess the instruction if desired.
       if (RepeatInstruction) {
         if (BI != BB.begin())
           --BI;
         MadeChange = true;
       }
     }
   }
 
   return MadeChange;
 }
 
 PreservedAnalyses MemCpyOptPass::run(Function &F, FunctionAnalysisManager &AM) {
 
+  auto *MSSA =
+      UseMemorySSA ? &AM.getResult<MemorySSAAnalysis>(F).getMSSA() : nullptr;
   auto &MD = AM.getResult<MemoryDependenceAnalysis>(F);
   auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
 
   auto LookupAliasAnalysis = [&]() -> AliasAnalysis & {
     return AM.getResult<AAManager>(F);
   };
   auto LookupAssumptionCache = [&]() -> AssumptionCache & {
     return AM.getResult<AssumptionAnalysis>(F);
   };
   auto LookupDomTree = [&]() -> DominatorTree & {
     return AM.getResult<DominatorTreeAnalysis>(F);
   };
 
-  bool MadeChange = runImpl(F, &MD, &TLI, LookupAliasAnalysis,
+  bool MadeChange = runImpl(F, MSSA, &MD, &TLI, LookupAliasAnalysis,
                             LookupAssumptionCache, LookupDomTree);
   if (!MadeChange)
     return PreservedAnalyses::all();
   PreservedAnalyses PA;
   PA.preserve<GlobalsAA>();
   PA.preserve<MemoryDependenceAnalysis>();
+
+  if (UseMemorySSA)
+    PA.preserve<MemorySSAAnalysis>();
+
   return PA;
 }
 
 bool MemCpyOptPass::runImpl(
-    Function &F, MemoryDependenceResults *MD_, TargetLibraryInfo *TLI_,
+    Function &F, MemorySSA *MSSA_, MemoryDependenceResults *MD_,
+    TargetLibraryInfo *TLI_,
     std::function<AliasAnalysis &()> LookupAliasAnalysis_,
     std::function<AssumptionCache &()> LookupAssumptionCache_,
     std::function<DominatorTree &()> LookupDomTree_) {
   bool MadeChange = false;
+  MSSA = MSSA_;
   MD = MD_;
   TLI = TLI_;
   LookupAliasAnalysis = std::move(LookupAliasAnalysis_);
   LookupAssumptionCache = std::move(LookupAssumptionCache_);
   LookupDomTree = std::move(LookupDomTree_);
 
   // If we don't have at least memset and memcpy, there is little point of doing
   // anything here.  These are required by a freestanding implementation, so if
   // even they are disabled, there is no point in trying hard.
   if (!TLI->has(LibFunc::memset) || !TLI->has(LibFunc::memcpy))
     return false;
 
   while (1) {
     if (!iterateOnFunction(F))
       break;
     MadeChange = true;
   }
 
   MD = nullptr;
+  MSSA = nullptr;
   return MadeChange;
 }
 
 /// This is the main transformation entry point for a function.
-bool MemCpyOptLegacyPass::runOnFunction(Function &F) {
+template <bool UseMSSA>
+bool MemCpyOptLegacyCommon<UseMSSA>::runOnFunction(Function &F) {
   if (skipFunction(F))
     return false;
 
+  auto *MSSA =
+      UseMSSA ? &getAnalysis<MemorySSAWrapperPass>().getMSSA() : nullptr;
   auto *MD = &getAnalysis<MemoryDependenceWrapperPass>().getMemDep();
   auto *TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
 
   auto LookupAliasAnalysis = [this]() -> AliasAnalysis & {
     return getAnalysis<AAResultsWrapperPass>().getAAResults();
   };
   auto LookupAssumptionCache = [this, &F]() -> AssumptionCache & {
     return getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
   };
   auto LookupDomTree = [this]() -> DominatorTree & {
     return getAnalysis<DominatorTreeWrapperPass>().getDomTree();
   };
 
-  return Impl.runImpl(F, MD, TLI, LookupAliasAnalysis, LookupAssumptionCache,
-                      LookupDomTree);
+  return Impl.runImpl(F, MSSA, MD, TLI, LookupAliasAnalysis,
+                      LookupAssumptionCache, LookupDomTree);
 }

lib/Transforms/Scalar/Scalar.cpp

void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeLoopUnrollPass(Registry);
   initializeLoopUnswitchPass(Registry);
   initializeLoopVersioningLICMPass(Registry);
   initializeLoopIdiomRecognizeLegacyPassPass(Registry);
   initializeLowerAtomicLegacyPassPass(Registry);
   initializeLowerExpectIntrinsicPass(Registry);
   initializeLowerGuardIntrinsicLegacyPassPass(Registry);
   initializeMemCpyOptLegacyPassPass(Registry);
+  initializeMemCpyOptMemSSALegacyPassPass(Registry);
   initializeMergedLoadStoreMotionLegacyPassPass(Registry);
   initializeNaryReassociateLegacyPassPass(Registry);
   initializePartiallyInlineLibCallsLegacyPassPass(Registry);
   initializeReassociateLegacyPassPass(Registry);
   initializeRegToMemPass(Registry);
   initializeRewriteStatepointsForGCPass(Registry);
   initializeSCCPLegacyPassPass(Registry);
   initializeIPSCCPLegacyPassPass(Registry);
void LLVMAddLoopUnrollPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createLoopUnrollPass());
 }
 
 void LLVMAddLoopUnswitchPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createLoopUnswitchPass());
 }
 
 void LLVMAddMemCpyOptPass(LLVMPassManagerRef PM) {
-  unwrap(PM)->add(createMemCpyOptPass());
+  unwrap(PM)->add(createMemCpyOptPass(false));
+}
+
+void LLVMAddMemCpyOptMemSSAPass(LLVMPassManagerRef PM) {
+  unwrap(PM)->add(createMemCpyOptPass(true));
 }
 
 void LLVMAddPartiallyInlineLibCallsPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createPartiallyInlineLibCallsPass());
 }
 
 void LLVMAddLowerSwitchPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createLowerSwitchPass());

test/Analysis/TypeBasedAliasAnalysis/memcpyopt.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -tbaa -basicaa -memcpyopt -instcombine < %s | FileCheck %s
+; RUN: opt -S -tbaa -basicaa -memcpyopt-mssa -instcombine < %s | FileCheck %s --check-prefix=MCO-MSSA
 
 target datalayout = "e-p:64:64:64"
 
 ; The second memcpy is redundant and can be deleted. There's an intervening store, but
 ; it has a TBAA tag which declares that it is unrelated.
 
 define void @foo(i8* nocapture %p, i8* nocapture %q, i8* nocapture %s) nounwind {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %p, i8* %q, i64 16, i32 1, i1 false), !tbaa !0
 ; CHECK-NEXT:    store i8 2, i8* %s, align 1, !tbaa !3
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @foo(
+; MCO-MSSA-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %p, i8* %q, i64 16, i32 1, i1 false), !tbaa !0
+; MCO-MSSA-NEXT:    store i8 2, i8* %s, align 1, !tbaa !3
+; MCO-MSSA-NEXT:    ret void
+;
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %p, i8* %q, i64 16, i32 1, i1 false), !tbaa !2
   store i8 2, i8* %s, align 1, !tbaa !1
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %q, i8* %p, i64 16, i32 1, i1 false), !tbaa !2
   ret void
 }
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind
 
 !0 = !{!"tbaa root", null}
 !1 = !{!3, !3, i64 0}
 !2 = !{!4, !4, i64 0}
 !3 = !{!"A", !0}
 !4 = !{!"B", !0}

test/Transforms/GVN/pr24426.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -memcpyopt -mldst-motion -gvn -S | FileCheck %s
+; RUN: opt < %s -memcpyopt-mssa -mldst-motion -gvn -S | FileCheck %s --check-prefix=MCO-MSSA
 
 declare void @check(i8)
 
 declare void @write(i8* %res)
 
 define void @test1() {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = alloca [10 x i8]
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast [10 x i8]* [[TMP1]] to i8*
 ; CHECK-NEXT:    call void @write(i8* [[TMP2]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i8, i8* [[TMP2]]
 ; CHECK-NEXT:    call void @check(i8 [[TMP3]])
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test1(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = alloca [10 x i8]
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = bitcast [10 x i8]* [[TMP1]] to i8*
+; MCO-MSSA-NEXT:    call void @write(i8* [[TMP2]])
+; MCO-MSSA-NEXT:    [[TMP3:%.*]] = load i8, i8* [[TMP2]]
+; MCO-MSSA-NEXT:    call void @check(i8 [[TMP3]])
+; MCO-MSSA-NEXT:    ret void
+;
   %1 = alloca [10 x i8]
   %2 = bitcast [10 x i8]* %1 to i8*
   call void @write(i8* %2)
   %3 = load i8, i8* %2
 
   call void @check(i8 %3)
 
   ret void
 }
 

test/Transforms/MemCpyOpt/2008-02-24-MultipleUseofSRet.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -memcpyopt -dse -S | grep "call.*initialize" | not grep memtmp
+; RUN: opt < %s -basicaa -memcpyopt-mssa -dse -S | grep "call.*initialize" | not grep memtmp
 ; PR2077
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
 target triple = "i386-pc-linux-gnu"
 
 %0 = type { x86_fp80, x86_fp80 }
 
 define internal fastcc void @initialize(%0* noalias nocapture sret %agg.result) nounwind {

test/Transforms/MemCpyOpt/2008-03-13-ReturnSlotBitcast.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -memcpyopt -S | not grep "call.*memcpy."
+; RUN: opt < %s -basicaa -memcpyopt-mssa -S | not grep "call.*memcpy."
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64"
 
 %a = type { i32 }
 %b = type { float }
 
 declare void @g(%a* nocapture)
 
 define float @f() {

test/Transforms/MemCpyOpt/2011-06-02-CallSlotOverwritten.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -memcpyopt -S | FileCheck %s
+; RUN: opt < %s -basicaa -memcpyopt-mssa -S | FileCheck %s --check-prefix=MCO-MSSA
 ; PR10067
 ; Make sure the call+copy isn't optimized in such a way that
 ; %ret ends up with the wrong value.
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
 target triple = "i386-apple-darwin10"
 
 %struct1 = type { i32, i32 }

 ; CHECK-NEXT:    [[BIT1:%.*]] = bitcast %struct1* [[X]] to i64*
 ; CHECK-NEXT:    [[BIT2:%.*]] = bitcast %struct2* [[Y]] to i64*
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i64, i64* [[BIT1]], align 8
 ; CHECK-NEXT:    store i64 [[LOAD]], i64* [[BIT2]], align 8
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr %struct2, %struct2* [[Y]], i32 0, i32 0, i32 0
 ; CHECK-NEXT:    [[RET:%.*]] = load i32, i32* [[GEP1]]
 ; CHECK-NEXT:    ret i32 [[RET]]
 ;
+; MCO-MSSA-LABEL: @foo(
+; MCO-MSSA-NEXT:    [[X:%.*]] = alloca %struct1, align 8
+; MCO-MSSA-NEXT:    [[Y:%.*]] = alloca %struct2, align 8
+; MCO-MSSA-NEXT:    call void @bar(%struct1* sret [[X]]) #0
+; MCO-MSSA-NEXT:    [[GEPN1:%.*]] = getelementptr inbounds %struct2, %struct2* [[Y]], i32 0, i32 0, i32 0
+; MCO-MSSA-NEXT:    store i32 0, i32* [[GEPN1]], align 8
+; MCO-MSSA-NEXT:    [[GEPN2:%.*]] = getelementptr inbounds %struct2, %struct2* [[Y]], i32 0, i32 0, i32 1
+; MCO-MSSA-NEXT:    store i32 0, i32* [[GEPN2]], align 4
+; MCO-MSSA-NEXT:    [[BIT1:%.*]] = bitcast %struct1* [[X]] to i64*
+; MCO-MSSA-NEXT:    [[BIT2:%.*]] = bitcast %struct2* [[Y]] to i64*
+; MCO-MSSA-NEXT:    [[LOAD:%.*]] = load i64, i64* [[BIT1]], align 8
+; MCO-MSSA-NEXT:    store i64 [[LOAD]], i64* [[BIT2]], align 8
+; MCO-MSSA-NEXT:    [[GEP1:%.*]] = getelementptr %struct2, %struct2* [[Y]], i32 0, i32 0, i32 0
+; MCO-MSSA-NEXT:    [[RET:%.*]] = load i32, i32* [[GEP1]]
+; MCO-MSSA-NEXT:    ret i32 [[RET]]
+;
   %x = alloca %struct1, align 8
   %y = alloca %struct2, align 8
   call void @bar(%struct1* sret %x) nounwind
 
   %gepn1 = getelementptr inbounds %struct2, %struct2* %y, i32 0, i32 0, i32 0
   store i32 0, i32* %gepn1, align 8
   %gepn2 = getelementptr inbounds %struct2, %struct2* %y, i32 0, i32 0, i32 1
   store i32 0, i32* %gepn2, align 4

test/Transforms/MemCpyOpt/align.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -S -basicaa -memcpyopt | FileCheck %s
+; RUN: opt < %s -S -basicaa -memcpyopt-mssa | FileCheck %s --check-prefix=MCO-MSSA
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64"
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1) nounwind
 
 ; The resulting memset is only 4-byte aligned, despite containing
 ; a 16-byte aligned store in the middle.
 
 define void @foo(i32* %p) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:    [[A0:%.*]] = getelementptr i32, i32* %p, i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[A0]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 0, i64 16, i32 4, i1 false)
 ; CHECK-NEXT:    [[A1:%.*]] = getelementptr i32, i32* %p, i64 1
 ; CHECK-NEXT:    [[A2:%.*]] = getelementptr i32, i32* %p, i64 2
 ; CHECK-NEXT:    [[A3:%.*]] = getelementptr i32, i32* %p, i64 3
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[A0]] to i8*
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 0, i64 16, i32 4, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @foo(
+; MCO-MSSA-NEXT:    [[A0:%.*]] = getelementptr i32, i32* %p, i64 0
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = bitcast i32* [[A0]] to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 0, i64 16, i32 4, i1 false)
+; MCO-MSSA-NEXT:    [[A1:%.*]] = getelementptr i32, i32* %p, i64 1
+; MCO-MSSA-NEXT:    [[A2:%.*]] = getelementptr i32, i32* %p, i64 2
+; MCO-MSSA-NEXT:    [[A3:%.*]] = getelementptr i32, i32* %p, i64 3
+; MCO-MSSA-NEXT:    ret void
+;
   %a0 = getelementptr i32, i32* %p, i64 0
   store i32 0, i32* %a0, align 4
   %a1 = getelementptr i32, i32* %p, i64 1
   store i32 0, i32* %a1, align 16
   %a2 = getelementptr i32, i32* %p, i64 2
   store i32 0, i32* %a2, align 4
   %a3 = getelementptr i32, i32* %p, i64 3
   store i32 0, i32* %a3, align 4
   ret void
 }
 
 ; Replacing %a8 with %a4 in the memset requires boosting the alignment of %a4.
 
 define void @bar() {
 ; CHECK-LABEL: @bar(
 ; CHECK-NEXT:    [[A4:%.*]] = alloca i32, align 8
 ; CHECK-NEXT:    [[A8:%.*]] = alloca i32, align 8
 ; CHECK-NEXT:    [[A8_CAST:%.*]] = bitcast i32* [[A8]] to i8*
 ; CHECK-NEXT:    [[A4_CAST:%.*]] = bitcast i32* [[A4]] to i8*
 ; CHECK-NEXT:    [[A41:%.*]] = bitcast i32* [[A4]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[A41]], i8 0, i64 4, i32 8, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @bar(
+; MCO-MSSA-NEXT:    [[A4:%.*]] = alloca i32, align 8
+; MCO-MSSA-NEXT:    [[A8:%.*]] = alloca i32, align 8
+; MCO-MSSA-NEXT:    [[A8_CAST:%.*]] = bitcast i32* [[A8]] to i8*
+; MCO-MSSA-NEXT:    [[A4_CAST:%.*]] = bitcast i32* [[A4]] to i8*
+; MCO-MSSA-NEXT:    [[A41:%.*]] = bitcast i32* [[A4]] to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[A41]], i8 0, i64 4, i32 8, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   %a4 = alloca i32, align 4
   %a8 = alloca i32, align 8
   %a8.cast = bitcast i32* %a8 to i8*
   %a4.cast = bitcast i32* %a4 to i8*
   call void @llvm.memset.p0i8.i64(i8* %a8.cast, i8 0, i64 4, i32 8, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a4.cast, i8* %a8.cast, i64 4, i32 4, i1 false)
   ret void
 }

test/Transforms/MemCpyOpt/atomic.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -basicaa -memcpyopt -S < %s | FileCheck %s
+; RUN: opt -basicaa -memcpyopt-mssa -S < %s | FileCheck %s --check-prefix=MCO-MSSA
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-macosx10.7.0"
 
 @x = global i32 0
 
 declare void @otherf(i32*)
 
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1) nounwind
 
 ; memcpyopt should not touch atomic ops
 define void @test1() nounwind uwtable ssp {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:    [[X:%.*]] = alloca [101 x i32], align 16
 ; CHECK-NEXT:    [[BC:%.*]] = bitcast [101 x i32]* [[X]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[BC]], i8 0, i64 400, i32 16, i1 false)
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds [101 x i32], [101 x i32]* [[X]], i32 0, i32 100
 ; CHECK-NEXT:    store atomic i32 0, i32* [[GEP1]] unordered, align 4
 ; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds [101 x i32], [101 x i32]* [[X]], i32 0, i32 0
 ; CHECK-NEXT:    call void @otherf(i32* [[GEP2]])
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test1(
+; MCO-MSSA-NEXT:    [[X:%.*]] = alloca [101 x i32], align 16
+; MCO-MSSA-NEXT:    [[BC:%.*]] = bitcast [101 x i32]* [[X]] to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[BC]], i8 0, i64 400, i32 16, i1 false)
+; MCO-MSSA-NEXT:    [[GEP1:%.*]] = getelementptr inbounds [101 x i32], [101 x i32]* [[X]], i32 0, i32 100
+; MCO-MSSA-NEXT:    store atomic i32 0, i32* [[GEP1]] unordered, align 4
+; MCO-MSSA-NEXT:    [[GEP2:%.*]] = getelementptr inbounds [101 x i32], [101 x i32]* [[X]], i32 0, i32 0
+; MCO-MSSA-NEXT:    call void @otherf(i32* [[GEP2]])
+; MCO-MSSA-NEXT:    ret void
+;
   %x = alloca [101 x i32], align 16
   %bc = bitcast [101 x i32]* %x to i8*
   call void @llvm.memset.p0i8.i64(i8* %bc, i8 0, i64 400, i32 16, i1 false)
   %gep1 = getelementptr inbounds [101 x i32], [101 x i32]* %x, i32 0, i32 100
   store atomic i32 0, i32* %gep1 unordered, align 4
   %gep2 = getelementptr inbounds [101 x i32], [101 x i32]* %x, i32 0, i32 0
   call void @otherf(i32* %gep2)
   ret void
 }
 
 ; memcpyopt across unordered store
 define void @test2() nounwind uwtable ssp {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:    [[OLD:%.*]] = alloca i32
 ; CHECK-NEXT:    [[NEW:%.*]] = alloca i32
 ; CHECK-NEXT:    call void @otherf(i32* nocapture [[NEW]])
 ; CHECK-NEXT:    store atomic i32 0, i32* @x unordered, align 4
 ; CHECK-NEXT:    call void @otherf(i32* nocapture [[NEW]])
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test2(
+; MCO-MSSA-NEXT:    [[OLD:%.*]] = alloca i32
+; MCO-MSSA-NEXT:    [[NEW:%.*]] = alloca i32
+; MCO-MSSA-NEXT:    call void @otherf(i32* nocapture [[NEW]])
+; MCO-MSSA-NEXT:    store atomic i32 0, i32* @x unordered, align 4
+; MCO-MSSA-NEXT:    call void @otherf(i32* nocapture [[NEW]])
+; MCO-MSSA-NEXT:    ret void
+;
   %old = alloca i32
   %new = alloca i32
   call void @otherf(i32* nocapture %old)
   store atomic i32 0, i32* @x unordered, align 4
   %v = load i32, i32* %old
   store i32 %v, i32* %new
   call void @otherf(i32* nocapture %new)
   ret void
 }
 

test/Transforms/MemCpyOpt/callslot_aa.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -S -basicaa -memcpyopt | FileCheck %s
+; RUN: opt < %s -S -basicaa -memcpyopt-mssa | FileCheck %s --check-prefix=MCO-MSSA
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 %T = type { i64, i64 }
 
 define void @test(i8* %src) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:    [[TMP:%.*]] = alloca i8
 ; CHECK-NEXT:    [[DST:%.*]] = alloca i8
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[DST]], i8* %src, i64 1, i32 8, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test(
+; MCO-MSSA-NEXT:    [[TMP:%.*]] = alloca i8
+; MCO-MSSA-NEXT:    [[DST:%.*]] = alloca i8
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[DST]], i8* %src, i64 1, i32 8, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   %tmp = alloca i8
   %dst = alloca i8
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %tmp, i8* %src, i64 1, i32 8, i1 false), !noalias !2
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %tmp, i64 1, i32 8, i1 false)
 
   ret void
 }
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8*, i64, i32, i1)
 
 ; Check that the noalias for "dst" was removed by checking that the metadata is gone
 ; CHECK-NOT: "dst"
 !0 = !{!0}
 !1 = distinct !{!1, !0, !"dst"}
 !2 = distinct !{!1}

test/Transforms/MemCpyOpt/callslot_deref.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -S -basicaa -memcpyopt | FileCheck %s
+; RUN: opt < %s -S -basicaa -memcpyopt-mssa | FileCheck %s --check-prefix=MCO-MSSA
 target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i32, i1) unnamed_addr nounwind
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1) nounwind
 
 ; all bytes of %dst that are touch by the memset are dereferenceable
 define void @must_remove_memcpy(i8* noalias nocapture dereferenceable(4096) %dst) {
 ; CHECK-LABEL: @must_remove_memcpy(
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [4096 x i8], align 1
 ; CHECK-NEXT:    [[P:%.*]] = getelementptr inbounds [4096 x i8], [4096 x i8]* [[SRC]], i64 0, i64 0
 ; CHECK-NEXT:    [[DST1:%.*]] = bitcast i8* %dst to [4096 x i8]*
 ; CHECK-NEXT:    [[DST12:%.*]] = bitcast [4096 x i8]* [[DST1]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[DST12]], i8 0, i64 4096, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @must_remove_memcpy(
+; MCO-MSSA-NEXT:    [[SRC:%.*]] = alloca [4096 x i8], align 1
+; MCO-MSSA-NEXT:    [[P:%.*]] = getelementptr inbounds [4096 x i8], [4096 x i8]* [[SRC]], i64 0, i64 0
+; MCO-MSSA-NEXT:    [[DST1:%.*]] = bitcast i8* %dst to [4096 x i8]*
+; MCO-MSSA-NEXT:    [[DST12:%.*]] = bitcast [4096 x i8]* [[DST1]] to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[DST12]], i8 0, i64 4096, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   %src = alloca [4096 x i8], align 1
   %p = getelementptr inbounds [4096 x i8], [4096 x i8]* %src, i64 0, i64 0
   call void @llvm.memset.p0i8.i64(i8* %p, i8 0, i64 4096, i32 1, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %p, i64 4096, i32 1, i1 false) #2
   ret void
 }
 
 ; memset touch more bytes than those guaranteed to be dereferenceable
 ; We can't remove the memcpy, but we can turn it into an independent memset.
 define void @must_not_remove_memcpy(i8* noalias nocapture dereferenceable(1024) %dst) {
 ; CHECK-LABEL: @must_not_remove_memcpy(
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [4096 x i8], align 1
 ; CHECK-NEXT:    [[P:%.*]] = getelementptr inbounds [4096 x i8], [4096 x i8]* [[SRC]], i64 0, i64 0
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[P]], i8 0, i64 4096, i32 1, i1 false)
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 4096, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @must_not_remove_memcpy(
+; MCO-MSSA-NEXT:    [[SRC:%.*]] = alloca [4096 x i8], align 1
+; MCO-MSSA-NEXT:    [[P:%.*]] = getelementptr inbounds [4096 x i8], [4096 x i8]* [[SRC]], i64 0, i64 0
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[P]], i8 0, i64 4096, i32 1, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 4096, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   %src = alloca [4096 x i8], align 1
   %p = getelementptr inbounds [4096 x i8], [4096 x i8]* %src, i64 0, i64 0
   call void @llvm.memset.p0i8.i64(i8* %p, i8 0, i64 4096, i32 1, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %p, i64 4096, i32 1, i1 false) #2
   ret void
 }

test/Transforms/MemCpyOpt/callslot_throw.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -memcpyopt < %s | FileCheck %s
+; RUN: opt -S -memcpyopt-mssa < %s | FileCheck %s --check-prefix=MCO-MSSA
 declare void @may_throw(i32* nocapture %x)
 
 define void @test1(i32* nocapture noalias dereferenceable(4) %x) {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[T:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    call void @may_throw(i32* nonnull [[T]])
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[T]], align 4
 ; CHECK-NEXT:    store i32 [[LOAD]], i32* %x, align 4
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test1(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[T:%.*]] = alloca i32, align 4
+; MCO-MSSA-NEXT:    call void @may_throw(i32* nonnull [[T]])
+; MCO-MSSA-NEXT:    [[LOAD:%.*]] = load i32, i32* [[T]], align 4
+; MCO-MSSA-NEXT:    store i32 [[LOAD]], i32* %x, align 4
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %t = alloca i32, align 4
   call void @may_throw(i32* nonnull %t)
   %load = load i32, i32* %t, align 4
   store i32 %load, i32* %x, align 4
   ret void
 }
 
 declare void @always_throws()
 
 define void @test2(i32* nocapture noalias dereferenceable(4) %x) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[T:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    call void @may_throw(i32* nonnull [[T]]) #0
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[T]], align 4
 ; CHECK-NEXT:    call void @always_throws()
 ; CHECK-NEXT:    store i32 [[LOAD]], i32* %x, align 4
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test2(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[T:%.*]] = alloca i32, align 4
+; MCO-MSSA-NEXT:    call void @may_throw(i32* nonnull [[T]]) #0
+; MCO-MSSA-NEXT:    [[LOAD:%.*]] = load i32, i32* [[T]], align 4
+; MCO-MSSA-NEXT:    call void @always_throws()
+; MCO-MSSA-NEXT:    store i32 [[LOAD]], i32* %x, align 4
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %t = alloca i32, align 4
   call void @may_throw(i32* nonnull %t) nounwind
   %load = load i32, i32* %t, align 4
   call void @always_throws()
   store i32 %load, i32* %x, align 4
   ret void
 }

test/Transforms/MemCpyOpt/capturing-func.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -memcpyopt -S | FileCheck %s
+; RUN: opt < %s -basicaa -memcpyopt-mssa -S | FileCheck %s --check-prefix=MCO-MSSA
 
 target datalayout = "e"
 
 declare void @foo(i8*)
 declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i32, i1) nounwind
 
 define void @test() {
 ; Check that the transformation isn't applied if the called function can
 ; capture the pointer argument (i.e. the nocapture attribute isn't present)
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:    [[PTR1:%.*]] = alloca i8
 ; CHECK-NEXT:    [[PTR2:%.*]] = alloca i8
 ; CHECK-NEXT:    call void @foo(i8* [[PTR2]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[PTR1]], i8* [[PTR2]], i32 1, i32 1, i1 false)
 ; CHECK-NEXT:    call void @foo(i8* [[PTR1]])
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test(
+; MCO-MSSA-NEXT:    [[PTR1:%.*]] = alloca i8
+; MCO-MSSA-NEXT:    [[PTR2:%.*]] = alloca i8
+; MCO-MSSA-NEXT:    call void @foo(i8* [[PTR2]])
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[PTR1]], i8* [[PTR2]], i32 1, i32 1, i1 false)
+; MCO-MSSA-NEXT:    call void @foo(i8* [[PTR1]])
+; MCO-MSSA-NEXT:    ret void
+;
   %ptr1 = alloca i8
   %ptr2 = alloca i8
   call void @foo(i8* %ptr2)
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %ptr1, i8* %ptr2, i32 1, i32 1, i1 false)
   call void @foo(i8* %ptr1)
   ret void
 }

test/Transforms/MemCpyOpt/crash.ll

 ; RUN: opt < %s -basicaa -memcpyopt -disable-output
+; RUN: opt < %s -basicaa -memcpyopt-mssa -disable-output
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64"
 target triple = "armv7-eabi"
 
 %struct.qw = type { [4 x float] }
 %struct.bar = type { %struct.qw, %struct.qw, %struct.qw, %struct.qw, %struct.qw, float, float}
 
 ; PR4882

test/Transforms/MemCpyOpt/fca2memcpy.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -memcpyopt -S < %s | FileCheck %s
+; RUN: opt -memcpyopt-mssa -S < %s | FileCheck %s --check-prefix=MCO-MSSA
 
 target datalayout = "e-i64:64-f80:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
 
 %S = type { i8*, i8, i32 }
 
 define void @copy(%S* %src, %S* %dst) {
 ; CHECK-LABEL: @copy(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast %S* %dst to i8*
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %S* %src to i8*
 ; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* [[TMP1]], i8* [[TMP2]], i64 16, i32 8, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @copy(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = bitcast %S* %dst to i8*
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = bitcast %S* %src to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* [[TMP1]], i8* [[TMP2]], i64 16, i32 8, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   %1 = load %S, %S* %src
   store %S %1, %S* %dst
   ret void
 }
 
 define void @noaliassrc(%S* noalias %src, %S* %dst) {
 ; CHECK-LABEL: @noaliassrc(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast %S* %dst to i8*
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %S* %src to i8*
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[TMP1]], i8* [[TMP2]], i64 16, i32 8, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @noaliassrc(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = bitcast %S* %dst to i8*
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = bitcast %S* %src to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[TMP1]], i8* [[TMP2]], i64 16, i32 8, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   %1 = load %S, %S* %src
   store %S %1, %S* %dst
   ret void
 }
 
 define void @noaliasdst(%S* %src, %S* noalias %dst) {
 ; CHECK-LABEL: @noaliasdst(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast %S* %dst to i8*
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %S* %src to i8*
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[TMP1]], i8* [[TMP2]], i64 16, i32 8, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @noaliasdst(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = bitcast %S* %dst to i8*
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = bitcast %S* %src to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[TMP1]], i8* [[TMP2]], i64 16, i32 8, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   %1 = load %S, %S* %src
   store %S %1, %S* %dst
   ret void
 }
 
 define void @destroysrc(%S* %src, %S* %dst) {
 ; CHECK-LABEL: @destroysrc(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load %S, %S* %src
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %S* %src to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP2]], i8 0, i64 16, i32 8, i1 false)
 ; CHECK-NEXT:    store %S [[TMP1]], %S* %dst
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @destroysrc(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = load %S, %S* %src
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = bitcast %S* %src to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP2]], i8 0, i64 16, i32 8, i1 false)
+; MCO-MSSA-NEXT:    store %S [[TMP1]], %S* %dst
+; MCO-MSSA-NEXT:    ret void
+;
   %1 = load %S, %S* %src
   store %S zeroinitializer, %S* %src
   store %S %1, %S* %dst
   ret void
 }
 
 define void @destroynoaliassrc(%S* noalias %src, %S* %dst) {
 ; CHECK-LABEL: @destroynoaliassrc(
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast %S* %src to i8*
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %S* %dst to i8*
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast %S* %src to i8*
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[TMP2]], i8* [[TMP3]], i64 16, i32 8, i1 false)
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 0, i64 16, i32 8, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @destroynoaliassrc(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = bitcast %S* %src to i8*
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = bitcast %S* %dst to i8*
+; MCO-MSSA-NEXT:    [[TMP3:%.*]] = bitcast %S* %src to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[TMP2]], i8* [[TMP3]], i64 16, i32 8, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 0, i64 16, i32 8, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   %1 = load %S, %S* %src
   store %S zeroinitializer, %S* %src
   store %S %1, %S* %dst
   ret void
 }
 
 define void @copyalias(%S* %src, %S* %dst) {
 ; CHECK-LABEL: @copyalias(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load %S, %S* %src
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %S* %dst to i8*
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast %S* %src to i8*
 ; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* [[TMP2]], i8* [[TMP3]], i64 16, i32 8, i1 false)
 ; CHECK-NEXT:    store %S [[TMP1]], %S* %dst
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @copyalias(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = load %S, %S* %src
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = bitcast %S* %dst to i8*
+; MCO-MSSA-NEXT:    [[TMP3:%.*]] = bitcast %S* %src to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* [[TMP2]], i8* [[TMP3]], i64 16, i32 8, i1 false)
+; MCO-MSSA-NEXT:    store %S [[TMP1]], %S* %dst
+; MCO-MSSA-NEXT:    ret void
+;
   %1 = load %S, %S* %src
   %2 = load %S, %S* %src
   store %S %1, %S* %dst
   store %S %2, %S* %dst
   ret void
 }
 
 ; If the store address is computed ina complex manner, make
 ; sure we lift the computation as well if needed and possible.
 define void @addrproducer(%S* %src, %S* %dst) {
 ; CHECK-LABEL: @addrproducer(
 ; CHECK-NEXT:    [[DST2:%.*]] = getelementptr %S, %S* %dst, i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast %S* [[DST2]] to i8*
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %S* %src to i8*
 ; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* [[TMP1]], i8* [[TMP2]], i64 16, i32 8, i1 false)
 ; CHECK-NEXT:    store %S undef, %S* %dst
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @addrproducer(
+; MCO-MSSA-NEXT:    [[DST2:%.*]] = getelementptr %S, %S* %dst, i64 1
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = bitcast %S* [[DST2]] to i8*
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = bitcast %S* %src to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* [[TMP1]], i8* [[TMP2]], i64 16, i32 8, i1 false)
+; MCO-MSSA-NEXT:    store %S undef, %S* %dst
+; MCO-MSSA-NEXT:    ret void
+;
   %1 = load %S, %S* %src
   store %S undef, %S* %dst
   %dst2 = getelementptr %S , %S* %dst, i64 1
   store %S %1, %S* %dst2
   ret void
 }
 
 define void @aliasaddrproducer(%S* %src, %S* %dst, i32* %dstidptr) {
 ; CHECK-LABEL: @aliasaddrproducer(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load %S, %S* %src
 ; CHECK-NEXT:    store %S undef, %S* %dst
 ; CHECK-NEXT:    [[DSTINDEX:%.*]] = load i32, i32* %dstidptr
 ; CHECK-NEXT:    [[DST2:%.*]] = getelementptr %S, %S* %dst, i32 [[DSTINDEX]]
 ; CHECK-NEXT:    store %S [[TMP1]], %S* [[DST2]]
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @aliasaddrproducer(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = load %S, %S* %src
+; MCO-MSSA-NEXT:    store %S undef, %S* %dst
+; MCO-MSSA-NEXT:    [[DSTINDEX:%.*]] = load i32, i32* %dstidptr
+; MCO-MSSA-NEXT:    [[DST2:%.*]] = getelementptr %S, %S* %dst, i32 [[DSTINDEX]]
+; MCO-MSSA-NEXT:    store %S [[TMP1]], %S* [[DST2]]
+; MCO-MSSA-NEXT:    ret void
+;
   %1 = load %S, %S* %src
   store %S undef, %S* %dst
   %dstindex = load i32, i32* %dstidptr
   %dst2 = getelementptr %S , %S* %dst, i32 %dstindex
   store %S %1, %S* %dst2
   ret void
 }
 
 define void @noaliasaddrproducer(%S* %src, %S* noalias %dst, i32* noalias %dstidptr) {
 ; CHECK-LABEL: @noaliasaddrproducer(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* %dstidptr
 ; CHECK-NEXT:    [[DSTINDEX:%.*]] = or i32 [[TMP1]], 1
 ; CHECK-NEXT:    [[DST2:%.*]] = getelementptr %S, %S* %dst, i32 [[DSTINDEX]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %S* [[DST2]] to i8*
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast %S* %src to i8*
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[TMP2]], i8* [[TMP3]], i64 16, i32 8, i1 false)
 ; CHECK-NEXT:    store %S undef, %S* %src
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @noaliasaddrproducer(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = load i32, i32* %dstidptr
+; MCO-MSSA-NEXT:    [[DSTINDEX:%.*]] = or i32 [[TMP1]], 1
+; MCO-MSSA-NEXT:    [[DST2:%.*]] = getelementptr %S, %S* %dst, i32 [[DSTINDEX]]
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = bitcast %S* [[DST2]] to i8*
+; MCO-MSSA-NEXT:    [[TMP3:%.*]] = bitcast %S* %src to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[TMP2]], i8* [[TMP3]], i64 16, i32 8, i1 false)
+; MCO-MSSA-NEXT:    store %S undef, %S* %src
+; MCO-MSSA-NEXT:    ret void
+;
   %1 = load %S, %S* %src
   store %S undef, %S* %src
   %2 = load i32, i32* %dstidptr
   %dstindex = or i32 %2, 1
   %dst2 = getelementptr %S , %S* %dst, i32 %dstindex
   store %S %1, %S* %dst2
   ret void
 }

test/Transforms/MemCpyOpt/form-memset.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -memcpyopt -S | FileCheck %s
+; RUN: opt < %s -memcpyopt-mssa -S | FileCheck %s --check-prefix=MCO-MSSA
 
 ; All the stores in this example should be merged into a single memset.
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i386-apple-darwin8"
 
 define void @test1(i8 signext  %c) nounwind  {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[X:%.*]] = alloca [19 x i8]
 ; CHECK-NEXT:    [[TMP:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 0
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP]], i8 %c, i64 19, i32 1, i1 false)
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 2
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 3
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 4
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 5
 ; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 6
 ; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 7
 ; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 8
 ; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 9
 ; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 10
 ; CHECK-NEXT:    [[TMP45:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 11
 ; CHECK-NEXT:    [[TMP49:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 12
 ; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 13
 ; CHECK-NEXT:    [[TMP57:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 14
 ; CHECK-NEXT:    [[TMP61:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 15
 ; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 16
 ; CHECK-NEXT:    [[TMP69:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 17
 ; CHECK-NEXT:    [[TMP73:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 18
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP]], i8 %c, i64 19, i32 1, i1 false)
 ; CHECK-NEXT:    [[TMP76:%.*]] = call i32 (...) @bar([19 x i8]* [[X]]) #0
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test1(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[X:%.*]] = alloca [19 x i8]
+; MCO-MSSA-NEXT:    [[TMP:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 0
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP]], i8 %c, i64 19, i32 1, i1 false)
+; MCO-MSSA-NEXT:    [[TMP5:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 1
+; MCO-MSSA-NEXT:    [[TMP9:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 2
+; MCO-MSSA-NEXT:    [[TMP13:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 3
+; MCO-MSSA-NEXT:    [[TMP17:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 4
+; MCO-MSSA-NEXT:    [[TMP21:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 5
+; MCO-MSSA-NEXT:    [[TMP25:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 6
+; MCO-MSSA-NEXT:    [[TMP29:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 7
+; MCO-MSSA-NEXT:    [[TMP33:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 8
+; MCO-MSSA-NEXT:    [[TMP37:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 9
+; MCO-MSSA-NEXT:    [[TMP41:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 10
+; MCO-MSSA-NEXT:    [[TMP45:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 11
+; MCO-MSSA-NEXT:    [[TMP49:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 12
+; MCO-MSSA-NEXT:    [[TMP53:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 13
+; MCO-MSSA-NEXT:    [[TMP57:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 14
+; MCO-MSSA-NEXT:    [[TMP61:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 15
+; MCO-MSSA-NEXT:    [[TMP65:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 16
+; MCO-MSSA-NEXT:    [[TMP69:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 17
+; MCO-MSSA-NEXT:    [[TMP73:%.*]] = getelementptr [19 x i8], [19 x i8]* [[X]], i32 0, i32 18
+; MCO-MSSA-NEXT:    [[TMP76:%.*]] = call i32 (...) @bar([19 x i8]* [[X]]) #0
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %x = alloca [19 x i8]    ; <[19 x i8]*> [#uses=20]
   %tmp = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 0    ; <i8*> [#uses=1]
   store i8 %c, i8* %tmp, align 1
   %tmp5 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 1    ; <i8*> [#uses=1]
   store i8 %c, i8* %tmp5, align 1
   %tmp9 = getelementptr [19 x i8], [19 x i8]* %x, i32 0, i32 2    ; <i8*> [#uses=1]
   store i8 %c, i8* %tmp9, align 1

 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 7
 ; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 6
 ; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 5
 ; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 4
 ; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 3
 ; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 2
 ; CHECK-NEXT:    [[TMP38:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 1
 ; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 0
-; CHECK-NEXT:    [[TMP43:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 7, i32 0
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP41]], i8 -1, i64 8, i32 1, i1 false)
+; CHECK-NEXT:    [[TMP43:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 7, i32 0
 ; CHECK-NEXT:    [[TMP46:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 7, i32 1
 ; CHECK-NEXT:    [[TMP57:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 6, i32 0
 ; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 6, i32 1
 ; CHECK-NEXT:    [[TMP71:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 5, i32 0
 ; CHECK-NEXT:    [[TMP74:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 5, i32 1
 ; CHECK-NEXT:    [[TMP85:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 4, i32 0
 ; CHECK-NEXT:    [[TMP88:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 4, i32 1
 ; CHECK-NEXT:    [[TMP99:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 3, i32 0
 ; CHECK-NEXT:    [[TMP102:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 3, i32 1
 ; CHECK-NEXT:    [[TMP113:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 2, i32 0
 ; CHECK-NEXT:    [[TMP116:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 2, i32 1
 ; CHECK-NEXT:    [[TMP127:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 1, i32 0
 ; CHECK-NEXT:    [[TMP130:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 1, i32 1
 ; CHECK-NEXT:    [[TMP141:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 0, i32 0
-; CHECK-NEXT:    [[TMP144:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 0, i32 1
-; CHECK-NEXT:    [[TMP148:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 7, i32 0
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i16* [[TMP141]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP0]], i8 0, i64 32, i32 8, i1 false)
+; CHECK-NEXT:    [[TMP144:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 0, i32 1
+; CHECK-NEXT:    [[TMP148:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 7, i32 0
 ; CHECK-NEXT:    [[TMP151:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 7, i32 1
 ; CHECK-NEXT:    [[TMP162:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 6, i32 0
 ; CHECK-NEXT:    [[TMP165:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 6, i32 1
 ; CHECK-NEXT:    [[TMP176:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 5, i32 0
 ; CHECK-NEXT:    [[TMP179:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 5, i32 1
 ; CHECK-NEXT:    [[TMP190:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 4, i32 0
 ; CHECK-NEXT:    [[TMP193:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 4, i32 1
 ; CHECK-NEXT:    [[TMP204:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 3, i32 0
 ; CHECK-NEXT:    [[TMP207:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 3, i32 1
 ; CHECK-NEXT:    [[TMP218:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 2, i32 0
 ; CHECK-NEXT:    [[TMP221:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 2, i32 1
 ; CHECK-NEXT:    [[TMP232:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 1, i32 0
 ; CHECK-NEXT:    [[TMP235:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 1, i32 1
 ; CHECK-NEXT:    [[TMP246:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[TMP246]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 0, i64 32, i32 8, i1 false)
 ; CHECK-NEXT:    [[TMP249:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 0, i32 1
 ; CHECK-NEXT:    [[UP_MVD252:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 0
 ; CHECK-NEXT:    [[LEFT_MVD253:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 0
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[TMP246]] to i8*
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 0, i64 32, i32 8, i1 false)
 ; CHECK-NEXT:    call void @foo(%struct.MV* [[UP_MVD252]], %struct.MV* [[LEFT_MVD253]], i8* [[TMP41]]) #0
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test2(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[REF_IDX:%.*]] = alloca [8 x i8]
+; MCO-MSSA-NEXT:    [[LEFT_MVD:%.*]] = alloca [8 x %struct.MV]
+; MCO-MSSA-NEXT:    [[UP_MVD:%.*]] = alloca [8 x %struct.MV]
+; MCO-MSSA-NEXT:    [[TMP20:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 7
+; MCO-MSSA-NEXT:    [[TMP23:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 6
+; MCO-MSSA-NEXT:    [[TMP26:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 5
+; MCO-MSSA-NEXT:    [[TMP29:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 4
+; MCO-MSSA-NEXT:    [[TMP32:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 3
+; MCO-MSSA-NEXT:    [[TMP35:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 2
+; MCO-MSSA-NEXT:    [[TMP38:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 1
+; MCO-MSSA-NEXT:    [[TMP41:%.*]] = getelementptr [8 x i8], [8 x i8]* [[REF_IDX]], i32 0, i32 0
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP41]], i8 -1, i64 8, i32 1, i1 false)
+; MCO-MSSA-NEXT:    [[TMP43:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 7, i32 0
+; MCO-MSSA-NEXT:    [[TMP46:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 7, i32 1
+; MCO-MSSA-NEXT:    [[TMP57:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 6, i32 0
+; MCO-MSSA-NEXT:    [[TMP60:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 6, i32 1
+; MCO-MSSA-NEXT:    [[TMP71:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 5, i32 0
+; MCO-MSSA-NEXT:    [[TMP74:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 5, i32 1
+; MCO-MSSA-NEXT:    [[TMP85:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 4, i32 0
+; MCO-MSSA-NEXT:    [[TMP88:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 4, i32 1
+; MCO-MSSA-NEXT:    [[TMP99:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 3, i32 0
+; MCO-MSSA-NEXT:    [[TMP102:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 3, i32 1
+; MCO-MSSA-NEXT:    [[TMP113:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 2, i32 0
+; MCO-MSSA-NEXT:    [[TMP116:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 2, i32 1
+; MCO-MSSA-NEXT:    [[TMP127:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 1, i32 0
+; MCO-MSSA-NEXT:    [[TMP130:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 1, i32 1
+; MCO-MSSA-NEXT:    [[TMP141:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 0, i32 0
+; MCO-MSSA-NEXT:    [[TMP0:%.*]] = bitcast i16* [[TMP141]] to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP0]], i8 0, i64 32, i32 8, i1 false)
+; MCO-MSSA-NEXT:    [[TMP144:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 0, i32 1
+; MCO-MSSA-NEXT:    [[TMP148:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 7, i32 0
+; MCO-MSSA-NEXT:    [[TMP151:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 7, i32 1
+; MCO-MSSA-NEXT:    [[TMP162:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 6, i32 0
+; MCO-MSSA-NEXT:    [[TMP165:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 6, i32 1
+; MCO-MSSA-NEXT:    [[TMP176:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 5, i32 0
+; MCO-MSSA-NEXT:    [[TMP179:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 5, i32 1
+; MCO-MSSA-NEXT:    [[TMP190:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 4, i32 0
+; MCO-MSSA-NEXT:    [[TMP193:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 4, i32 1
+; MCO-MSSA-NEXT:    [[TMP204:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 3, i32 0
+; MCO-MSSA-NEXT:    [[TMP207:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 3, i32 1
+; MCO-MSSA-NEXT:    [[TMP218:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 2, i32 0
+; MCO-MSSA-NEXT:    [[TMP221:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 2, i32 1
+; MCO-MSSA-NEXT:    [[TMP232:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 1, i32 0
+; MCO-MSSA-NEXT:    [[TMP235:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 1, i32 1
+; MCO-MSSA-NEXT:    [[TMP246:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 0, i32 0
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = bitcast i16* [[TMP246]] to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 0, i64 32, i32 8, i1 false)
+; MCO-MSSA-NEXT:    [[TMP249:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 0, i32 1
+; MCO-MSSA-NEXT:    [[UP_MVD252:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[UP_MVD]], i32 0, i32 0
+; MCO-MSSA-NEXT:    [[LEFT_MVD253:%.*]] = getelementptr [8 x %struct.MV], [8 x %struct.MV]* [[LEFT_MVD]], i32 0, i32 0
+; MCO-MSSA-NEXT:    call void @foo(%struct.MV* [[UP_MVD252]], %struct.MV* [[LEFT_MVD253]], i8* [[TMP41]]) #0
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %ref_idx = alloca [8 x i8]    ; <[8 x i8]*> [#uses=8]
   %left_mvd = alloca [8 x %struct.MV]    ; <[8 x %struct.MV]*> [#uses=17]
   %up_mvd = alloca [8 x %struct.MV]    ; <[8 x %struct.MV]*> [#uses=17]
   %tmp20 = getelementptr [8 x i8], [8 x i8]* %ref_idx, i32 0, i32 7    ; <i8*> [#uses=1]
   store i8 -1, i8* %tmp20, align 1
   %tmp23 = getelementptr [8 x i8], [8 x i8]* %ref_idx, i32 0, i32 6    ; <i8*> [#uses=1]
   store i8 -1, i8* %tmp23, align 1

 declare void @foo(%struct.MV*, %struct.MV*, i8*)
 
 
 ; Store followed by memset.
 define void @test3(i32* nocapture %P) nounwind ssp {
 ; CHECK-LABEL: @test3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* %P, i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[ARRAYIDX]] to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP0]], i8 0, i64 15, i32 4, i1 false)
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* %P, i64 2
-; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[ARRAYIDX]] to i8*
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 0, i64 15, i32 4, i1 false)
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test3(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* %P, i64 1
+; MCO-MSSA-NEXT:    [[TMP0:%.*]] = bitcast i32* [[ARRAYIDX]] to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP0]], i8 0, i64 15, i32 4, i1 false)
+; MCO-MSSA-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* %P, i64 2
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %arrayidx = getelementptr inbounds i32, i32* %P, i64 1
   store i32 0, i32* %arrayidx, align 4
   %add.ptr = getelementptr inbounds i32, i32* %P, i64 2
   %0 = bitcast i32* %add.ptr to i8*
   tail call void @llvm.memset.p0i8.i64(i8* %0, i8 0, i64 11, i32 1, i1 false)
   ret void
 }
 
 ; store followed by memset, different offset scenario
 define void @test4(i32* nocapture %P) nounwind ssp {
 ; CHECK-LABEL: @test4(
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* %P to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP0]], i8 0, i64 15, i32 4, i1 false)
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* %P, i64 1
-; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* %P to i8*
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 0, i64 15, i32 4, i1 false)
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test4(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[TMP0:%.*]] = bitcast i32* %P to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP0]], i8 0, i64 15, i32 4, i1 false)
+; MCO-MSSA-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* %P, i64 1
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   store i32 0, i32* %P, align 4
   %add.ptr = getelementptr inbounds i32, i32* %P, i64 1
   %0 = bitcast i32* %add.ptr to i8*
   tail call void @llvm.memset.p0i8.i64(i8* %0, i8 0, i64 11, i32 1, i1 false)
   ret void
 }
 
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1) nounwind
 
 ; Memset followed by store.
 define void @test5(i32* nocapture %P) nounwind ssp {
 ; CHECK-LABEL: @test5(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* %P, i64 2
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* %P, i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[ARRAYIDX]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 0, i64 15, i32 4, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test5(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* %P, i64 2
+; MCO-MSSA-NEXT:    [[TMP0:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
+; MCO-MSSA-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* %P, i64 1
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = bitcast i32* [[ARRAYIDX]] to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 0, i64 15, i32 4, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %add.ptr = getelementptr inbounds i32, i32* %P, i64 2
   %0 = bitcast i32* %add.ptr to i8*
   tail call void @llvm.memset.p0i8.i64(i8* %0, i8 0, i64 11, i32 1, i1 false)
   %arrayidx = getelementptr inbounds i32, i32* %P, i64 1
   store i32 0, i32* %arrayidx, align 4
   ret void
 }
 
 ;; Memset followed by memset.
 define void @test6(i32* nocapture %P) nounwind ssp {
 ; CHECK-LABEL: @test6(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* %P to i8*
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* %P to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 0, i64 24, i32 1, i1 false)
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* %P, i64 3
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* %P to i8*
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP2]], i8 0, i64 24, i32 1, i1 false)
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test6(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[TMP0:%.*]] = bitcast i32* %P to i8*
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = bitcast i32* %P to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 0, i64 24, i32 1, i1 false)
+; MCO-MSSA-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* %P, i64 3
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %0 = bitcast i32* %P to i8*
   tail call void @llvm.memset.p0i8.i64(i8* %0, i8 0, i64 12, i32 1, i1 false)
   %add.ptr = getelementptr inbounds i32, i32* %P, i64 3
   %1 = bitcast i32* %add.ptr to i8*
   tail call void @llvm.memset.p0i8.i64(i8* %1, i8 0, i64 12, i32 1, i1 false)
   ret void
 }
 
 ; More aggressive heuristic
 ; rdar://9892684
 define void @test7(i32* nocapture %c) nounwind optsize {
 ; CHECK-LABEL: @test7(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* %c, i32 1
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* %c, i32 2
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* %c, i32 3
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* %c, i32 4
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32* %c to i8*
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP5]], i8 -1, i64 20, i32 4, i1 false)
-; CHECK-NEXT:    ret void
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* %c to i8*
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 -1, i64 20, i32 4, i1 false)
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* %c, i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* %c, i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* %c, i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* %c, i32 4
+; CHECK-NEXT:    ret void
+;
+; MCO-MSSA-LABEL: @test7(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = bitcast i32* %c to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 -1, i64 20, i32 4, i1 false)
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* %c, i32 1
+; MCO-MSSA-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* %c, i32 2
+; MCO-MSSA-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* %c, i32 3
+; MCO-MSSA-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* %c, i32 4
+; MCO-MSSA-NEXT:    ret void
 ;
   store i32 -1, i32* %c, align 4
   %1 = getelementptr inbounds i32, i32* %c, i32 1
   store i32 -1, i32* %1, align 4
   %2 = getelementptr inbounds i32, i32* %c, i32 2
   store i32 -1, i32* %2, align 4
   %3 = getelementptr inbounds i32, i32* %c, i32 3
   store i32 -1, i32* %3, align 4
   %4 = getelementptr inbounds i32, i32* %c, i32 4
   store i32 -1, i32* %4, align 4
   ret void
 }
 
 %struct.test8 = type { [4 x i32] }
 
 define void @test8() {
 ; CHECK-LABEL: @test8(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[MEMTMP:%.*]] = alloca %struct.test8, align 16
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast %struct.test8* [[MEMTMP]] to <4 x i32>*
 ; CHECK-NEXT:    store <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32>* [[TMP0]], align 16
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test8(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[MEMTMP:%.*]] = alloca %struct.test8, align 16
+; MCO-MSSA-NEXT:    [[TMP0:%.*]] = bitcast %struct.test8* [[MEMTMP]] to <4 x i32>*
+; MCO-MSSA-NEXT:    store <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32>* [[TMP0]], align 16
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %memtmp = alloca %struct.test8, align 16
   %0 = bitcast %struct.test8* %memtmp to <4 x i32>*
   store <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32>* %0, align 16
   ret void
 }
 
 @test9buf = internal unnamed_addr global [16 x i64] zeroinitializer, align 16
 
 define void @test9() nounwind {
 ; CHECK-LABEL: @test9(
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* bitcast ([16 x i64]* @test9buf to i8*), i8 -1, i64 16, i32 16, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test9(
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* bitcast ([16 x i64]* @test9buf to i8*), i8 -1, i64 16, i32 16, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   store i8 -1, i8* bitcast ([16 x i64]* @test9buf to i8*), align 16
   store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 1), align 1
   store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 2), align 2
   store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 3), align 1
   store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 4), align 4
   store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 5), align 1
   store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 6), align 2
   store i8 -1, i8* getelementptr (i8, i8* bitcast ([16 x i64]* @test9buf to i8*), i64 7), align 1

 }
 
 ; PR19092
 define void @test10(i8* nocapture %P) nounwind {
 ; CHECK-LABEL: @test10(
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %P, i8 0, i64 42, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test10(
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %P, i8 0, i64 42, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   tail call void @llvm.memset.p0i8.i64(i8* %P, i8 0, i64 42, i32 1, i1 false)
   tail call void @llvm.memset.p0i8.i64(i8* %P, i8 0, i64 23, i32 1, i1 false)
   ret void
 }
 
 ; Memset followed by odd store.
 define void @test11(i32* nocapture %P) nounwind ssp {
 ; CHECK-LABEL: @test11(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* %P, i64 3
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* %P, i64 0
 ; CHECK-NEXT:    [[ARRAYIDX_CAST:%.*]] = bitcast i32* [[ARRAYIDX]] to i96*
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i96* [[ARRAYIDX_CAST]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 1, i64 23, i32 4, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test11(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* %P, i64 3
+; MCO-MSSA-NEXT:    [[TMP0:%.*]] = bitcast i32* [[ADD_PTR]] to i8*
+; MCO-MSSA-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* %P, i64 0
+; MCO-MSSA-NEXT:    [[ARRAYIDX_CAST:%.*]] = bitcast i32* [[ARRAYIDX]] to i96*
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = bitcast i96* [[ARRAYIDX_CAST]] to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP1]], i8 1, i64 23, i32 4, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %add.ptr = getelementptr inbounds i32, i32* %P, i64 3
   %0 = bitcast i32* %add.ptr to i8*
   tail call void @llvm.memset.p0i8.i64(i8* %0, i8 1, i64 11, i32 1, i1 false)
   %arrayidx = getelementptr inbounds i32, i32* %P, i64 0
   %arrayidx.cast = bitcast i32* %arrayidx to i96*
   store i96 310698676526526814092329217, i96* %arrayidx.cast, align 4
   ret void
 }

test/Transforms/MemCpyOpt/invariant.start.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; MemCpy optimizations should take place even in presence of invariant.start
 ; RUN: opt < %s -basicaa -memcpyopt -dse -S | FileCheck %s
+; RUN: opt < %s -basicaa -memcpyopt-mssa -dse -S | FileCheck %s --check-prefix=MCO-MSSA
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 
 target triple = "i686-apple-darwin9"
 
 %0 = type { x86_fp80, x86_fp80 }
 declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i32,
 i1) nounwind

 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:    [[MEMTMP:%.*]] = alloca %0, align 16
 ; CHECK-NEXT:    [[R:%.*]] = bitcast %0* [[MEMTMP]] to i8*
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[R]], i8* %P, i32 32, i32 16, i1 false)
 ; CHECK-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 32, i8* %P)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* %Q, i8* [[R]], i32 32, i32 16, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test1(
+; MCO-MSSA-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 32, i8* %P)
+; MCO-MSSA-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* %Q, i8* %P, i32 32, i32 16, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   %memtmp = alloca %0, align 16
   %R = bitcast %0* %memtmp to i8*
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %R, i8* %P, i32 32, i32 16, i1 false)
   %i = call {}* @llvm.invariant.start.p0i8(i64 32, i8* %P)
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %Q, i8* %R, i32 32, i32 16, i1 false)
   ret void
 }
 
 
 ; The invariant.start intrinsic does not inhibit tranforming the memcpy to a
 ; memset.
 define void @test2(i8* %dst1, i8* %dst2, i8 %c) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
 ; CHECK-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 32, i8* %dst1)
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 128, i32 8, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test2(
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
+; MCO-MSSA-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 32, i8* %dst1)
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 128, i32 8, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
   %i = call {}* @llvm.invariant.start.p0i8(i64 32, i8* %dst1)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i32 8, i1 false)
   ret void
 }

test/Transforms/MemCpyOpt/lifetime.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -O1 -S | FileCheck %s
+; RUN: opt -mco-mssa < %s -O1 -S | FileCheck %s --check-prefix=MCO-MSSA
 
 ; performCallSlotOptzn in MemCpy should not exchange the calls to
 ; @llvm.lifetime.start and @llvm.memcpy.
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i32, i1) #1
 declare void @llvm.lifetime.start(i64, i8* nocapture) #1
 declare void @llvm.lifetime.end(i64, i8* nocapture) #1
 
 define void @_ZN4CordC2EOS_(i8* nocapture dereferenceable(16) %arg1) {
 ; CHECK-LABEL: @_ZN4CordC2EOS_(
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[TMP_SROA_3_0_ARG1_SROA_RAW_IDX:%.*]] = getelementptr inbounds i8, i8* %arg1, i64 7
 ; CHECK-NEXT:    store i8 0, i8* [[TMP_SROA_3_0_ARG1_SROA_RAW_IDX]], align 1
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @_ZN4CordC2EOS_(
+; MCO-MSSA-NEXT:  bb:
+; MCO-MSSA-NEXT:    [[TMP_SROA_3_0_ARG1_SROA_RAW_IDX:%.*]] = getelementptr inbounds i8, i8* %arg1, i64 7
+; MCO-MSSA-NEXT:    store i8 0, i8* [[TMP_SROA_3_0_ARG1_SROA_RAW_IDX]], align 1
+; MCO-MSSA-NEXT:    ret void
+;
 bb:
   %tmp = alloca [8 x i8], align 8
   %tmp5 = bitcast [8 x i8]* %tmp to i8*
   call void @llvm.lifetime.start(i64 16, i8* %tmp5)
   %tmp10 = getelementptr inbounds i8, i8* %tmp5, i64 7
   store i8 0, i8* %tmp10, align 1
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %arg1, i8* %tmp5, i64 16, i32 8, i1 false)
   call void @llvm.lifetime.end(i64 16, i8* %tmp5)
   ret void
 }
 
 attributes #1 = { argmemonly nounwind }

test/Transforms/MemCpyOpt/loadstore-sret.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S < %s -basicaa -memcpyopt | FileCheck %s
+; RUN: opt -S < %s -basicaa -memcpyopt-mssa | FileCheck %s --check-prefix=MCO-MSSA
 ; <rdar://problem/8536696>
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-darwin10.0.0"
 
 %"class.std::auto_ptr" = type { i32* }
 
 define void @_Z3foov(%"class.std::auto_ptr"* noalias nocapture sret %agg.result) ssp {
 ; CHECK-LABEL: @_Z3foov(
 ; CHECK-NEXT:  _ZNSt8auto_ptrIiED1Ev.exit:
 ; CHECK-NEXT:    [[TEMP_LVALUE:%.*]] = alloca %"class.std::auto_ptr", align 8
 ; CHECK-NEXT:    call void @_Z3barv(%"class.std::auto_ptr"* sret %agg.result)
 ; CHECK-NEXT:    [[TMP_I_I:%.*]] = getelementptr inbounds %"class.std::auto_ptr", %"class.std::auto_ptr"* [[TEMP_LVALUE]], i64 0, i32 0
 ; CHECK-NEXT:    [[TMP_I_I4:%.*]] = getelementptr inbounds %"class.std::auto_ptr", %"class.std::auto_ptr"* %agg.result, i64 0, i32 0
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @_Z3foov(
+; MCO-MSSA-NEXT:  _ZNSt8auto_ptrIiED1Ev.exit:
+; MCO-MSSA-NEXT:    [[TEMP_LVALUE:%.*]] = alloca %"class.std::auto_ptr", align 8
+; MCO-MSSA-NEXT:    call void @_Z3barv(%"class.std::auto_ptr"* sret %agg.result)
+; MCO-MSSA-NEXT:    [[TMP_I_I:%.*]] = getelementptr inbounds %"class.std::auto_ptr", %"class.std::auto_ptr"* [[TEMP_LVALUE]], i64 0, i32 0
+; MCO-MSSA-NEXT:    [[TMP_I_I4:%.*]] = getelementptr inbounds %"class.std::auto_ptr", %"class.std::auto_ptr"* %agg.result, i64 0, i32 0
+; MCO-MSSA-NEXT:    ret void
+;
 _ZNSt8auto_ptrIiED1Ev.exit:
   %temp.lvalue = alloca %"class.std::auto_ptr", align 8
   call void @_Z3barv(%"class.std::auto_ptr"* sret %temp.lvalue)
   %tmp.i.i = getelementptr inbounds %"class.std::auto_ptr", %"class.std::auto_ptr"* %temp.lvalue, i64 0, i32 0
   %tmp2.i.i = load i32*, i32** %tmp.i.i, align 8
   %tmp.i.i4 = getelementptr inbounds %"class.std::auto_ptr", %"class.std::auto_ptr"* %agg.result, i64 0, i32 0
   store i32* %tmp2.i.i, i32** %tmp.i.i4, align 8
   ret void
 }
 
 declare void @_Z3barv(%"class.std::auto_ptr"* nocapture sret) nounwind

test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -basicaa -memcpyopt -instcombine -S < %s | FileCheck %s
+; RUN: opt -basicaa -memcpyopt-mssa -instcombine -S < %s | FileCheck %s --check-prefix=MCO-MSSA
 
 target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo([8 x i64]* noalias nocapture sret dereferenceable(64) %sret) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  entry-block:
 ; CHECK-NEXT:    [[SRET1:%.*]] = bitcast [8 x i64]* %sret to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[SRET1]], i8 0, i64 64, i32 8, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @foo(
+; MCO-MSSA-NEXT:  entry-block:
+; MCO-MSSA-NEXT:    [[SRET1:%.*]] = bitcast [8 x i64]* %sret to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[SRET1]], i8 0, i64 64, i32 8, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
 entry-block:
   %a = alloca [8 x i64], align 8
   %a.cast = bitcast [8 x i64]* %a to i8*
   call void @llvm.lifetime.start(i64 64, i8* %a.cast)
   call void @llvm.memset.p0i8.i64(i8* %a.cast, i8 0, i64 64, i32 8, i1 false)
   %sret.cast = bitcast [8 x i64]* %sret to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %sret.cast, i8* %a.cast, i64 64, i32 8, i1 false)
   call void @llvm.lifetime.end(i64 64, i8* %a.cast)

 ; CHECK-NEXT:    [[SRET_CAST:%.*]] = bitcast [8 x i64]* %sret to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[SRET_CAST]], i8 0, i64 64, i32 8, i1 false)
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[A_CAST]], i8 42, i64 32, i32 8, i1 false)
 ; CHECK-NEXT:    [[OUT_CAST:%.*]] = bitcast [8 x i64]* %out to i8*
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[OUT_CAST]], i8* [[A_CAST]], i64 64, i32 8, i1 false)
 ; CHECK-NEXT:    call void @llvm.lifetime.end(i64 64, i8* [[A_CAST]])
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @bar(
+; MCO-MSSA-NEXT:  entry-block:
+; MCO-MSSA-NEXT:    [[A:%.*]] = alloca [8 x i64], align 8
+; MCO-MSSA-NEXT:    [[A_CAST:%.*]] = bitcast [8 x i64]* [[A]] to i8*
+; MCO-MSSA-NEXT:    call void @llvm.lifetime.start(i64 64, i8* [[A_CAST]])
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[A_CAST]], i8 0, i64 64, i32 8, i1 false)
+; MCO-MSSA-NEXT:    [[SRET_CAST:%.*]] = bitcast [8 x i64]* %sret to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[SRET_CAST]], i8 0, i64 64, i32 8, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[A_CAST]], i8 42, i64 32, i32 8, i1 false)
+; MCO-MSSA-NEXT:    [[OUT_CAST:%.*]] = bitcast [8 x i64]* %out to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[OUT_CAST]], i8* [[A_CAST]], i64 64, i32 8, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.lifetime.end(i64 64, i8* [[A_CAST]])
+; MCO-MSSA-NEXT:    ret void
+;
 entry-block:
   %a = alloca [8 x i64], align 8
   %a.cast = bitcast [8 x i64]* %a to i8*
   call void @llvm.lifetime.start(i64 64, i8* %a.cast)
   call void @llvm.memset.p0i8.i64(i8* %a.cast, i8 0, i64 64, i32 8, i1 false)
   %sret.cast = bitcast [8 x i64]* %sret to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %sret.cast, i8* %a.cast, i64 64, i32 8, i1 false)
   call void @llvm.memset.p0i8.i64(i8* %a.cast, i8 42, i64 32, i32 8, i1 false)

test/Transforms/MemCpyOpt/memcpy-to-memset.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -memcpyopt -S < %s | FileCheck %s
+; RUN: opt -memcpyopt-mssa -S < %s | FileCheck %s --check-prefix=MCO-MSSA
 
 @cst = internal constant [3 x i32] [i32 -1, i32 -1, i32 -1], align 4
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind
 declare void @foo(i32*) nounwind
 
 define void @test1() nounwind {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:    [[ARR:%.*]] = alloca [3 x i32], align 4
 ; CHECK-NEXT:    [[ARR_I8:%.*]] = bitcast [3 x i32]* [[ARR]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[ARR_I8]], i8 -1, i64 12, i32 4, i1 false)
 ; CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* [[ARR]], i64 0, i64 0
 ; CHECK-NEXT:    call void @foo(i32* [[ARRAYDECAY]]) #1
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test1(
+; MCO-MSSA-NEXT:    [[ARR:%.*]] = alloca [3 x i32], align 4
+; MCO-MSSA-NEXT:    [[ARR_I8:%.*]] = bitcast [3 x i32]* [[ARR]] to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[ARR_I8]], i8 -1, i64 12, i32 4, i1 false)
+; MCO-MSSA-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* [[ARR]], i64 0, i64 0
+; MCO-MSSA-NEXT:    call void @foo(i32* [[ARRAYDECAY]]) #1
+; MCO-MSSA-NEXT:    ret void
+;
   %arr = alloca [3 x i32], align 4
   %arr_i8 = bitcast [3 x i32]* %arr to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %arr_i8, i8* bitcast ([3 x i32]* @cst to i8*), i64 12, i32 4, i1 false)
   %arraydecay = getelementptr inbounds [3 x i32], [3 x i32]* %arr, i64 0, i64 0
   call void @foo(i32* %arraydecay) nounwind
   ret void
 }

test/Transforms/MemCpyOpt/memcpy-undef.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -memcpyopt -S | FileCheck %s
+; RUN: opt < %s -basicaa -memcpyopt-mssa -S | FileCheck %s --check-prefix=MCO-MSSA
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
 %struct.foo = type { i8, [7 x i8], i32 }
 
 ; Check that the memcpy is removed.
 define i32 @test1(%struct.foo* nocapture %foobie) nounwind noinline ssp uwtable {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:    [[BLETCH_SROA_1:%.*]] = alloca [7 x i8], align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds %struct.foo, %struct.foo* %foobie, i64 0, i32 0
 ; CHECK-NEXT:    store i8 98, i8* [[TMP1]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds %struct.foo, %struct.foo* %foobie, i64 0, i32 1, i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [7 x i8], [7 x i8]* [[BLETCH_SROA_1]], i64 0, i64 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds %struct.foo, %struct.foo* %foobie, i64 0, i32 2
 ; CHECK-NEXT:    store i32 20, i32* [[TMP4]], align 4
 ; CHECK-NEXT:    ret i32 undef
 ;
+; MCO-MSSA-LABEL: @test1(
+; MCO-MSSA-NEXT:    [[BLETCH_SROA_1:%.*]] = alloca [7 x i8], align 1
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = getelementptr inbounds %struct.foo, %struct.foo* %foobie, i64 0, i32 0
+; MCO-MSSA-NEXT:    store i8 98, i8* [[TMP1]], align 4
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = getelementptr inbounds %struct.foo, %struct.foo* %foobie, i64 0, i32 1, i64 0
+; MCO-MSSA-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [7 x i8], [7 x i8]* [[BLETCH_SROA_1]], i64 0, i64 0
+; MCO-MSSA-NEXT:    [[TMP4:%.*]] = getelementptr inbounds %struct.foo, %struct.foo* %foobie, i64 0, i32 2
+; MCO-MSSA-NEXT:    store i32 20, i32* [[TMP4]], align 4
+; MCO-MSSA-NEXT:    ret i32 undef
+;
   %bletch.sroa.1 = alloca [7 x i8], align 1
   %1 = getelementptr inbounds %struct.foo, %struct.foo* %foobie, i64 0, i32 0
   store i8 98, i8* %1, align 4
   %2 = getelementptr inbounds %struct.foo, %struct.foo* %foobie, i64 0, i32 1, i64 0
   %3 = getelementptr inbounds [7 x i8], [7 x i8]* %bletch.sroa.1, i64 0, i64 0
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 7, i32 1, i1 false)
   %4 = getelementptr inbounds %struct.foo, %struct.foo* %foobie, i64 0, i32 2
   store i32 20, i32* %4, align 4
   ret i32 undef
 }
 
 ; Check that the memcpy is removed.
 define void @test2(i8* sret noalias nocapture %out, i8* %in) nounwind noinline ssp uwtable {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:    call void @llvm.lifetime.start(i64 8, i8* %in)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test2(
+; MCO-MSSA-NEXT:    call void @llvm.lifetime.start(i64 8, i8* %in)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.lifetime.start(i64 8, i8* %in)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %out, i8* %in, i64 8, i32 1, i1 false)
   ret void
 }
 
 ; Check that the memcpy is not removed.
 define void @test3(i8* sret noalias nocapture %out, i8* %in) nounwind noinline ssp uwtable {
 ; CHECK-LABEL: @test3(
 ; CHECK-NEXT:    call void @llvm.lifetime.start(i64 4, i8* %in)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %out, i8* %in, i64 8, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test3(
+; MCO-MSSA-NEXT:    call void @llvm.lifetime.start(i64 4, i8* %in)
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %out, i8* %in, i64 8, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.lifetime.start(i64 4, i8* %in)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %out, i8* %in, i64 8, i32 1, i1 false)
   ret void
 }
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind
 
 declare void @llvm.lifetime.start(i64, i8* nocapture) nounwind

test/Transforms/MemCpyOpt/memcpy.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -memcpyopt -dse -S | FileCheck %s
+; RUN: opt < %s -basicaa -memcpyopt-mssa -dse -S | FileCheck %s --check-prefix=MCO-MSSA
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i686-apple-darwin9"
 
 %0 = type { x86_fp80, x86_fp80 }
 %1 = type { i32, i32 }
 
 define void @test1(%0* sret  %agg.result, x86_fp80 %z.0, x86_fp80 %z.1) nounwind  {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP2:%.*]] = alloca %0
 ; CHECK-NEXT:    [[TMP5:%.*]] = fsub x86_fp80 0xK80000000000000000000, %z.1
 ; CHECK-NEXT:    call void @ccoshl(%0* sret [[TMP2]], x86_fp80 [[TMP5]], x86_fp80 %z.0) #0
 ; CHECK-NEXT:    [[TMP219:%.*]] = bitcast %0* [[TMP2]] to i8*
 ; CHECK-NEXT:    [[AGG_RESULT21:%.*]] = bitcast %0* %agg.result to i8*
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[AGG_RESULT21]], i8* [[TMP219]], i32 32, i32 16, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test1(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = alloca %0
+; MCO-MSSA-NEXT:    [[TMP5:%.*]] = fsub x86_fp80 0xK80000000000000000000, %z.1
+; MCO-MSSA-NEXT:    call void @ccoshl(%0* sret [[TMP2]], x86_fp80 [[TMP5]], x86_fp80 %z.0) #0
+; MCO-MSSA-NEXT:    [[TMP219:%.*]] = bitcast %0* [[TMP2]] to i8*
+; MCO-MSSA-NEXT:    [[AGG_RESULT21:%.*]] = bitcast %0* %agg.result to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[AGG_RESULT21]], i8* [[TMP219]], i32 32, i32 16, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %tmp2 = alloca %0
   %memtmp = alloca %0, align 16
   %tmp5 = fsub x86_fp80 0xK80000000000000000000, %z.1
   call void @ccoshl(%0* sret %memtmp, x86_fp80 %tmp5, x86_fp80 %z.0) nounwind
   %tmp219 = bitcast %0* %tmp2 to i8*
   %memtmp20 = bitcast %0* %memtmp to i8*
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %tmp219, i8* %memtmp20, i32 32, i32 16, i1 false)

 
 ; The intermediate alloca and one of the memcpy's should be eliminated, the
 ; other should be related with a memmove.
 define void @test2(i8* %P, i8* %Q) nounwind  {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* %Q, i8* %P, i32 32, i32 16, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test2(
+; MCO-MSSA-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* %Q, i8* %P, i32 32, i32 16, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   %memtmp = alloca %0, align 16
   %R = bitcast %0* %memtmp to i8*
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %R, i8* %P, i32 32, i32 16, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %Q, i8* %R, i32 32, i32 16, i1 false)
   ret void
 
 }
 
 
 
 
 @x = external global %0
 
 define void @test3(%0* noalias sret %agg.result) nounwind  {
 ; CHECK-LABEL: @test3(
 ; CHECK-NEXT:    [[AGG_RESULT1:%.*]] = bitcast %0* %agg.result to i8*
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[AGG_RESULT1]], i8* bitcast (%0* @x to i8*), i32 32, i32 16, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test3(
+; MCO-MSSA-NEXT:    [[AGG_RESULT1:%.*]] = bitcast %0* %agg.result to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[AGG_RESULT1]], i8* bitcast (%0* @x to i8*), i32 32, i32 16, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   %x.0 = alloca %0
   %x.01 = bitcast %0* %x.0 to i8*
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %x.01, i8* bitcast (%0* @x to i8*), i32 32, i32 16, i1 false)
   %agg.result2 = bitcast %0* %agg.result to i8*
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %agg.result2, i8* %x.01, i32 32, i32 16, i1 false)
   ret void
 }
 
 
 ; PR8644
 define void @test4(i8 *%P) {
 ; CHECK-LABEL: @test4(
 ; CHECK-NEXT:    call void @test4a(i8* byval align 1 %P)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test4(
+; MCO-MSSA-NEXT:    call void @test4a(i8* byval align 1 %P)
+; MCO-MSSA-NEXT:    ret void
+;
   %A = alloca %1
   %a = bitcast %1* %A to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %P, i64 8, i32 4, i1 false)
   call void @test4a(i8* align 1 byval %a)
   ret void
 }
 
 declare void @test4a(i8* align 1 byval)

 ; CHECK-NEXT:    [[Y:%.*]] = alloca %struct.S, align 16
 ; CHECK-NEXT:    [[TMP:%.*]] = bitcast %struct.S* [[Y]] to i8*
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[TMP]], i8* bitcast (%struct.S* @sS to i8*), i64 32, i32 16, i1 false)
 ; CHECK-NEXT:    [[A:%.*]] = getelementptr %struct.S, %struct.S* [[Y]], i64 0, i32 1, i64 0
 ; CHECK-NEXT:    store i8 4, i8* [[A]]
 ; CHECK-NEXT:    call void @test5a(%struct.S* byval align 16 [[Y]])
 ; CHECK-NEXT:    ret i32 0
 ;
+; MCO-MSSA-LABEL: @test5(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[Y:%.*]] = alloca %struct.S, align 16
+; MCO-MSSA-NEXT:    [[TMP:%.*]] = bitcast %struct.S* [[Y]] to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[TMP]], i8* bitcast (%struct.S* @sS to i8*), i64 32, i32 16, i1 false)
+; MCO-MSSA-NEXT:    [[A:%.*]] = getelementptr %struct.S, %struct.S* [[Y]], i64 0, i32 1, i64 0
+; MCO-MSSA-NEXT:    store i8 4, i8* [[A]]
+; MCO-MSSA-NEXT:    call void @test5a(%struct.S* byval align 16 [[Y]])
+; MCO-MSSA-NEXT:    ret i32 0
+;
 entry:
   %y = alloca %struct.S, align 16
   %tmp = bitcast %struct.S* %y to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %tmp, i8* bitcast (%struct.S* @sS to i8*), i64 32, i32 16, i1 false)
   %a = getelementptr %struct.S, %struct.S* %y, i64 0, i32 1, i64 0
   store i8 4, i8* %a
   call void @test5a(%struct.S* align 16 byval %y)
   ret i32 0
 }
 
 ;; Noop memcpy should be zapped.
 define void @test6(i8 *%P) {
 ; CHECK-LABEL: @test6(
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test6(
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* %P, i64 8, i32 4, i1 false)
   ret void
 }
 
 
 ; PR9794 - Should forward memcpy into byval argument even though the memcpy
 ; isn't itself 8 byte aligned.
 %struct.p = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }
 
 define i32 @test7(%struct.p* nocapture align 8 byval %q) nounwind ssp {
 ; CHECK-LABEL: @test7(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @g(%struct.p* byval align 8 %q) #0
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
+; MCO-MSSA-LABEL: @test7(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[CALL:%.*]] = call i32 @g(%struct.p* byval align 8 %q) #0
+; MCO-MSSA-NEXT:    ret i32 [[CALL]]
+;
 entry:
   %agg.tmp = alloca %struct.p, align 4
   %tmp = bitcast %struct.p* %agg.tmp to i8*
   %tmp1 = bitcast %struct.p* %q to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %tmp, i8* %tmp1, i64 48, i32 4, i1 false)
   %call = call i32 @g(%struct.p* align 8 byval %agg.tmp) nounwind
   ret i32 %call
 }
 
 declare i32 @g(%struct.p* align 8 byval)
 
 declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i32, i1) nounwind
 
 ; PR11142 - When looking for a memcpy-memcpy dependency, don't get stuck on
 ; instructions between the memcpy's that only affect the destination pointer.
 @test8.str = internal constant [7 x i8] c"ABCDEF\00"
 
 define void @test8() {
 ; CHECK-LABEL: @test8(
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test8(
+; MCO-MSSA-NEXT:    ret void
+;
   %A = tail call i8* @malloc(i32 10)
   %B = getelementptr inbounds i8, i8* %A, i64 2
   tail call void @llvm.memcpy.p0i8.p0i8.i32(i8* %B, i8* getelementptr inbounds ([7 x i8], [7 x i8]* @test8.str, i64 0, i64 0), i32 7, i32 1, i1 false)
   %C = tail call i8* @malloc(i32 10)
   %D = getelementptr inbounds i8, i8* %C, i64 2
   tail call void @llvm.memcpy.p0i8.p0i8.i32(i8* %D, i8* %B, i32 7, i32 1, i1 false)
   ret void
 }
 
 declare noalias i8* @malloc(i32)
 
 ; rdar://11341081
 %struct.big = type { [50 x i32] }
 
 define void @test9_addrspacecast() nounwind ssp uwtable {
 ; CHECK-LABEL: @test9_addrspacecast(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[B:%.*]] = alloca %struct.big, align 4
 ; CHECK-NEXT:    call void @f1(%struct.big* sret [[B]])
 ; CHECK-NEXT:    call void @f2(%struct.big* [[B]])
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test9_addrspacecast(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[B:%.*]] = alloca %struct.big, align 4
+; MCO-MSSA-NEXT:    call void @f1(%struct.big* sret [[B]])
+; MCO-MSSA-NEXT:    call void @f2(%struct.big* [[B]])
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %b = alloca %struct.big, align 4
   %tmp = alloca %struct.big, align 4
   call void @f1(%struct.big* sret %tmp)
   %0 = addrspacecast %struct.big* %b to i8 addrspace(1)*
   %1 = addrspacecast %struct.big* %tmp to i8 addrspace(1)*
   call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %0, i8 addrspace(1)* %1, i64 200, i32 4, i1 false)
   call void @f2(%struct.big* %b)
   ret void
 }
 
 define void @test9() nounwind ssp uwtable {
 ; CHECK-LABEL: @test9(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[B:%.*]] = alloca %struct.big, align 4
 ; CHECK-NEXT:    call void @f1(%struct.big* sret [[B]])
 ; CHECK-NEXT:    call void @f2(%struct.big* [[B]])
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test9(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[B:%.*]] = alloca %struct.big, align 4
+; MCO-MSSA-NEXT:    call void @f1(%struct.big* sret [[B]])
+; MCO-MSSA-NEXT:    call void @f2(%struct.big* [[B]])
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %b = alloca %struct.big, align 4
   %tmp = alloca %struct.big, align 4
   call void @f1(%struct.big* sret %tmp)
   %0 = bitcast %struct.big* %b to i8*
   %1 = bitcast %struct.big* %tmp to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 200, i32 4, i1 false)
   call void @f2(%struct.big* %b)

 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 %y, i32* [[A]]
 ; CHECK-NEXT:    call void @foo(i32* noalias nocapture [[A]])
 ; CHECK-NEXT:    [[C:%.*]] = load i32, i32* [[A]]
 ; CHECK-NEXT:    [[D:%.*]] = bitcast %opaque* %x to i32*
 ; CHECK-NEXT:    store i32 [[C]], i32* [[D]]
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test10(
+; MCO-MSSA-NEXT:    [[A:%.*]] = alloca i32, align 4
+; MCO-MSSA-NEXT:    store i32 %y, i32* [[A]]
+; MCO-MSSA-NEXT:    call void @foo(i32* noalias nocapture [[A]])
+; MCO-MSSA-NEXT:    [[C:%.*]] = load i32, i32* [[A]]
+; MCO-MSSA-NEXT:    [[D:%.*]] = bitcast %opaque* %x to i32*
+; MCO-MSSA-NEXT:    store i32 [[C]], i32* [[D]]
+; MCO-MSSA-NEXT:    ret void
+;
   %a = alloca i32, align 4
   store i32 %y, i32* %a
   call void @foo(i32* noalias nocapture %a)
   %c = load i32, i32* %a
   %d = bitcast %opaque* %x to i32*
   store i32 %c, i32* %d
   ret void
 }
 
 declare void @f1(%struct.big* nocapture sret)
 declare void @f2(%struct.big*)

test/Transforms/MemCpyOpt/memmove.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -memcpyopt -S | FileCheck %s
+; RUN: opt < %s -basicaa -memcpyopt-mssa -S | FileCheck %s --check-prefix=MCO-MSSA
 ; These memmoves should get optimized to memcpys.
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 target triple = "x86_64-apple-darwin9.0"
 
 declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i32, i1) nounwind
 
 define i8* @test1(i8* nocapture %src) nounwind {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[MALLOCCALL:%.*]] = tail call i8* @malloc(i32 trunc (i64 mul nuw (i64 ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i64), i64 13) to i32))
 ; CHECK-NEXT:    [[CALL3:%.*]] = bitcast i8* [[MALLOCCALL]] to [13 x i8]*
 ; CHECK-NEXT:    [[CALL3_SUB:%.*]] = getelementptr inbounds [13 x i8], [13 x i8]* [[CALL3]], i64 0, i64 0
 ; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[CALL3_SUB]], i8* %src, i64 13, i32 1, i1 false)
 ; CHECK-NEXT:    ret i8* [[CALL3_SUB]]
 ;
+; MCO-MSSA-LABEL: @test1(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[MALLOCCALL:%.*]] = tail call i8* @malloc(i32 trunc (i64 mul nuw (i64 ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i64), i64 13) to i32))
+; MCO-MSSA-NEXT:    [[CALL3:%.*]] = bitcast i8* [[MALLOCCALL]] to [13 x i8]*
+; MCO-MSSA-NEXT:    [[CALL3_SUB:%.*]] = getelementptr inbounds [13 x i8], [13 x i8]* [[CALL3]], i64 0, i64 0
+; MCO-MSSA-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[CALL3_SUB]], i8* %src, i64 13, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret i8* [[CALL3_SUB]]
+;
 entry:
 
   %malloccall = tail call i8* @malloc(i32 trunc (i64 mul nuw (i64 ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i64), i64 13) to i32))
   %call3 = bitcast i8* %malloccall to [13 x i8]*
   %call3.sub = getelementptr inbounds [13 x i8], [13 x i8]* %call3, i64 0, i64 0
   tail call void @llvm.memmove.p0i8.p0i8.i64(i8* %call3.sub, i8* %src, i64 13, i32 1, i1 false)
   ret i8* %call3.sub
 }
 declare noalias i8* @malloc(i32)
 
 
 define void @test2(i8* %P) nounwind {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr i8, i8* %P, i64 16
 ; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* [[ADD_PTR]], i64 16, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test2(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[ADD_PTR:%.*]] = getelementptr i8, i8* %P, i64 16
+; MCO-MSSA-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %P, i8* [[ADD_PTR]], i64 16, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %add.ptr = getelementptr i8, i8* %P, i64 16
   tail call void @llvm.memmove.p0i8.p0i8.i64(i8* %P, i8* %add.ptr, i64 16, i32 1, i1 false)
   ret void
 }
 
 ; This cannot be optimize because the src/dst really do overlap.
 define void @test3(i8* %P) nounwind {
 ; CHECK-LABEL: @test3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr i8, i8* %P, i64 16
 ; CHECK-NEXT:    tail call void @llvm.memmove.p0i8.p0i8.i64(i8* %P, i8* [[ADD_PTR]], i64 17, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test3(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[ADD_PTR:%.*]] = getelementptr i8, i8* %P, i64 16
+; MCO-MSSA-NEXT:    tail call void @llvm.memmove.p0i8.p0i8.i64(i8* %P, i8* [[ADD_PTR]], i64 17, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %add.ptr = getelementptr i8, i8* %P, i64 16
   tail call void @llvm.memmove.p0i8.p0i8.i64(i8* %P, i8* %add.ptr, i64 17, i32 1, i1 false)
   ret void
 }

test/Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -basicaa -memcpyopt -S %s | FileCheck %s
+; RUN: opt -basicaa -memcpyopt-mssa -S %s | FileCheck %s --check-prefix=MCO-MSSA
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 define void @test(i8* %src, i64 %src_size, i8* %dst, i64 %dst_size, i8 %c) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i64 %dst_size, %src_size
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 %dst_size, %src_size
 ; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i64 0, i64 [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, i8* %dst, i64 %src_size
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP4]], i8 %c, i64 [[TMP3]], i32 1, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = icmp ule i64 %dst_size, %src_size
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = sub i64 %dst_size, %src_size
+; MCO-MSSA-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i64 0, i64 [[TMP2]]
+; MCO-MSSA-NEXT:    [[TMP4:%.*]] = getelementptr i8, i8* %dst, i64 %src_size
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP4]], i8 %c, i64 [[TMP3]], i32 1, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i64(i8* %dst, i8 %c, i64 %dst_size, i32 1, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i32 1, i1 false)
   ret void
 }
 
 define void @test_different_types_i32_i64(i8* %dst, i8* %src, i32 %dst_size, i64 %src_size, i8 %c) {
 ; CHECK-LABEL: @test_different_types_i32_i64(
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 %dst_size to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule i64 [[TMP1]], %src_size
 ; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP1]], %src_size
 ; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i64 0, i64 [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, i8* %dst, i64 %src_size
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP5]], i8 %c, i64 [[TMP4]], i32 1, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_different_types_i32_i64(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = zext i32 %dst_size to i64
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = icmp ule i64 [[TMP1]], %src_size
+; MCO-MSSA-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP1]], %src_size
+; MCO-MSSA-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i64 0, i64 [[TMP3]]
+; MCO-MSSA-NEXT:    [[TMP5:%.*]] = getelementptr i8, i8* %dst, i64 %src_size
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP5]], i8 %c, i64 [[TMP4]], i32 1, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i32(i8* %dst, i8 %c, i32 %dst_size, i32 1, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i32 1, i1 false)
   ret void
 }
 
 define void @test_different_types_i128_i32(i8* %dst, i8* %src, i128 %dst_size, i32 %src_size, i8 %c) {
 ; CHECK-LABEL: @test_different_types_i128_i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 %src_size to i128
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule i128 %dst_size, [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = sub i128 %dst_size, [[TMP1]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i128 0, i128 [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, i8* %dst, i128 [[TMP1]]
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i128(i8* [[TMP5]], i8 %c, i128 [[TMP4]], i32 1, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %src_size, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_different_types_i128_i32(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = zext i32 %src_size to i128
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = icmp ule i128 %dst_size, [[TMP1]]
+; MCO-MSSA-NEXT:    [[TMP3:%.*]] = sub i128 %dst_size, [[TMP1]]
+; MCO-MSSA-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i128 0, i128 [[TMP3]]
+; MCO-MSSA-NEXT:    [[TMP5:%.*]] = getelementptr i8, i8* %dst, i128 [[TMP1]]
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i128(i8* [[TMP5]], i8 %c, i128 [[TMP4]], i32 1, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %src_size, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i128(i8* %dst, i8 %c, i128 %dst_size, i32 1, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %src_size, i32 1, i1 false)
   ret void
 }
 
 define void @test_different_types_i32_i128(i8* %dst, i8* %src, i32 %dst_size, i128 %src_size, i8 %c) {
 ; CHECK-LABEL: @test_different_types_i32_i128(
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 %dst_size to i128
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule i128 [[TMP1]], %src_size
 ; CHECK-NEXT:    [[TMP3:%.*]] = sub i128 [[TMP1]], %src_size
 ; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i128 0, i128 [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, i8* %dst, i128 %src_size
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i128(i8* [[TMP5]], i8 %c, i128 [[TMP4]], i32 1, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i128(i8* %dst, i8* %src, i128 %src_size, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_different_types_i32_i128(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = zext i32 %dst_size to i128
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = icmp ule i128 [[TMP1]], %src_size
+; MCO-MSSA-NEXT:    [[TMP3:%.*]] = sub i128 [[TMP1]], %src_size
+; MCO-MSSA-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i128 0, i128 [[TMP3]]
+; MCO-MSSA-NEXT:    [[TMP5:%.*]] = getelementptr i8, i8* %dst, i128 %src_size
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i128(i8* [[TMP5]], i8 %c, i128 [[TMP4]], i32 1, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i128(i8* %dst, i8* %src, i128 %src_size, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i32(i8* %dst, i8 %c, i32 %dst_size, i32 1, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i128(i8* %dst, i8* %src, i128 %src_size, i32 1, i1 false)
   ret void
 }
 
 define void @test_different_types_i64_i32(i8* %dst, i8* %src, i64 %dst_size, i32 %src_size, i8 %c) {
 ; CHECK-LABEL: @test_different_types_i64_i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 %src_size to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule i64 %dst_size, [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 %dst_size, [[TMP1]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i64 0, i64 [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, i8* %dst, i64 [[TMP1]]
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP5]], i8 %c, i64 [[TMP4]], i32 1, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %src_size, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_different_types_i64_i32(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = zext i32 %src_size to i64
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = icmp ule i64 %dst_size, [[TMP1]]
+; MCO-MSSA-NEXT:    [[TMP3:%.*]] = sub i64 %dst_size, [[TMP1]]
+; MCO-MSSA-NEXT:    [[TMP4:%.*]] = select i1 [[TMP2]], i64 0, i64 [[TMP3]]
+; MCO-MSSA-NEXT:    [[TMP5:%.*]] = getelementptr i8, i8* %dst, i64 [[TMP1]]
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP5]], i8 %c, i64 [[TMP4]], i32 1, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %src_size, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i64(i8* %dst, i8 %c, i64 %dst_size, i32 1, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %src, i32 %src_size, i32 1, i1 false)
   ret void
 }
 
 define void @test_align_same(i8* %src, i8* %dst, i64 %dst_size) {
 ; CHECK-LABEL: @test_align_same(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i64 %dst_size, 80
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 %dst_size, 80
 ; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i64 0, i64 [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, i8* %dst, i64 80
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP4]], i8 0, i64 [[TMP3]], i32 8, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 80, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_align_same(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = icmp ule i64 %dst_size, 80
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = sub i64 %dst_size, 80
+; MCO-MSSA-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i64 0, i64 [[TMP2]]
+; MCO-MSSA-NEXT:    [[TMP4:%.*]] = getelementptr i8, i8* %dst, i64 80
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP4]], i8 0, i64 [[TMP3]], i32 8, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 80, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 %dst_size, i32 8, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 80, i32 1, i1 false)
   ret void
 }
 
 define void @test_align_min(i8* %src, i8* %dst, i64 %dst_size) {
 ; CHECK-LABEL: @test_align_min(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i64 %dst_size, 36
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 %dst_size, 36
 ; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i64 0, i64 [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, i8* %dst, i64 36
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP4]], i8 0, i64 [[TMP3]], i32 4, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 36, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_align_min(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = icmp ule i64 %dst_size, 36
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = sub i64 %dst_size, 36
+; MCO-MSSA-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i64 0, i64 [[TMP2]]
+; MCO-MSSA-NEXT:    [[TMP4:%.*]] = getelementptr i8, i8* %dst, i64 36
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP4]], i8 0, i64 [[TMP3]], i32 4, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 36, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 %dst_size, i32 8, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 36, i32 1, i1 false)
   ret void
 }
 
 define void @test_align_memcpy(i8* %src, i8* %dst, i64 %dst_size) {
 ; CHECK-LABEL: @test_align_memcpy(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i64 %dst_size, 80
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 %dst_size, 80
 ; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i64 0, i64 [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, i8* %dst, i64 80
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP4]], i8 0, i64 [[TMP3]], i32 8, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 80, i32 8, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_align_memcpy(
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = icmp ule i64 %dst_size, 80
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = sub i64 %dst_size, 80
+; MCO-MSSA-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i64 0, i64 [[TMP2]]
+; MCO-MSSA-NEXT:    [[TMP4:%.*]] = getelementptr i8, i8* %dst, i64 80
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP4]], i8 0, i64 [[TMP3]], i32 8, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 80, i32 8, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 %dst_size, i32 1, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 80, i32 8, i1 false)
   ret void
 }
 
 define void @test_non_i8_dst_type(i8* %src, i64 %src_size, i64* %dst_pi64, i64 %dst_size, i8 %c) {
 ; CHECK-LABEL: @test_non_i8_dst_type(
 ; CHECK-NEXT:    [[DST:%.*]] = bitcast i64* [[DST:%.*]]_pi64 to i8*
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i64 [[DST]]_size, %src_size
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[DST]]_size, %src_size
 ; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i64 0, i64 [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, i8* [[DST]], i64 %src_size
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP4]], i8 %c, i64 [[TMP3]], i32 1, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[DST]], i8* %src, i64 %src_size, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_non_i8_dst_type(
+; MCO-MSSA-NEXT:    [[DST:%.*]] = bitcast i64* [[DST:%.*]]_pi64 to i8*
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = icmp ule i64 [[DST]]_size, %src_size
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = sub i64 [[DST]]_size, %src_size
+; MCO-MSSA-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i64 0, i64 [[TMP2]]
+; MCO-MSSA-NEXT:    [[TMP4:%.*]] = getelementptr i8, i8* [[DST]], i64 %src_size
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP4]], i8 %c, i64 [[TMP3]], i32 1, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[DST]], i8* %src, i64 %src_size, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   %dst = bitcast i64* %dst_pi64 to i8*
   call void @llvm.memset.p0i8.i64(i8* %dst, i8 %c, i64 %dst_size, i32 1, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %src_size, i32 1, i1 false)
   ret void
 }
 
 define void @test_different_dst(i8* %dst2, i8* %src, i64 %src_size, i8* %dst, i64 %dst_size) {
 ; CHECK-LABEL: @test_different_dst(
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 %dst_size, i32 1, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src, i64 %src_size, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_different_dst(
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 %dst_size, i32 1, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src, i64 %src_size, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i64(i8* %dst, i8 0, i64 %dst_size, i32 1, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src, i64 %src_size, i32 1, i1 false)
   ret void
 }
 
 ; Make sure we also take into account dependencies on the destination.
 
 define i8 @test_intermediate_read(i8* %a, i8* %b) #0 {
 ; CHECK-LABEL: @test_intermediate_read(
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 64, i32 1, i1 false)
 ; CHECK-NEXT:    [[R:%.*]] = load i8, i8* %a
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %b, i64 24, i32 1, i1 false)
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
+; MCO-MSSA-LABEL: @test_intermediate_read(
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 64, i32 1, i1 false)
+; MCO-MSSA-NEXT:    [[R:%.*]] = load i8, i8* %a
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %b, i64 24, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret i8 [[R]]
+;
   call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 64, i32 1, i1 false)
   %r = load i8, i8* %a
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %b, i64 24, i32 1, i1 false)
   ret i8 %r
 }
 
 %struct = type { [8 x i8], [8 x i8] }
 
 define void @test_intermediate_write(i8* %b) #0 {
 ; CHECK-LABEL: @test_intermediate_write(
 ; CHECK-NEXT:    [[A:%.*]] = alloca %struct
 ; CHECK-NEXT:    [[A0:%.*]] = getelementptr %struct, %struct* [[A]], i32 0, i32 0, i32 0
 ; CHECK-NEXT:    [[A1:%.*]] = getelementptr %struct, %struct* [[A]], i32 0, i32 1, i32 0
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[A0]], i8 0, i64 16, i32 1, i1 false)
 ; CHECK-NEXT:    store i8 1, i8* [[A1]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[A0]], i8* %b, i64 8, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_intermediate_write(
+; MCO-MSSA-NEXT:    [[A:%.*]] = alloca %struct
+; MCO-MSSA-NEXT:    [[A0:%.*]] = getelementptr %struct, %struct* [[A]], i32 0, i32 0, i32 0
+; MCO-MSSA-NEXT:    [[A1:%.*]] = getelementptr %struct, %struct* [[A]], i32 0, i32 1, i32 0
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[A0]], i8 0, i64 16, i32 1, i1 false)
+; MCO-MSSA-NEXT:    store i8 1, i8* [[A1]]
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[A0]], i8* %b, i64 8, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   %a = alloca %struct
   %a0 = getelementptr %struct, %struct* %a, i32 0, i32 0, i32 0
   %a1 = getelementptr %struct, %struct* %a, i32 0, i32 1, i32 0
   call void @llvm.memset.p0i8.i64(i8* %a0, i8 0, i64 16, i32 1, i1 false)
   store i8 1, i8* %a1
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a0, i8* %b, i64 8, i32 1, i1 false)
   ret void
 }
 
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1)
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i32, i1)
 declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1)
 declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i32, i1)
 declare void @llvm.memset.p0i8.i128(i8* nocapture, i8, i128, i32, i1)
 declare void @llvm.memcpy.p0i8.p0i8.i128(i8* nocapture, i8* nocapture readonly, i128, i32, i1)

test/Transforms/MemCpyOpt/memset-memcpy-to-2x-memset.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -memcpyopt -S %s | FileCheck %s
+; RUN: opt -memcpyopt-mssa -S %s | FileCheck %s --check-prefix=MCO-MSSA
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
 define void @test(i8* %dst1, i8* %dst2, i8 %c) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 128, i32 8, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test(
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 128, i32 8, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i32 8, i1 false)
   ret void
 }
 
 define void @test_smaller_memcpy(i8* %dst1, i8* %dst2, i8 %c) {
 ; CHECK-LABEL: @test_smaller_memcpy(
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 100, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_smaller_memcpy(
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 100, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 100, i32 1, i1 false)
   ret void
 }
 
 define void @test_smaller_memset(i8* %dst1, i8* %dst2, i8 %c) {
 ; CHECK-LABEL: @test_smaller_memset(
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 100, i32 1, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_smaller_memset(
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 100, i32 1, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 100, i32 1, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i32 1, i1 false)
   ret void
 }
 
 define void @test_align_memset(i8* %dst1, i8* %dst2, i8 %c) {
 ; CHECK-LABEL: @test_align_memset(
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 8, i1 false)
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 128, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_align_memset(
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 8, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 128, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 8, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i32 1, i1 false)
   ret void
 }
 
 define void @test_different_types(i8* %dst1, i8* %dst2, i8 %c) {
 ; CHECK-LABEL: @test_different_types(
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 8, i1 false)
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* %dst2, i8 %c, i32 100, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_different_types(
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 8, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i32(i8* %dst2, i8 %c, i32 100, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 8, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst2, i8* %dst1, i32 100, i32 1, i1 false)
   ret void
 }
 
 define void @test_different_types_2(i8* %dst1, i8* %dst2, i8 %c) {
 ; CHECK-LABEL: @test_different_types_2(
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* %dst1, i8 %c, i32 128, i32 8, i1 false)
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 100, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_different_types_2(
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i32(i8* %dst1, i8 %c, i32 128, i32 8, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst2, i8 %c, i64 100, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i32(i8* %dst1, i8 %c, i32 128, i32 8, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 100, i32 1, i1 false)
   ret void
 }
 
 define void @test_different_source_gep(i8* %dst1, i8* %dst2, i8 %c) {
 ; CHECK-LABEL: @test_different_source_gep(
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
 ; CHECK-NEXT:    [[P:%.*]] = getelementptr i8, i8* %dst1, i64 64
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* [[P]], i64 64, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_different_source_gep(
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
+; MCO-MSSA-NEXT:    [[P:%.*]] = getelementptr i8, i8* %dst1, i64 64
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* [[P]], i64 64, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
   ; FIXME: We could optimize this as well.
   %p = getelementptr i8, i8* %dst1, i64 64
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %p, i64 64, i32 1, i1 false)
   ret void
 }
 
 define void @test_variable_size_1(i8* %dst1, i64 %dst1_size, i8* %dst2, i8 %c) {
 ; CHECK-LABEL: @test_variable_size_1(
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 %dst1_size, i32 1, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_variable_size_1(
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 %dst1_size, i32 1, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 %dst1_size, i32 1, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 128, i32 1, i1 false)
   ret void
 }
 
 define void @test_variable_size_2(i8* %dst1, i8* %dst2, i64 %dst2_size, i8 %c) {
 ; CHECK-LABEL: @test_variable_size_2(
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 %dst2_size, i32 1, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test_variable_size_2(
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 %dst2_size, i32 1, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   call void @llvm.memset.p0i8.i64(i8* %dst1, i8 %c, i64 128, i32 1, i1 false)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %dst1, i64 %dst2_size, i32 1, i1 false)
   ret void
 }
 
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i32, i1)
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i32, i1)
 declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1)
 declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i32, i1)

test/Transforms/MemCpyOpt/nontemporal.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -memcpyopt -S | FileCheck %s
+; RUN: opt < %s -memcpyopt-mssa -S | FileCheck %s --check-prefix=MCO-MSSA
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; Verify that we don't combine nontemporal stores into memset calls.
 
 define void @nontemporal_stores_1(<4 x float>* nocapture %dst) {
 ; CHECK-LABEL: @nontemporal_stores_1(
 ; CHECK-NEXT:  entry:

 ; CHECK-NEXT:    [[PTR5:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 5
 ; CHECK-NEXT:    store <4 x float> zeroinitializer, <4 x float>* [[PTR5]], align 16, !nontemporal !0
 ; CHECK-NEXT:    [[PTR6:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 6
 ; CHECK-NEXT:    store <4 x float> zeroinitializer, <4 x float>* [[PTR6]], align 16, !nontemporal !0
 ; CHECK-NEXT:    [[PTR7:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 7
 ; CHECK-NEXT:    store <4 x float> zeroinitializer, <4 x float>* [[PTR7]], align 16, !nontemporal !0
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @nontemporal_stores_1(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    store <4 x float> zeroinitializer, <4 x float>* %dst, align 16, !nontemporal !0
+; MCO-MSSA-NEXT:    [[PTR1:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 1
+; MCO-MSSA-NEXT:    store <4 x float> zeroinitializer, <4 x float>* [[PTR1]], align 16, !nontemporal !0
+; MCO-MSSA-NEXT:    [[PTR2:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 2
+; MCO-MSSA-NEXT:    store <4 x float> zeroinitializer, <4 x float>* [[PTR2]], align 16, !nontemporal !0
+; MCO-MSSA-NEXT:    [[PTR3:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 3
+; MCO-MSSA-NEXT:    store <4 x float> zeroinitializer, <4 x float>* [[PTR3]], align 16, !nontemporal !0
+; MCO-MSSA-NEXT:    [[PTR4:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 4
+; MCO-MSSA-NEXT:    store <4 x float> zeroinitializer, <4 x float>* [[PTR4]], align 16, !nontemporal !0
+; MCO-MSSA-NEXT:    [[PTR5:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 5
+; MCO-MSSA-NEXT:    store <4 x float> zeroinitializer, <4 x float>* [[PTR5]], align 16, !nontemporal !0
+; MCO-MSSA-NEXT:    [[PTR6:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 6
+; MCO-MSSA-NEXT:    store <4 x float> zeroinitializer, <4 x float>* [[PTR6]], align 16, !nontemporal !0
+; MCO-MSSA-NEXT:    [[PTR7:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 7
+; MCO-MSSA-NEXT:    store <4 x float> zeroinitializer, <4 x float>* [[PTR7]], align 16, !nontemporal !0
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   store <4 x float> zeroinitializer, <4 x float>* %dst, align 16, !nontemporal !0
   %ptr1 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 1
   store <4 x float> zeroinitializer, <4 x float>* %ptr1, align 16, !nontemporal !0
   %ptr2 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 2
   store <4 x float> zeroinitializer, <4 x float>* %ptr2, align 16, !nontemporal !0
   %ptr3 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 3
   store <4 x float> zeroinitializer, <4 x float>* %ptr3, align 16, !nontemporal !0

 define void @nontemporal_stores_2(<4 x float>* nocapture %dst) {
 ; CHECK-LABEL: @nontemporal_stores_2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    store <4 x float> zeroinitializer, <4 x float>* %dst, align 16, !nontemporal !0
 ; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 1
 ; CHECK-NEXT:    store <4 x float> zeroinitializer, <4 x float>* [[PTR1]], align 16, !nontemporal !0
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @nontemporal_stores_2(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    store <4 x float> zeroinitializer, <4 x float>* %dst, align 16, !nontemporal !0
+; MCO-MSSA-NEXT:    [[PTR1:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 1
+; MCO-MSSA-NEXT:    store <4 x float> zeroinitializer, <4 x float>* [[PTR1]], align 16, !nontemporal !0
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   store <4 x float> zeroinitializer, <4 x float>* %dst, align 16, !nontemporal !0
   %ptr1 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 1
   store <4 x float> zeroinitializer, <4 x float>* %ptr1, align 16, !nontemporal !0
   ret void
 }
 
 !0 = !{i32 1}

test/Transforms/MemCpyOpt/pr29105.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -memcpyopt -instcombine -S %s | FileCheck %s
+; RUN: opt -memcpyopt-mssa -instcombine -S %s | FileCheck %s --check-prefix=MCO-MSSA
 %Foo = type { [2048 x i64] }
 
 ; Make sure that all mempcy calls are converted to memset calls, or removed.
 define void @baz() unnamed_addr #0 {
 ; CHECK-LABEL: @baz(
 ; CHECK-NEXT:  entry-block:
 ; CHECK-NEXT:    [[TMP2:%.*]] = alloca %Foo, align 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast %Foo* [[TMP2]] to i8*
 ; CHECK-NEXT:    call void @llvm.lifetime.start(i64 16384, i8* [[TMP0]])
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP0]], i8 0, i64 16384, i32 8, i1 false)
 ; CHECK-NEXT:    call void @bar(%Foo* noalias nocapture nonnull dereferenceable(16384) [[TMP2]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end(i64 16384, i8* [[TMP0]])
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @baz(
+; MCO-MSSA-NEXT:  entry-block:
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = alloca %Foo, align 8
+; MCO-MSSA-NEXT:    [[TMP223:%.*]] = bitcast %Foo* [[TMP2]] to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP223]], i8 0, i64 16384, i32 8, i1 false)
+; MCO-MSSA-NEXT:    [[TMP0:%.*]] = bitcast %Foo* [[TMP2]] to i8*
+; MCO-MSSA-NEXT:    call void @llvm.lifetime.start(i64 16384, i8* [[TMP0]])
+; MCO-MSSA-NEXT:    call void @bar(%Foo* noalias nocapture nonnull dereferenceable(16384) [[TMP2]])
+; MCO-MSSA-NEXT:    call void @llvm.lifetime.end(i64 16384, i8* [[TMP0]])
+; MCO-MSSA-NEXT:    ret void
+;
 entry-block:
   %x.sroa.0 = alloca [2048 x i64], align 8
   %tmp0 = alloca [2048 x i64], align 8
   %0 = bitcast [2048 x i64]* %tmp0 to i8*
   %tmp2 = alloca %Foo, align 8
   %x.sroa.0.0..sroa_cast6 = bitcast [2048 x i64]* %x.sroa.0 to i8*
   call void @llvm.lifetime.start(i64 16384, i8* %x.sroa.0.0..sroa_cast6)
   call void @llvm.lifetime.start(i64 16384, i8* %0)

test/Transforms/MemCpyOpt/profitable-memset.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -memcpyopt -S | FileCheck %s
+; RUN: opt < %s -memcpyopt-mssa -S | FileCheck %s --check-prefix=MCO-MSSA
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 
 define void @foo(i64* nocapture %P) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64* %P to i16*
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, i16* [[TMP0]], i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[ARRAYIDX]] to i32*
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i16, i16* [[TMP0]], i64 3
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16* [[TMP0]] to i8*
 ; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP2]], i8 0, i64 8, i32 2, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @foo(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[TMP0:%.*]] = bitcast i64* %P to i16*
+; MCO-MSSA-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, i16* [[TMP0]], i64 1
+; MCO-MSSA-NEXT:    [[TMP1:%.*]] = bitcast i16* [[ARRAYIDX]] to i32*
+; MCO-MSSA-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i16, i16* [[TMP0]], i64 3
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = bitcast i16* [[TMP0]] to i8*
+; MCO-MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[TMP2]], i8 0, i64 8, i32 2, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %0 = bitcast i64* %P to i16*
   %arrayidx = getelementptr inbounds i16, i16* %0, i64 1
   %1 = bitcast i16* %arrayidx to i32*
   %arrayidx1 = getelementptr inbounds i16, i16* %0, i64 3
   store i16 0, i16* %0, align 2
   store i32 0, i32* %1, align 4
   store i16 0, i16* %arrayidx1, align 2
   ret void
 }
 

test/Transforms/MemCpyOpt/smaller.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -memcpyopt -S < %s | FileCheck %s
 ; RUN: opt -passes=memcpyopt -S < %s | FileCheck %s
+; RUN: opt -memcpyopt-mssa -S < %s | FileCheck %s --check-prefix=MCO-MSSA
+; RUN: opt -passes=memcpyopt-mssa -S < %s | FileCheck %s --check-prefix=MCO-MSSA
 ; rdar://8875553
 
 ; Memcpyopt shouldn't optimize the second memcpy using the first
 ; because the first has a smaller size.
 
 target datalayout = "e-p:32:32:32"
 
 %struct.s = type { [11 x i8], i32 }

 ; CHECK-NEXT:    [[AGG_TMP:%.*]] = alloca %struct.s, align 4
 ; CHECK-NEXT:    store i32 99, i32* getelementptr inbounds (%struct.s, %struct.s* @cell, i32 0, i32 1), align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* getelementptr inbounds (%struct.s, %struct.s* @cell, i32 0, i32 0, i32 0), i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 11, i32 1, i1 false)
 ; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds %struct.s, %struct.s* [[AGG_TMP]], i32 0, i32 0, i32 0
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[TMP]], i8* getelementptr inbounds (%struct.s, %struct.s* @cell, i32 0, i32 0, i32 0), i32 16, i32 4, i1 false)
 ; CHECK-NEXT:    call void @check(%struct.s* byval [[AGG_TMP]])
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @foo(
+; MCO-MSSA-NEXT:  entry:
+; MCO-MSSA-NEXT:    [[AGG_TMP:%.*]] = alloca %struct.s, align 4
+; MCO-MSSA-NEXT:    store i32 99, i32* getelementptr inbounds (%struct.s, %struct.s* @cell, i32 0, i32 1), align 4
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* getelementptr inbounds (%struct.s, %struct.s* @cell, i32 0, i32 0, i32 0), i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 11, i32 1, i1 false)
+; MCO-MSSA-NEXT:    [[TMP:%.*]] = getelementptr inbounds %struct.s, %struct.s* [[AGG_TMP]], i32 0, i32 0, i32 0
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[TMP]], i8* getelementptr inbounds (%struct.s, %struct.s* @cell, i32 0, i32 0, i32 0), i32 16, i32 4, i1 false)
+; MCO-MSSA-NEXT:    call void @check(%struct.s* byval [[AGG_TMP]])
+; MCO-MSSA-NEXT:    ret void
+;
 entry:
   %agg.tmp = alloca %struct.s, align 4
   store i32 99, i32* getelementptr inbounds (%struct.s, %struct.s* @cell, i32 0, i32 1), align 4
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* getelementptr inbounds (%struct.s, %struct.s* @cell, i32 0, i32 0, i32 0), i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 11, i32 1, i1 false)
   %tmp = getelementptr inbounds %struct.s, %struct.s* %agg.tmp, i32 0, i32 0, i32 0
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %tmp, i8* getelementptr inbounds (%struct.s, %struct.s* @cell, i32 0, i32 0, i32 0), i32 16, i32 4, i1 false)
   call void @check(%struct.s* byval %agg.tmp)
   ret void
 }

test/Transforms/MemCpyOpt/sret.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -memcpyopt -S | not grep "call.*memcpy"
+; RUN: opt < %s -basicaa -memcpyopt-mssa -S | not grep "call.*memcpy"
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 target triple = "i686-apple-darwin9"
 
 %0 = type { x86_fp80, x86_fp80 }
 
 define void @ccosl(%0* noalias sret %agg.result, %0* byval align 8 %z) nounwind {
 entry:

test/Transforms/Util/combine-alias-scope-metadata.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -S -basicaa -memcpyopt | FileCheck %s
+; RUN: opt < %s -S -basicaa -memcpyopt-mssa | FileCheck %s --check-prefix=MCO-MSSA
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define void @test(i8* noalias dereferenceable(1) %in, i8* noalias dereferenceable(1) %out) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:    [[TMP:%.*]] = alloca i8
 ; CHECK-NEXT:    [[TMP2:%.*]] = alloca i8
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %out, i8* %in, i64 1, i32 8, i1 false)
 ; CHECK-NEXT:    ret void
 ;
+; MCO-MSSA-LABEL: @test(
+; MCO-MSSA-NEXT:    [[TMP:%.*]] = alloca i8
+; MCO-MSSA-NEXT:    [[TMP2:%.*]] = alloca i8
+; MCO-MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* %out, i8* %in, i64 1, i32 8, i1 false)
+; MCO-MSSA-NEXT:    ret void
+;
   %tmp = alloca i8
   %tmp2 = alloca i8
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %tmp, i8* %in, i64 1, i32 8, i1 false), !alias.scope !4
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %tmp2, i8* %tmp, i64 1, i32 8, i1 false), !alias.scope !5
 
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %out, i8* %tmp2, i64 1, i32 8, i1 false), !noalias !6
 
   ret void