[PPC] WIP - Add statistics for mem* fucntions

[sfertile]

Summary

Add Statistics for memset/memcpy/memmove/memcmp functions.

Counts the number of times one of these functions is lowered, and if the call has a know size at compile time. If the Call has a know size then we bucket the size into small/med/large.

For the intrinsics, the alignment of the arguments is checked and recorded if they have

1. worst-case alignment
2. ideal alignment

Finally I add an option that allows tuning the maximum number of memory-operatons done in a n expansion of memcpy/memset/memmove.

[inouehrs]

lib/CodeGen/SelectionDAG/SelectionDAG.cpp

78

a small nit: memove -> memmove

119

Is if (Align % 16 == 0) better to support Align = 32, 64, ... ?

[sfertile]

lib/CodeGen/SelectionDAG/SelectionDAG.cpp

119

Your right, I'll change this to Align >= 16, I don't necessarily care that its exactly 16 byte aligned, rather I'm interested in it being at least 16 byte aligned.

[sfertile]

Fixed some spelling mistakes.
Changed some of the stat descriptions so they are the same for each memory function.
Changed Aligned16 to check for at least 16 byte alignment rather than exactly 16 byte alignment.
Expanded the number of sizes tracked.

[sfertile]

Changed some statistic descriptions, and expanded alignment stats to record 2, 4, 8 alignments as well.

[sfertile]

rebased patch to top of trunk

lib/CodeGen/SelectionDAG/SelectionDAG.cpp

 
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "SDNodeDbgValue.h"
 #include "llvm/ADT/APSInt.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
 #include "llvm/IR/CallingConv.h"

 #include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
 #include <algorithm>
 #include <cmath>
 #include <utility>
 
 using namespace llvm;
 
+#define DEBUG_TYPE "mem_funcs"
+
+// Stats related to expansion of memcpy calls.
+STATISTIC(NumMemcpyCalls, "Count of all memcpy calls lowered");
+STATISTIC(NumMemcpyNonConstSize,
+          "memcpy calls whose size is not known at compile time");
+STATISTIC(NumMemcpyTooLarge,
+          "memcpy calls with a known size larger then the max we expand");
+STATISTIC(NumMemCpyTargetExpanded,
+          "memcpy calls expanded with the target specific expansion");
+// Align related stats for memcpy
+STATISTIC(NumMemcpyUnaligned,
+          "memcpy calls whose operands have worst case alignment info");
+STATISTIC(NumMemcpyAligned16,
+          "memcpy calls whose operands are 16 byte aligned");
+// Size related stats for memcpy
+STATISTIC(NumMemcpy8, "memcpy calls with size <= 8");
+STATISTIC(NumMemcpy32, "memcpy calls with 8 < size <= 32");
+STATISTIC(NumMemcpy128, "memcpy calls with 32 < size <= 128");
+STATISTIC(NumMemcpy384, "memcpy calls with 128 < size <= 384");
+STATISTIC(NumMemcpyXL, "memcpy calls with 384 < size");
+
+
+// Stats related to expansion of memmove calls.
+STATISTIC(NumMemmoveCalls, "Count of all memmove calls lowered");
+STATISTIC(NumMemmoveTooLarge,
+          "memmove calls with a known size larger then the max we expand");
+STATISTIC(NumMemmoveNonConstSize,
+          "memmove calls whose size is not known at compile time");
+STATISTIC(NumMemmoveTargetExpanded,
+          "memmove calls expanded with the target specific expansion");
+// Align related stats for memmove
+STATISTIC(NumMemmoveUnaligned,
+          "memmove calls whose operands have worst case alignment info");
+STATISTIC(NumMemmoveAligned16,
+          "memmove calls whose operands are 16 byte aligned");
+// Size related stats for memmove
+STATISTIC(NumMemmove8, "memmove calls with size <= 8");
+STATISTIC(NumMemmove32, "memmove calls with 8 < size <= 32");
+STATISTIC(NumMemmove128, "memmove calls with 32 < size <= 128");
+STATISTIC(NumMemmove384, "memmove calls with 128 < size <= 384");
+STATISTIC(NumMemmoveXL, "memmove calls with 384 < size");
+
+// Stats relate to expansion of memset calls.
+STATISTIC(NumMemsetCalls, "Count of all memset calls lowered");
+STATISTIC(NumMemsetNonConstSize,
+          "memset calls whose size is not known at compile time");
+STATISTIC(NumMemsetTooLarge,
+          "memset calls with a known size larger then the max we expand");
+STATISTIC(NumMemsetTargetExpanded,
+          "memset calls expanded with the target specific expansion");
+// Align related stats for memset
+STATISTIC(NumMemsetUnaligned,
+          "memset calls whose operands have worst case alignment info");
+STATISTIC(NumMemsetAligned16,
+          "memmove calls whose operand is 16 byte aligned");
+// Size related stats for memmove
+STATISTIC(NumMemset8, "memset calls with size <= 8");
+STATISTIC(NumMemset32, "memset calls with 8 < size <= 32");
+STATISTIC(NumMemset128, "memset calls with 32 < size <= 128");
+STATISTIC(NumMemset384, "memset calls with 128 < size <= 384");
+STATISTIC(NumMemsetXL, "memset calls with 384 < size");
+
+static void
+memAlignStat(unsigned Align, Statistic * const Unaligned,
+                    Statistic * const VectorAligned) {
+  if (Align == 0 || Align == 1)
+    *Unaligned += 1;
+  else if (Align >= 16)
+    *VectorAligned += 1;
+}
+
+static void
+memSizeStat(unsigned Size, Statistic * const Size8, Statistic *const Size32,
+            Statistic * const Size128, Statistic * const Size384,
+            Statistic * const SizeXL) {
+  if (Size <= 8)
+    *Size8 += 1;
+  else if (Size <= 32)
+    *Size32 += 1;
+  else if (Size <= 128)
+    *Size128 += 1;
+  else if (Size <= 384)
+    *Size384 += 1;
+  else
+    *SizeXL += 1;
+}
+
 /// makeVTList - Return an instance of the SDVTList struct initialized with the
 /// specified members.
 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
   SDVTList Res = {VTs, NumVTs};
   return Res;
 }
 
 // Default null implementations of the callbacks.
static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
 
   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
                                 (DstAlignCanChange ? 0 : Align),
                                 (isZeroStr ? 0 : SrcAlign),
                                 false, false, CopyFromStr, true,
                                 DstPtrInfo.getAddrSpace(),
                                 SrcPtrInfo.getAddrSpace(),
-                                DAG, TLI))
+                                DAG, TLI)) {
+    // If we failed to find an Optimal memory-op lowering its because the
+    // expansion is too large.
+    ++NumMemcpyTooLarge;
     return SDValue();
+  }
 
   if (DstAlignCanChange) {
     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
     unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
 
     // Don't promote to an alignment that would require dynamic stack
     // realignment.
     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
if (Align > SrcAlign)
     SrcAlign = Align;
   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
 
   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
                                 (DstAlignCanChange ? 0 : Align), SrcAlign,
                                 false, false, false, false,
                                 DstPtrInfo.getAddrSpace(),
                                 SrcPtrInfo.getAddrSpace(),
-                                DAG, TLI))
+                                DAG, TLI)) {
+    // If we failed to find an Optimal memory-op lowering its because the
+    // expansion is too large.
+    ++NumMemmoveTooLarge;
     return SDValue();
+  }
 
   if (DstAlignCanChange) {
     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
     unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
     if (NewAlign > Align) {
       // Give the stack frame object a larger alignment if needed.
       if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
         MFI.setObjectAlignment(FI->getIndex(), NewAlign);
static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
     DstAlignCanChange = true;
   bool IsZeroVal =
     isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
   if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
                                 Size, (DstAlignCanChange ? 0 : Align), 0,
                                 true, IsZeroVal, false, true,
                                 DstPtrInfo.getAddrSpace(), ~0u,
-                                DAG, TLI))
+                                DAG, TLI)) {
+    // If we failed to find an Optimal memory-op lowering its because the
+    // expansion is too large.
+    ++NumMemsetTooLarge;
     return SDValue();
+  }
 
   if (DstAlignCanChange) {
     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
     unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
     if (NewAlign > Align) {
       // Give the stack frame object a larger alignment if needed.
       if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
         MFI.setObjectAlignment(FI->getIndex(), NewAlign);

 
 SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
                                 SDValue Src, SDValue Size, unsigned Align,
                                 bool isVol, bool AlwaysInline, bool isTailCall,
                                 MachinePointerInfo DstPtrInfo,
                                 MachinePointerInfo SrcPtrInfo) {
   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
 
+  ++NumMemcpyCalls;
+  memAlignStat(Align, &NumMemcpyUnaligned, &NumMemcpyAligned16);
+
   // Check to see if we should lower the memcpy to loads and stores first.
   // For cases within the target-specified limits, this is the best choice.
   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
   if (ConstantSize) {
     // Memcpy with size zero? Just return the original chain.
     if (ConstantSize->isNullValue())
       return Chain;
 
+    uint64_t Size = ConstantSize->getZExtValue();
+    memSizeStat(Size, &NumMemcpy8, &NumMemcpy32, &NumMemcpy128,
+                &NumMemcpy384, &NumMemcpyXL);
     SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
-                                             ConstantSize->getZExtValue(),Align,
-                                isVol, false, DstPtrInfo, SrcPtrInfo);
+                                             Size, Align, isVol, false,
+                                             DstPtrInfo, SrcPtrInfo);
     if (Result.getNode())
       return Result;
+  } else {
+    // Size isn't know at compile time.
+    ++NumMemcpyNonConstSize;
   }
 
   // Then check to see if we should lower the memcpy with target-specific
   // code. If the target chooses to do this, this is the next best.
   if (TSI) {
     SDValue Result = TSI->EmitTargetCodeForMemcpy(
         *this, dl, Chain, Dst, Src, Size, Align, isVol, AlwaysInline,
         DstPtrInfo, SrcPtrInfo);
-    if (Result.getNode())
+    if (Result.getNode()) {
+      ++NumMemCpyTargetExpanded;
       return Result;
     }
+  }
 
   // If we really need inline code and the target declined to provide it,
   // use a (potentially long) sequence of loads and stores.
   if (AlwaysInline) {
     assert(ConstantSize && "AlwaysInline requires a constant size!");
     return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
                                    ConstantSize->getZExtValue(), Align, isVol,
                                    true, DstPtrInfo, SrcPtrInfo);

 }
 
 SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
                                  SDValue Src, SDValue Size, unsigned Align,
                                  bool isVol, bool isTailCall,
                                  MachinePointerInfo DstPtrInfo,
                                  MachinePointerInfo SrcPtrInfo) {
   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
+  ++NumMemmoveCalls;
+  memAlignStat(Align, &NumMemmoveUnaligned, &NumMemmoveAligned16);
 
   // Check to see if we should lower the memmove to loads and stores first.
   // For cases within the target-specified limits, this is the best choice.
   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
   if (ConstantSize) {
     // Memmove with size zero? Just return the original chain.
     if (ConstantSize->isNullValue())
       return Chain;
 
+    uint64_t Size = ConstantSize->getZExtValue();
+    memSizeStat(Size, &NumMemmove8, &NumMemmove32, &NumMemmove128,
+                &NumMemmove384, &NumMemmoveXL);
     SDValue Result =
       getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
-                               ConstantSize->getZExtValue(), Align, isVol,
-                               false, DstPtrInfo, SrcPtrInfo);
+                               Size, Align, isVol, false, DstPtrInfo,
+                               SrcPtrInfo);
     if (Result.getNode())
       return Result;
+  } else {
+    ++NumMemmoveNonConstSize;
   }
 
   // Then check to see if we should lower the memmove with target-specific
   // code. If the target chooses to do this, this is the next best.
   if (TSI) {
     SDValue Result = TSI->EmitTargetCodeForMemmove(
         *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
-    if (Result.getNode())
+    if (Result.getNode()) {
+      ++NumMemmoveTargetExpanded;
       return Result;
     }
+  }
 
   checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
   checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
 
   // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
   // not be safe.  See memcpy above for more details.
 
   // Emit a library call.
SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
   return CallResult.second;
 }
 
 SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
                                 SDValue Src, SDValue Size, unsigned Align,
                                 bool isVol, bool isTailCall,
                                 MachinePointerInfo DstPtrInfo) {
   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
+  ++NumMemsetCalls;
+  memAlignStat(Align, &NumMemsetUnaligned, &NumMemsetAligned16);
 
   // Check to see if we should lower the memset to stores first.
   // For cases within the target-specified limits, this is the best choice.
   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
   if (ConstantSize) {
     // Memset with size zero? Just return the original chain.
     if (ConstantSize->isNullValue())
       return Chain;
 
+    uint64_t Size = ConstantSize->getZExtValue();
+    memSizeStat(Size, &NumMemset8, &NumMemset32, &NumMemset128,
+                &NumMemset384, &NumMemsetXL);
     SDValue Result =
-      getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
-                      Align, isVol, DstPtrInfo);
+      getMemsetStores(*this, dl, Chain, Dst, Src, Size, Align, isVol,
+                      DstPtrInfo);
 
     if (Result.getNode())
       return Result;
+  } else {
+    ++NumMemsetNonConstSize;
   }
 
   // Then check to see if we should lower the memset with target-specific
   // code. If the target chooses to do this, this is the next best.
   if (TSI) {
     SDValue Result = TSI->EmitTargetCodeForMemset(
         *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo);
-    if (Result.getNode())
+    if (Result.getNode()) {
+      ++NumMemsetTargetExpanded;
       return Result;
     }
+  }
 
   checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
 
   // Emit a library call.
   Type *IntPtrTy = getDataLayout().getIntPtrType(*getContext());
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
   Entry.Node = Dst; Entry.Ty = IntPtrTy;

lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Target/TargetSubtargetInfo.h"
 #include <algorithm>
 #include <utility>
 using namespace llvm;
 
 #define DEBUG_TYPE "isel"
 
+STATISTIC(NumMemcmpCalls, "Count of all memcmp calls lowered");
+STATISTIC(NumMemcmpNonConstSize,
+          "memcmp calls whose size is not known at compile time");
+STATISTIC(NumSmallMemcmp, "memcmp calls with size <= 32");
+STATISTIC(NumMedMemcmp, "memcmp calls with 32 < size <= 128");
+STATISTIC(NumLargMemcmp, "memcmp calls with 128 < size");
+
 /// LimitFloatPrecision - Generate low-precision inline sequences for
 /// some float libcalls (6, 8 or 12 bits).
 static unsigned LimitFloatPrecision;
 
 static cl::opt<unsigned, true>
 LimitFPPrecision("limit-float-precision",
                  cl::desc("Generate low-precision inline sequences "
                           "for some float libcalls"),
else
     Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
   setValue(&I, Value);
 }
 
 /// visitMemCmpCall - See if we can lower a call to memcmp in an optimized form.
 /// If so, return true and lower it, otherwise return false and it will be
 /// lowered like a normal call.
 bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
+  ++NumMemcmpCalls;
+
   // Verify that the prototype makes sense.  int memcmp(void*,void*,size_t)
   if (I.getNumArgOperands() != 3)
     return false;
 
   const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
   if (!LHS->getType()->isPointerTy() || !RHS->getType()->isPointerTy() ||
       !I.getArgOperand(2)->getType()->isIntegerTy() ||
       !I.getType()->isIntegerTy())
     return false;
 
   const Value *Size = I.getArgOperand(2);
   const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
   if (CSize && CSize->getZExtValue() == 0) {
     EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                           I.getType(), true);
     setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
     return true;
   }
 
+  if(!CSize) {
+    ++NumMemcmpNonConstSize;
+  } else {
+    uint64_t CmpSize = CSize->getZExtValue();
+    if (CmpSize <= 32)
+         ++NumSmallMemcmp;
+    else if (CmpSize <= 128)
+         ++NumMedMemcmp;
+    else
+         ++NumLargMemcmp;
+  }
+
   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
   std::pair<SDValue, SDValue> Res =
     TSI.EmitTargetCodeForMemcmp(DAG, getCurSDLoc(), DAG.getRoot(),
                                 getValue(LHS), getValue(RHS), getValue(Size),
                                 MachinePointerInfo(LHS),
                                 MachinePointerInfo(RHS));
   if (Res.first.getNode()) {
     processIntegerCallValue(I, Res.first, true);

lib/Target/PowerPC/PPCISelLowering.cpp

 using namespace llvm;
 
 #define DEBUG_TYPE "ppc-lowering"
 
 static cl::opt<bool> DisablePPCPreinc("disable-ppc-preinc",
 cl::desc("disable preincrement load/store generation on PPC"), cl::Hidden);
 
 static cl::opt<bool> DisableILPPref("disable-ppc-ilp-pref",
-cl::desc("disable setting the node scheduling preference to ILP on PPC"), cl::Hidden);
+cl::desc("disable setting the node scheduling preference to ILP on PPC"),
+ cl::Hidden);
 
 static cl::opt<bool> DisablePPCUnaligned("disable-ppc-unaligned",
 cl::desc("disable unaligned load/store generation on PPC"), cl::Hidden);
 
 static cl::opt<bool> DisableSCO("disable-ppc-sco",
 cl::desc("disable sibling call optimization on ppc"), cl::Hidden);
 
+static cl::opt<unsigned> OpMaxStoresPerMemset("max-stores-memset",
+cl::desc("Set the maximum number of stores in a memset expansion"),
+cl::Hidden, cl::init(8U));
+
+static cl::opt<unsigned> OpMaxStoresPerMemcpy("max-stores-memcpy",
+cl::desc("Set the maximum number of Load/Store pairs in a memcpy expansion"),
+cl::Hidden, cl::init(8U));
+
+static cl::opt<unsigned> OpMaxStoresPerMemmove("max-stores-memmove",
+cl::desc("Set the maximum number of Load/Store pairs in a memmove expansion"),
+cl::Hidden, cl::init(8U));
+
 STATISTIC(NumTailCalls, "Number of tail calls");
 STATISTIC(NumSiblingCalls, "Number of sibling calls");
 
 // FIXME: Remove this once the bug has been fixed!
 extern cl::opt<bool> ANDIGlueBug;
 
 PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
                                      const PPCSubtarget &STI)
PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
   }
 
   setTruncStoreAction(MVT::f64, MVT::f32, Expand);
 
   // PowerPC has pre-inc load and store's.
   setIndexedLoadAction(ISD::PRE_INC, MVT::i1, Legal);
   setIndexedLoadAction(ISD::PRE_INC, MVT::i8, Legal);
   setIndexedLoadAction(ISD::PRE_INC, MVT::i16, Legal);
+
   setIndexedLoadAction(ISD::PRE_INC, MVT::i32, Legal);
   setIndexedLoadAction(ISD::PRE_INC, MVT::i64, Legal);
   setIndexedLoadAction(ISD::PRE_INC, MVT::f32, Legal);
   setIndexedLoadAction(ISD::PRE_INC, MVT::f64, Legal);
   setIndexedStoreAction(ISD::PRE_INC, MVT::i1, Legal);
   setIndexedStoreAction(ISD::PRE_INC, MVT::i8, Legal);
   setIndexedStoreAction(ISD::PRE_INC, MVT::i16, Legal);
   setIndexedStoreAction(ISD::PRE_INC, MVT::i32, Legal);
if (Subtarget.getDarwinDirective() == PPC::DIR_E500mc ||
     MaxStoresPerMemmoveOptSize = 8;
   } else if (Subtarget.getDarwinDirective() == PPC::DIR_A2) {
     // The A2 also benefits from (very) aggressive inlining of memcpy and
     // friends. The overhead of a the function call, even when warm, can be
     // over one hundred cycles.
     MaxStoresPerMemset = 128;
     MaxStoresPerMemcpy = 128;
     MaxStoresPerMemmove = 128;
+  } else {
+    MaxStoresPerMemset = OpMaxStoresPerMemset;
+    MaxStoresPerMemcpy = OpMaxStoresPerMemcpy;
+    MaxStoresPerMemmove = OpMaxStoresPerMemmove;
   }
 }
 
 /// getMaxByValAlign - Helper for getByValTypeAlignment to determine
 /// the desired ByVal argument alignment.
 static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign,
                              unsigned MaxMaxAlign) {
   if (MaxAlign == MaxMaxAlign)