[DAG] Relax type restriction for store merge

[niravd]

Summary

Allow stores of bitcastable types to be merged by peeking through BITCAST nodes and recasting stored values constant and vector extract nodes as necessary.

[niravd]

Add missing check for typesize equality and explicitly avoid truncating stores.

[niravd]

Resolve outstanding comments and rebase over latned extracted NFCI patch.

[niravd]

Ignore this last patch. It should have gone to another diff. Will overwrite with correct version presently.

[RKSimon]

This looks like the D34472 patch?

[niravd]

overwrite accidental patch.

[RKSimon]

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

12496

Val.getOpcode()

12959

StVal.getOpcode()

[niravd]

Simpify, add comments, and address nits discussed.

[niravd]

Addressed missed nits.

[RKSimon]

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

12640

It'd probably be better if you avoid the ternary inside the if().

if (MemVT.isInteger() && !MemVT.bitsEq(Other->getMemoryVT()))
   return false;
if (!MemVT.isInteger() && Other->getMemoryVT() != MemVT)
   return false;

or

if ((MemVT.isInteger() && !MemVT.bitsEq(Other->getMemoryVT())) ||
    (!MemVT.isInteger() && Other->getMemoryVT() != MemVT))
   return false;

That or pull it out as a bool variable in the line above.

12654

Another ternary inside an if()

[niravd]

extract ternary operators from conditionals

[niravd]

Cleanup the mergestore helper function casting and add missing extract_subvector casting logic.

[niravd]

Rework to regularize and simplify bitcasting logic.

[niravd]

Rebase past landed NFC commits.

[RKSimon]

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

484

A lot of these comment changes look like NFCs that can go in separately.

12469

Please can you create a static helper that performs this? We have a lot of peek through bitcasts in this patch alone....

12471

Val.getValueSizeInBits()

12505

Merge these ifs() to reduce indentation?

if (MemVT != Val.getValueType() &&
    (Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
     Val.getOpcode() == ISD::EXTRACT_SUBVECTOR) {

[niravd]

Pull out some NFC changes and use peekThroughBitcast function

[RKSimon]

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

12474

Why isn't this CFP->getValueAPF().bitcastToAPInt().zextOrTrunc(8 * ElementSizeBytes) ?

12633

Pull the ternary operator out of the if()

12667

NFC change

[niravd]

Excise NFC change and address comments.

[RKSimon]

LGTM - thanks

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
                           EVT LoadResultTy, EVT &ExtVT, EVT &LoadedVT,
                           bool &NarrowLoad);
 
     /// Helper function for MergeConsecutiveStores which merges the
     /// component store chains.
     SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
                                 unsigned NumStores);
 
-    /// This is a helper function for MergeConsecutiveStores. When the source
-    /// elements of the consecutive stores are all constants or all extracted
-    /// vector elements, try to merge them into one larger store.
+    /// This is a helper function for MergeConsecutiveStores. When the
+    /// source elements of the consecutive stores are all constants or
+    /// all extracted vector elements, try to merge them into one
+    /// larger store introducing bitcasts if necessary.
     /// \return True if a merged store was created.
     bool MergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
                                          EVT MemVT, unsigned NumStores,
                                          bool IsConstantSrc, bool UseVector,
                                          bool UseTrunc);
 
-    /// This is a helper function for MergeConsecutiveStores.
-    /// Stores that may be merged are placed in StoreNodes.
+    /// This is a helper function for MergeConsecutiveStores.  Stores
+    /// that may potentially merged with St are placed in
+    /// StoreNodes.
     void getStoreMergeCandidates(StoreSDNode *St,
                                  SmallVectorImpl<MemOpLink> &StoreNodes);
 
     /// Helper function for MergeConsecutiveStores. Checks if
     /// Candidate stores have indirect dependency through their
-    /// operands. \return True if safe to merge
+    /// operands. \return True if safe to merge.
     bool checkMergeStoreCandidatesForDependencies(
         SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores);
 
     /// Merge consecutive store operations into a wide store.
     /// This optimization uses wide integers or vectors when possible.
     /// \return number of stores that were merged into a merged store (the
     /// affected nodes are stored as a prefix in \p StoreNodes).
     bool MergeConsecutiveStores(StoreSDNode *N);

 
 bool DAGCombiner::MergeStoresOfConstantsOrVecElts(
     SmallVectorImpl<MemOpLink> &StoreNodes, EVT MemVT, unsigned NumStores,
     bool IsConstantSrc, bool UseVector, bool UseTrunc) {
   // Make sure we have something to merge.
   if (NumStores < 2)
     return false;
 
-  int64_t ElementSizeBytes = MemVT.getSizeInBits() / 8;
-
   // The latest Node in the DAG.
   SDLoc DL(StoreNodes[0].MemNode);
 
-  SDValue StoredVal;
+  int64_t ElementSizeBytes = MemVT.getSizeInBits() / 8;
+  unsigned SizeInBits = NumStores * ElementSizeBytes * 8;
+
+  EVT StoreTy;
   if (UseVector) {
-    bool IsVec = MemVT.isVector();
     unsigned Elts = NumStores;
-    if (IsVec) {
-      // When merging vector stores, get the total number of elements.
+    if (MemVT.isVector())
       Elts *= MemVT.getVectorNumElements();
-    }
     // Get the type for the merged vector store.
-    EVT Ty = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(), Elts);
-    assert(TLI.isTypeLegal(Ty) && "Illegal vector store");
+    StoreTy = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(), Elts);
+  } else
+    StoreTy = EVT::getIntegerVT(*DAG.getContext(), SizeInBits);
 
+  SDValue StoredVal;
+  if (UseVector) {
     if (IsConstantSrc) {
       SmallVector<SDValue, 8> BuildVector;
-      for (unsigned I = 0, E = Ty.getVectorNumElements(); I != E; ++I) {
+      for (unsigned I = 0; I != NumStores; ++I) {
         StoreSDNode *St = cast<StoreSDNode>(StoreNodes[I].MemNode);
         SDValue Val = St->getValue();
-        if (MemVT.getScalarType().isInteger())
-          if (auto *CFP = dyn_cast<ConstantFPSDNode>(St->getValue()))
+        // Make sure constant is correct bit-size.
+        if (MemVT != Val.getValueType()) {
+          while (Val.getOpcode() == ISD::BITCAST)
+            Val = Val.getOperand(0);
+          if (ElementSizeBytes * 8 != Val.getValueType().getSizeInBits()) {
+            EVT IntMemVT =
+                EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits());
+            if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val))
               Val = DAG.getConstant(
                   (uint32_t)CFP->getValueAPF().bitcastToAPInt().getZExtValue(),
-                SDLoc(CFP), MemVT);
+                  SDLoc(CFP), IntMemVT);
+            if (auto *C = dyn_cast<ConstantSDNode>(Val))
+              Val = DAG.getConstant(
+                  C->getAPIntValue().zextOrTrunc(8 * ElementSizeBytes),
+                  SDLoc(C), IntMemVT);
+          }
+          // make sure it's the correct type
+          Val = DAG.getBitcast(MemVT, Val);
+        }
         BuildVector.push_back(Val);
       }
-      StoredVal = DAG.getBuildVector(Ty, DL, BuildVector);
+      StoredVal = DAG.getNode(MemVT.isVector() ? ISD::CONCAT_VECTORS
+                                               : ISD::BUILD_VECTOR,
+                              DL, StoreTy, BuildVector);
     } else {
       SmallVector<SDValue, 8> Ops;
       for (unsigned i = 0; i < NumStores; ++i) {
         StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
         SDValue Val = St->getValue();
-        // All operands of BUILD_VECTOR / CONCAT_VECTOR must have the same type.
-        if (Val.getValueType() != MemVT)
-          return false;
+        // Peek through bitcasts.
+         while (Val.getOpcode() == ISD::BITCAST)
+          Val = Val.getOperand(0);
+        // All operands of BUILD_VECTOR / CONCAT_VECTOR must be of
+        // type MemVT.  If the underlying value is not the correct
+        // type, but it is an extraction of an appropriate vector we
+        // can recast Val to be of the correct type. This may require
+        // converying between EXTRACT_VECTOR_ELT and
+        // EXTRACT_SUBVECTOR.
+        if (MemVT != Val.getValueType()) {
+          if (Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
+              Val.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
+            SDValue Vec = Val.getOperand(0);
+            // We may need to add a bitcast here to get types to line up.
+            if (MemVT != Val.getValueType()) {
+              EVT NewVecScalarTy = MemVT.getScalarType();
+              unsigned Elts = Vec.getValueType().getSizeInBits() /
+                              NewVecScalarTy.getSizeInBits();
+              EVT NewVecTy =
+                  EVT::getVectorVT(*DAG.getContext(), NewVecScalarTy, Elts);
+              Vec = DAG.getBitcast(NewVecTy, Vec);
+            }
+            if (MemVT.isVector())
+              Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Val), MemVT, Vec,
+                                Val.getOperand(1));
+            else
+              Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Val), MemVT, Vec,
+                                Val.getOperand(1));
+          }
+        }
         Ops.push_back(Val);
       }
 
       // Build the extracted vector elements back into a vector.
-      StoredVal = DAG.getNode(IsVec ? ISD::CONCAT_VECTORS : ISD::BUILD_VECTOR,
-                              DL, Ty, Ops);    }
+      StoredVal = DAG.getNode(MemVT.isVector() ? ISD::CONCAT_VECTORS
+                                               : ISD::BUILD_VECTOR,
+                              DL, StoreTy, Ops);
+    }
   } else {
     // We should always use a vector store when merging extracted vector
     // elements, so this path implies a store of constants.
     assert(IsConstantSrc && "Merged vector elements should use vector store");
 
-    unsigned SizeInBits = NumStores * ElementSizeBytes * 8;
     APInt StoreInt(SizeInBits, 0);
 
     // Construct a single integer constant which is made of the smaller
     // constant inputs.
     bool IsLE = DAG.getDataLayout().isLittleEndian();
     for (unsigned i = 0; i < NumStores; ++i) {
       unsigned Idx = IsLE ? (NumStores - 1 - i) : i;
       StoreSDNode *St  = cast<StoreSDNode>(StoreNodes[Idx].MemNode);
 
       SDValue Val = St->getValue();
       StoreInt <<= ElementSizeBytes * 8;
       if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val)) {
         StoreInt |= C->getAPIntValue().zextOrTrunc(SizeInBits);
       } else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val)) {
         StoreInt |= C->getValueAPF().bitcastToAPInt().zextOrTrunc(SizeInBits);
       } else {
         llvm_unreachable("Invalid constant element type");
       }
     }
 
     // Create the new Load and Store operations.
-    EVT StoreTy = EVT::getIntegerVT(*DAG.getContext(), SizeInBits);
     StoredVal = DAG.getConstant(StoreInt, DL, StoreTy);
   }
 
   LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
   SDValue NewChain = getMergeStoreChains(StoreNodes, NumStores);
 
   // make sure we use trunc store if it's necessary to be legal.
   SDValue NewStore;
-  if (UseVector || !UseTrunc) {
+  if (!UseTrunc) {
     NewStore = DAG.getStore(NewChain, DL, StoredVal, FirstInChain->getBasePtr(),
                             FirstInChain->getPointerInfo(),
                             FirstInChain->getAlignment());
   } else { // Must be realized as a trunc store
     EVT LegalizedStoredValueTy =
         TLI.getTypeToTransformTo(*DAG.getContext(), StoredVal.getValueType());
     unsigned LegalizedStoreSize = LegalizedStoredValueTy.getSizeInBits();
     ConstantSDNode *C = cast<ConstantSDNode>(StoredVal);

 
 void DAGCombiner::getStoreMergeCandidates(
     StoreSDNode *St, SmallVectorImpl<MemOpLink> &StoreNodes) {
   // This holds the base pointer, index, and the offset in bytes from the base
   // pointer.
   BaseIndexOffset BasePtr = BaseIndexOffset::match(St->getBasePtr(), DAG);
   EVT MemVT = St->getMemoryVT();
 
+  SDValue Val = St->getValue();
+  // Only peek through bitcasts of non-truncstores
+  while (Val->getOpcode() == ISD::BITCAST)
+    Val = Val.getOperand(0);
+
   // We must have a base and an offset.
   if (!BasePtr.getBase().getNode())
     return;
 
   // Do not handle stores to undef base pointers.
   if (BasePtr.getBase().isUndef())
     return;
 
-  bool IsConstantSrc = isa<ConstantSDNode>(St->getValue()) ||
-                       isa<ConstantFPSDNode>(St->getValue());
-  bool IsExtractVecSrc =
-      (St->getValue().getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
-       St->getValue().getOpcode() == ISD::EXTRACT_SUBVECTOR);
-  bool IsLoadSrc = isa<LoadSDNode>(St->getValue());
+  bool IsConstantSrc = isa<ConstantSDNode>(Val) || isa<ConstantFPSDNode>(Val);
+  bool IsExtractVecSrc = (Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
+                          Val.getOpcode() == ISD::EXTRACT_SUBVECTOR);
+  bool IsLoadSrc = isa<LoadSDNode>(Val);
   BaseIndexOffset LBasePtr;
   // Match on loadbaseptr if relevant.
-  if (IsLoadSrc)
-    LBasePtr = BaseIndexOffset::match(
-        cast<LoadSDNode>(St->getValue())->getBasePtr(), DAG);
-
+  EVT LoadVT;
+  if (IsLoadSrc) {
+    auto *Ld = cast<LoadSDNode>(Val);
+    LBasePtr = BaseIndexOffset::match(Ld->getBasePtr(), DAG);
+    LoadVT = Ld->getMemoryVT();
+    // Don't allow hidden truncations / extensions in merge candidates.
+    if (!MemVT.bitsEq(LoadVT))
+      return;
+  }
   auto CandidateMatch = [&](StoreSDNode *Other, BaseIndexOffset &Ptr,
                             int64_t &Offset) -> bool {
     if (Other->isVolatile() || Other->isIndexed())
       return false;
-    // We can merge constant floats to equivalent integers
-    if (Other->getMemoryVT() != MemVT)
-      if (!(MemVT.isInteger() && MemVT.bitsEq(Other->getMemoryVT()) &&
-            isa<ConstantFPSDNode>(Other->getValue())))
-        return false;
+    // Peek through bitcasts.
+    SDValue Val = Other->getValue();
+    while (Val.getOpcode() == ISD::BITCAST)
+      Val = Val.getOperand(0);
     if (IsLoadSrc) {
+      // Allow loads of different types to merge as integers.
+      if (MemVT.isInteger() ? !MemVT.bitsEq(Other->getMemoryVT())
+                            : Other->getMemoryVT() != MemVT)
+        return false;
       // The Load's Base Ptr must also match
-      if (LoadSDNode *OtherLd = dyn_cast<LoadSDNode>(Other->getValue())) {
+      if (LoadSDNode *OtherLd = dyn_cast<LoadSDNode>(Val)) {
         auto LPtr = BaseIndexOffset::match(OtherLd->getBasePtr(), DAG);
+        if (!LoadVT.bitsEq(OtherLd->getMemoryVT()))
+          return false;
         if (!(LBasePtr.equalBaseIndex(LPtr, DAG)))
           return false;
       } else
         return false;
     }
-    if (IsConstantSrc)
-      if (!(isa<ConstantSDNode>(Other->getValue()) ||
-            isa<ConstantFPSDNode>(Other->getValue())))
-        return false;
-    if (IsExtractVecSrc)
-      if (!(Other->getValue().getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
-            Other->getValue().getOpcode() == ISD::EXTRACT_SUBVECTOR))
+    if (IsConstantSrc) {
+      // Allow merging constants of different types as integers.
+      if (MemVT.isInteger() ? !MemVT.bitsEq(Other->getMemoryVT())
+                            : Other->getMemoryVT() != MemVT)
+        return false;
+      if (!(isa<ConstantSDNode>(Val) || isa<ConstantFPSDNode>(Val)))
+        return false;
+    }
+    if (IsExtractVecSrc) {
+      // Do not merge truncated stores here.
+      if (Other->isTruncatingStore())
+        return false;
+      if (!MemVT.bitsEq(Val.getValueType()) ||
+          !(Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
+            Val.getOpcode() == ISD::EXTRACT_SUBVECTOR))
         return false;
+    }
     Ptr = BaseIndexOffset::match(Other->getBasePtr(), DAG);
     return (BasePtr.equalBaseIndex(Ptr, DAG, Offset));
   };
   // We looking for a root node which is an ancestor to all mergable
   // stores. We search up through a load, to our root and then down
   // through all children. For instance we will find Store{1,2,3} if
   // St is Store1, Store2. or Store3 where the root is not a load
   // which always true for nonvolatile ops. TODO: Expand
if (ElementSizeBytes * 8 != MemVT.getSizeInBits())
     return false;
 
   if (!MemVT.isSimple())
     return false;
 
   // Perform an early exit check. Do not bother looking at stored values that
   // are not constants, loads, or extracted vector elements.
   SDValue StoredVal = St->getValue();
+  while (StoredVal->getOpcode() == ISD::BITCAST)
+    StoredVal = StoredVal.getOperand(0);
+
   bool IsLoadSrc = isa<LoadSDNode>(StoredVal);
   bool IsConstantSrc = isa<ConstantSDNode>(StoredVal) ||
                        isa<ConstantFPSDNode>(StoredVal);
   bool IsExtractVecSrc = (StoredVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
                           StoredVal.getOpcode() == ISD::EXTRACT_SUBVECTOR);
 
   if (!IsConstantSrc && !IsLoadSrc && !IsExtractVecSrc)
     return false;
 
-  // Don't merge vectors into wider vectors if the source data comes from loads.
-  // TODO: This restriction can be lifted by using logic similar to the
-  // ExtractVecSrc case.
-  if (MemVT.isVector() && IsLoadSrc)
-    return false;
-
   SmallVector<MemOpLink, 8> StoreNodes;
   // Find potential store merge candidates by searching through chain sub-DAG
   getStoreMergeCandidates(St, StoreNodes);
 
   // Check if there is anything to merge.
   if (StoreNodes.size() < 2)
     return false;
 
while (StoreNodes.size() > 1) {
     if (IsExtractVecSrc) {
       LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
       unsigned FirstStoreAS = FirstInChain->getAddressSpace();
       unsigned FirstStoreAlign = FirstInChain->getAlignment();
       unsigned NumStoresToMerge = 1;
       bool IsVec = MemVT.isVector();
       for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
         StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
-        unsigned StoreValOpcode = St->getValue().getOpcode();
+        SDValue StVal = St->getValue();
+        // Peek through bitcasts
+        while (StVal->getOpcode() == ISD::BITCAST)
+          StVal = StVal.getOperand(0);
+
         // This restriction could be loosened.
         // Bail out if any stored values are not elements extracted from a
         // vector. It should be possible to handle mixed sources, but load
         // sources need more careful handling (see the block of code below that
         // handles consecutive loads).
-        if (StoreValOpcode != ISD::EXTRACT_VECTOR_ELT &&
-            StoreValOpcode != ISD::EXTRACT_SUBVECTOR)
+        if (StVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT &&
+            StVal.getOpcode() != ISD::EXTRACT_SUBVECTOR)
           return RV;
 
         // Find a legal type for the vector store.
         unsigned Elts = i + 1;
         if (IsVec) {
           // When merging vector stores, get the total number of elements.
           Elts *= MemVT.getVectorNumElements();
         }
while (StoreNodes.size() > 1) {
     // Look for load nodes which are used by the stored values.
     SmallVector<MemOpLink, 8> LoadNodes;
 
     // Find acceptable loads. Loads need to have the same chain (token factor),
     // must not be zext, volatile, indexed, and they must be consecutive.
     BaseIndexOffset LdBasePtr;
     for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
       StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
-      LoadSDNode *Ld = dyn_cast<LoadSDNode>(St->getValue());
+      SDValue Val = St->getValue();
+      while (Val.getOpcode() == ISD::BITCAST)
+        Val = Val.getOperand(0);
+
+      LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val);
       if (!Ld)
         break;
 
       // Loads must only have one use.
       if (!Ld->hasNUsesOfValue(1, 0))
         break;
 
       // The memory operands must not be volatile.
       if (Ld->isVolatile() || Ld->isIndexed())
         break;
 
-      // We do not accept ext loads.
-      if (Ld->getExtensionType() != ISD::NON_EXTLOAD)
-        break;
-
-      // The stored memory type must be the same.
-      if (Ld->getMemoryVT() != MemVT)
+      // The stored memory type must be the same size as MemVT.
+      if (!MemVT.bitsEq(Ld->getMemoryVT()))
         break;
 
       BaseIndexOffset LdPtr = BaseIndexOffset::match(Ld->getBasePtr(), DAG);
       // If this is not the first ptr that we check.
       int64_t LdOffset = 0;
       if (LdBasePtr.getBase().getNode()) {
         // The base ptr must be the same.
         if (!LdBasePtr.equalBaseIndex(LdPtr, DAG, LdOffset))
for (unsigned i = 1; i < LoadNodes.size(); ++i) {
       if (CurrAddress - StartAddress != (ElementSizeBytes * i))
         break;
       LastConsecutiveLoad = i;
 
       if (isDereferenceable && !LoadNodes[i].MemNode->isDereferenceable())
         isDereferenceable = false;
 
       // Find a legal type for the vector store.
-      EVT StoreTy = EVT::getVectorVT(Context, MemVT, i + 1);
+      unsigned Elts =
+          (i + 1) * ((MemVT.isVector()) ? MemVT.getVectorNumElements() : 1);
+      EVT StoreTy = EVT::getVectorVT(Context, MemVT.getScalarType(), Elts);
+
       bool IsFastSt, IsFastLd;
       if (TLI.isTypeLegal(StoreTy) &&
           TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) &&
           TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS,
                                  FirstStoreAlign, &IsFastSt) &&
           IsFastSt &&
           TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstLoadAS,
                                  FirstLoadAlign, &IsFastLd) &&
if (NumElem < 2) {
       StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + 1);
       continue;
     }
 
     // Find if it is better to use vectors or integers to load and store
     // to memory.
     EVT JointMemOpVT;
     if (UseVectorTy) {
-      JointMemOpVT = EVT::getVectorVT(Context, MemVT, NumElem);
+      // Find a legal type for the vector store.
+      unsigned Elts = NumElem;
+      if (MemVT.isVector())
+        Elts *= MemVT.getVectorNumElements();
+      JointMemOpVT = EVT::getVectorVT(Context, MemVT.getScalarType(), Elts);
     } else {
       unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
       JointMemOpVT = EVT::getIntegerVT(Context, SizeInBits);
     }
 
     SDLoc LoadDL(LoadNodes[0].MemNode);
     SDLoc StoreDL(StoreNodes[0].MemNode);
 
if (UseVectorTy || !DoIntegerTruncate) {
                                    FirstInChain->getPointerInfo(), JointMemOpVT,
                                    FirstInChain->getAlignment(),
                                    FirstInChain->getMemOperand()->getFlags());
     }
 
     // Transfer chain users from old loads to the new load.
     for (unsigned i = 0; i < NumElem; ++i) {
       LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
+      AddToWorklist(Ld);
       DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1),
                                     SDValue(NewLoad.getNode(), 1));
     }
 
     // Replace the all stores with the new store.
     for (unsigned i = 0; i < NumElem; ++i)
       CombineTo(StoreNodes[i].MemNode, NewStore);
     RV = true;

test/CodeGen/X86/MergeConsecutiveStores.ll

 
 ; CHECK-LABEL: merge_vec_extract_stores
 ; CHECK:      vmovups %ymm0, 48(%rdi)
 ; CHECK-NEXT: vmovups %ymm1, 80(%rdi)
 ; CHECK-NEXT: vzeroupper
 ; CHECK-NEXT: retq
 }
 
-; Merging vector stores when sourced from vector loads is not currently handled.
+; Merging vector stores when sourced from vector loads.
 define void @merge_vec_stores_from_loads(<4 x float>* %v, <4 x float>* %ptr) {
   %load_idx0 = getelementptr inbounds <4 x float>, <4 x float>* %v, i64 0
   %load_idx1 = getelementptr inbounds <4 x float>, <4 x float>* %v, i64 1
   %v0 = load <4 x float>, <4 x float>* %load_idx0
   %v1 = load <4 x float>, <4 x float>* %load_idx1
   %store_idx0 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 0
   %store_idx1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 1
   store <4 x float> %v0, <4 x float>* %store_idx0, align 16
   store <4 x float> %v1, <4 x float>* %store_idx1, align 16
   ret void
 
 ; CHECK-LABEL: merge_vec_stores_from_loads
-; CHECK:      vmovaps
-; CHECK-NEXT: vmovaps
-; CHECK-NEXT: vmovaps
-; CHECK-NEXT: vmovaps
+; CHECK:      vmovups
+; CHECK-NEXT: vmovups
+; CHECK-NEXT: vzeroupper
 ; CHECK-NEXT: retq
 }
 
 ; Merging vector stores when sourced from a constant vector is not currently handled. 
 define void @merge_vec_stores_of_constants(<4 x i32>* %ptr) {
   %idx0 = getelementptr inbounds <4 x i32>, <4 x i32>* %ptr, i64 3
   %idx1 = getelementptr inbounds <4 x i32>, <4 x i32>* %ptr, i64 4
   store <4 x i32> <i32 0, i32 0, i32 0, i32 0>, <4 x i32>* %idx0, align 16
define void @merge_bitcast(<4 x i32> %v, float* %ptr) {
   %idx3 = getelementptr inbounds float, float* %ptr, i64 3
   store float %f0, float* %idx0, align 4
   store float %f1, float* %idx1, align 4
   store float %f2, float* %idx2, align 4
   store float %f3, float* %idx3, align 4
   ret void
 
 ; CHECK-LABEL: merge_bitcast
-; CHECK:      vmovd %xmm0, (%rdi)
-; CHECK-NEXT: vpextrd $1, %xmm0, 4(%rdi)
-; CHECK-NEXT: vpextrd $2, %xmm0, 8(%rdi)
-; CHECK-NEXT: vpextrd $3, %xmm0, 12(%rdi)
+; CHECK:      vmovups %xmm0, (%rdi)
 ; CHECK-NEXT: retq
 }