[X86][DAG] Switch X86 Target to post-legalized store merge

[niravd]

Summary

Move store merge to happen after intrinsic lowering to allow lowered
stores to be merged.

Some regressions due in MergeConsecutiveStores to missing
insert_subvector that are addressed in follow up patch.

[niravd]

Rebase on improved bitcast store

[RKSimon]

test/CodeGen/X86/oddshuffles.ll

86

This is scaring me - what is the reasoning behind permitting the store of a <4 x i32> and not just the <3 x i32> ?

[niravd]

Rebase given corrected D34569 patch which was failing to check candidate store sizes in some merging cases.

[niravd]

test/CodeGen/X86/oddshuffles.ll

86

This is due to a mistake in a parent patch which dropped a type size check. It should be fixed now.

[niravd]

Rebase and general update. No real change.

[RKSimon]

lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp

56

One use and fits on the line, so we can just do:

SDValue Base = DAG.getTargetLoweringInfo().unwrapAddress(Ptr);

test/CodeGen/X86/bitcast-i256.ll

9

Get rid of this - did the script add it?

[niravd]

Address comments.

[RKSimon]

test/CodeGen/X86/bigstructret.ll

1

I've regenerated this test file, so it needs rebasing.

test/CodeGen/X86/extract-store.ll

528 ↗

(On Diff #108915)

Any idea why this still doesn't merge? Same for @extract_f128_1 as well.

[niravd]

Update test.

[niravd]

test/CodeGen/X86/extract-store.ll

528 ↗

(On Diff #108915)

At merge time the component stores have an alignment of 1 and TLI.allowsMisalignedMemoryAccess believe this is legal but slow because Subtarget.isUnalignedMem16Slow() is set so we don't bother merging.

[RKSimon]

LGTM

[RKSimon]

@niravd Anything stopping this going in?

include/llvm/CodeGen/SelectionDAGAddressAnalysis.h

bool equalBaseIndex(BaseIndexOffset &Other, const SelectionDAG &DAG) {
     int64_t Off;
     return equalBaseIndex(Other, DAG, Off);
   }
 
   bool equalBaseIndex(BaseIndexOffset &Other, const SelectionDAG &DAG,
                       int64_t &Off);
 
   /// Parses tree in Ptr for base, index, offset addresses.
-  static BaseIndexOffset match(SDValue Ptr);
+  static BaseIndexOffset match(SDValue Ptr, const SelectionDAG &DAG);
 };
 } // namespace llvm
 
 #endif

include/llvm/Target/TargetLowering.h

public:
   bool isExtendedTrueVal(const ConstantSDNode *N, EVT VT, bool Signed) const;
 
   /// Try to simplify a setcc built with the specified operands and cc. If it is
   /// unable to simplify it, return a null SDValue.
   SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
                         bool foldBooleans, DAGCombinerInfo &DCI,
                         const SDLoc &dl) const;
 
+  // For targets which wrap address, unwrap for analysis.
+  virtual SDValue unwrapAddress(SDValue N) const { return N; }
+
   /// Returns true (and the GlobalValue and the offset) if the node is a
   /// GlobalAddress + offset.
   virtual bool
   isGAPlusOffset(SDNode *N, const GlobalValue* &GA, int64_t &Offset) const;
 
   /// This method will be invoked for all target nodes and for any
   /// target-independent nodes that the target has registered with invoke it
   /// for.

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

for (unsigned i = 0; i < ByteWidth; i++) {
     // All loads must share the same chain
     SDValue LChain = L->getChain();
     if (!Chain)
       Chain = LChain;
     else if (Chain != LChain)
       return SDValue();
 
     // Loads must share the same base address
-    BaseIndexOffset Ptr = BaseIndexOffset::match(L->getBasePtr());
+    BaseIndexOffset Ptr = BaseIndexOffset::match(L->getBasePtr(), DAG);
     int64_t ByteOffsetFromBase = 0;
     if (!Base)
       Base = Ptr;
     else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
       return SDValue();
 
     // Calculate the offset of the current byte from the base address
     ByteOffsetFromBase += MemoryByteOffset(*P);
if (IsConstantSrc) {
       }
       StoredVal = DAG.getBuildVector(Ty, DL, BuildVector);
     } else {
       SmallVector<SDValue, 8> Ops;
       for (unsigned i = 0; i < NumStores; ++i) {
         StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
         SDValue Val = St->getValue();
         // All operands of BUILD_VECTOR / CONCAT_VECTOR must have the same type.
+        while (Val.getValueType() != MemVT && Val->getOpcode() == ISD::BITCAST)
+          Val = Val.getOperand(0);
+        // If we see an extract of a Bitcast, and we could remove both
+        // bitcasts, do so.
+        if (Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+            Val.getOperand(0).getOpcode() == ISD::BITCAST) {
+          SDValue Vec = Val.getOperand(0).getOperand(0);
+          if (Vec.getValueType().getVectorElementType() == MemVT)
+            Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Val), MemVT, Vec,
+                              Val.getOperand(1));
+        }
+        // TODO: add this check for EXTRACT_SUBVECTOR as well.
         if (Val.getValueType() != MemVT)
           return false;
+
         Ops.push_back(Val);
       }
 
       // Build the extracted vector elements back into a vector.
       StoredVal = DAG.getNode(IsVec ? ISD::CONCAT_VECTORS : ISD::BUILD_VECTOR,
                               DL, Ty, Ops);    }
   } else {
     // We should always use a vector store when merging extracted vector
bool DAGCombiner::MergeStoresOfConstantsOrVecElts(
   AddToWorklist(NewChain.getNode());
   return true;
 }
 
 void DAGCombiner::getStoreMergeCandidates(
     StoreSDNode *St, SmallVectorImpl<MemOpLink> &StoreNodes) {
   // This holds the base pointer, index, and the offset in bytes from the base
   // pointer.
-  BaseIndexOffset BasePtr = BaseIndexOffset::match(St->getBasePtr());
+  BaseIndexOffset BasePtr = BaseIndexOffset::match(St->getBasePtr(), DAG);
+
   EVT MemVT = St->getMemoryVT();
 
+  SDValue Val = St->getValue();
+  while (Val.getValueType() != MemVT && Val->getOpcode() == ISD::BITCAST)
+    Val = Val.getOperand(0);
+
   // We must have a base and an offset.
   if (!BasePtr.getBase().getNode())
     return;
 
   // Do not handle stores to undef base pointers.
   if (BasePtr.getBase().isUndef())
     return;
 
   bool IsConstantSrc = isa<ConstantSDNode>(St->getValue()) ||
                        isa<ConstantFPSDNode>(St->getValue());
-  bool IsExtractVecSrc =
-      (St->getValue().getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
-       St->getValue().getOpcode() == ISD::EXTRACT_SUBVECTOR);
+
+  bool IsExtractVecSrc = (Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
+                          Val.getOpcode() == ISD::EXTRACT_SUBVECTOR);
   bool IsLoadSrc = isa<LoadSDNode>(St->getValue());
   BaseIndexOffset LBasePtr;
   // Match on loadbaseptr if relevant.
   if (IsLoadSrc)
-    LBasePtr =
-        BaseIndexOffset::match(cast<LoadSDNode>(St->getValue())->getBasePtr());
+    LBasePtr = BaseIndexOffset::match(
+        cast<LoadSDNode>(St->getValue())->getBasePtr(), DAG);
 
   auto CandidateMatch = [&](StoreSDNode *Other, BaseIndexOffset &Ptr,
                             int64_t &Offset) -> bool {
+    bool EquivType = (Other->getMemoryVT() == MemVT) ||
+                     (MemVT.isInteger() && MemVT.bitsEq(Other->getMemoryVT()));
     if (Other->isVolatile() || Other->isIndexed())
       return false;
-    // We can merge constant floats to equivalent integers
-    if (Other->getMemoryVT() != MemVT)
-      if (!(MemVT.isInteger() && MemVT.bitsEq(Other->getMemoryVT()) &&
-            isa<ConstantFPSDNode>(Other->getValue())))
-        return false;
     if (IsLoadSrc) {
       // The Load's Base Ptr must also match
+      if (!EquivType)
+        return false;
       if (LoadSDNode *OtherLd = dyn_cast<LoadSDNode>(Other->getValue())) {
-        auto LPtr = BaseIndexOffset::match(OtherLd->getBasePtr());
+        auto LPtr = BaseIndexOffset::match(OtherLd->getBasePtr(), DAG);
         if (!(LBasePtr.equalBaseIndex(LPtr, DAG)))
           return false;
       } else
         return false;
     }
-    if (IsConstantSrc)
-      if (!(isa<ConstantSDNode>(Other->getValue()) ||
+    if (IsConstantSrc) {
+      // May truncate things converted to integers
+      if (!EquivType || !(isa<ConstantSDNode>(Other->getValue()) ||
                           isa<ConstantFPSDNode>(Other->getValue())))
         return false;
-    if (IsExtractVecSrc)
-      if (!(Other->getValue().getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
-            Other->getValue().getOpcode() == ISD::EXTRACT_SUBVECTOR))
+    }
+    if (IsExtractVecSrc) {
+      SDValue Val = Other->getValue();
+      // Peel off bitcasts.
+      bool MatchVT = false;
+      while (Val.getOpcode() == ISD::BITCAST) {
+        if (Val.getValueType() == MemVT)
+          MatchVT = true;
+        Val = Val.getOperand(0);
+      }
+      if (!(Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
+            Val.getOpcode() == ISD::EXTRACT_SUBVECTOR))
         return false;
-    Ptr = BaseIndexOffset::match(Other->getBasePtr());
+    }
+
+    Ptr = BaseIndexOffset::match(Other->getBasePtr(), DAG);
     return (BasePtr.equalBaseIndex(Ptr, DAG, Offset));
   };
   // We looking for a root node which is an ancestor to all mergable
   // stores. We search up through a load, to our root and then down
   // through all children. For instance we will find Store{1,2,3} if
   // St is Store1, Store2. or Store3 where the root is not a load
   // which always true for nonvolatile ops. TODO: Expand
   // the search to find all valid candidates through multiple layers of loads.
while (StoreNodes.size() > 1) {
     if (IsExtractVecSrc) {
       LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
       unsigned FirstStoreAS = FirstInChain->getAddressSpace();
       unsigned FirstStoreAlign = FirstInChain->getAlignment();
       unsigned NumStoresToMerge = 1;
       bool IsVec = MemVT.isVector();
       for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
         StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
-        unsigned StoreValOpcode = St->getValue().getOpcode();
+        SDValue StVal = St->getValue();
+        // peek through bitcasts
+        while (StVal->getOpcode() == ISD::BITCAST)
+          StVal = StVal.getOperand(0);
+
+        unsigned StoreValOpcode = StVal.getOpcode();
+
         // This restriction could be loosened.
         // Bail out if any stored values are not elements extracted from a
         // vector. It should be possible to handle mixed sources, but load
         // sources need more careful handling (see the block of code below that
         // handles consecutive loads).
         if (StoreValOpcode != ISD::EXTRACT_VECTOR_ELT &&
             StoreValOpcode != ISD::EXTRACT_SUBVECTOR)
           return RV;
for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
       // We do not accept ext loads.
       if (Ld->getExtensionType() != ISD::NON_EXTLOAD)
         break;
 
       // The stored memory type must be the same.
       if (Ld->getMemoryVT() != MemVT)
         break;
 
-      BaseIndexOffset LdPtr = BaseIndexOffset::match(Ld->getBasePtr());
+      BaseIndexOffset LdPtr = BaseIndexOffset::match(Ld->getBasePtr(), DAG);
       // If this is not the first ptr that we check.
       int64_t LdOffset = 0;
       if (LdBasePtr.getBase().getNode()) {
         // The base ptr must be the same.
         if (!LdBasePtr.equalBaseIndex(LdPtr, DAG, LdOffset))
           break;
       } else {
         // Check that all other base pointers are the same as this one.
SDValue DAGCombiner::visitINSERT_SUBVECTOR(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   SDValue N2 = N->getOperand(2);
 
   // If inserting an UNDEF, just return the original vector.
   if (N1.isUndef())
     return N0;
 
+  // If we N0 is also an insert_subvector and we've the only use fix it first.
+  if (N0.hasOneUse() && N0->getOpcode() == ISD::INSERT_SUBVECTOR)
+    if (SDValue NN0 = visitINSERT_SUBVECTOR(N0.getNode()))
+      return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, NN0, N1, N2);
+
   // If this is an insert of an extracted vector into an undef vector, we can
   // just use the input to the extract.
   if (N0.isUndef() && N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
       N1.getOperand(1) == N2 && N1.getOperand(0).getValueType() == VT)
     return N1.getOperand(0);
 
+  // If inserting a bitcast values into an undef vector, w/ same num
+  // elements, we can just use the bitcast input of the extract
+  if (N0.isUndef() && N1.getOpcode() == ISD::BITCAST &&
+      N1.getOperand(0).getOpcode() == ISD::EXTRACT_SUBVECTOR &&
+      N1.getOperand(0).getOperand(1) == N2 &&
+      N1.getOperand(0).getOperand(0).getValueType().getVectorNumElements() ==
+          VT.getVectorNumElements()) {
+    return DAG.getBitcast(VT, N1.getOperand(0).getOperand(0));
+  }
+
+  // If Both N1 and N2 are bit cast values on which insert_subvector
+  // makes sense, pull bitcast through the insert.
+  if (N0.getOpcode() == ISD::BITCAST && N1.getOpcode() == ISD::BITCAST) {
+    SDValue CN0 = N0.getOperand(0);
+    SDValue CN1 = N1.getOperand(0);
+    if (CN0.getValueType().getVectorElementType() ==
+            CN1.getValueType().getVectorElementType() &&
+        CN0.getValueType().getVectorNumElements() ==
+            VT.getVectorNumElements()) {
+      SDValue NewINSERT = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N),
+                                      CN0.getValueType(), CN0, CN1, N2);
+      return DAG.getBitcast(VT, NewINSERT);
+    }
+  }
+
   // Combine INSERT_SUBVECTORs where we are inserting to the same index.
   // INSERT_SUBVECTOR( INSERT_SUBVECTOR( Vec, SubOld, Idx ), SubNew, Idx )
   // --> INSERT_SUBVECTOR( Vec, SubNew, Idx )
   if (N0.getOpcode() == ISD::INSERT_SUBVECTOR &&
       N0.getOperand(1).getValueType() == N1.getValueType() &&
       N0.getOperand(2) == N2)
     return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0.getOperand(0),
                        N1, N2);
bool DAGCombiner::isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const {
 
   if (Op1->isInvariant() && Op0->writeMem())
     return false;
 
   unsigned NumBytes0 = Op0->getMemoryVT().getSizeInBits() >> 3;
   unsigned NumBytes1 = Op1->getMemoryVT().getSizeInBits() >> 3;
 
   // Check for BaseIndexOffset matching.
-  BaseIndexOffset BasePtr0 = BaseIndexOffset::match(Op0->getBasePtr());
-  BaseIndexOffset BasePtr1 = BaseIndexOffset::match(Op1->getBasePtr());
+  BaseIndexOffset BasePtr0 = BaseIndexOffset::match(Op0->getBasePtr(), DAG);
+  BaseIndexOffset BasePtr1 = BaseIndexOffset::match(Op1->getBasePtr(), DAG);
   int64_t PtrDiff;
   if (BasePtr0.equalBaseIndex(BasePtr1, DAG, PtrDiff))
     return !((NumBytes0 <= PtrDiff) || (PtrDiff + NumBytes1 <= 0));
 
   // FIXME: findBaseOffset and ConstantValue/GlobalValue/FrameIndex analysis
   // modified to use BaseIndexOffset.
 
   // Gather base node and offset information.

 // we improve the chains of all the potential candidates up-front
 // before running MergeConsecutiveStores, it might only see some of
 // the nodes that will eventually be candidates, and then not be able
 // to go from a partially-merged state to the desired final
 // fully-merged state.
 bool DAGCombiner::findBetterNeighborChains(StoreSDNode *St) {
   // This holds the base pointer, index, and the offset in bytes from the base
   // pointer.
-  BaseIndexOffset BasePtr = BaseIndexOffset::match(St->getBasePtr());
+  BaseIndexOffset BasePtr = BaseIndexOffset::match(St->getBasePtr(), DAG);
 
   // We must have a base and an offset.
   if (!BasePtr.getBase().getNode())
     return false;
 
   // Do not handle stores to undef base pointers.
   if (BasePtr.getBase().isUndef())
     return false;
while (Index) {
     // If the chain has more than one use, then we can't reorder the mem ops.
     if (Index != St && !SDValue(Index, 0)->hasOneUse())
       break;
 
     if (Index->isVolatile() || Index->isIndexed())
       break;
 
     // Find the base pointer and offset for this memory node.
-    BaseIndexOffset Ptr = BaseIndexOffset::match(Index->getBasePtr());
+    BaseIndexOffset Ptr = BaseIndexOffset::match(Index->getBasePtr(), DAG);
 
     // Check that the base pointer is the same as the original one.
     if (!BasePtr.equalBaseIndex(Ptr, DAG))
       break;
 
     // Walk up the chain to find the next store node, ignoring any
     // intermediate loads. Any other kind of node will halt the loop.
     SDNode *NextInChain = Index->getChain().getNode();

lib/CodeGen/SelectionDAG/SelectionDAG.cpp

 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
+#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/SelectionDAGTargetInfo.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DebugLoc.h"
bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
   if (LD->getChain() != Base->getChain())
     return false;
   EVT VT = LD->getValueType(0);
   if (VT.getSizeInBits() / 8 != Bytes)
     return false;
 
   SDValue Loc = LD->getOperand(1);
   SDValue BaseLoc = Base->getOperand(1);
-  if (Loc.getOpcode() == ISD::FrameIndex) {
-    if (BaseLoc.getOpcode() != ISD::FrameIndex)
-      return false;
-    const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
-    int FI  = cast<FrameIndexSDNode>(Loc)->getIndex();
-    int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex();
-    int FS  = MFI.getObjectSize(FI);
-    int BFS = MFI.getObjectSize(BFI);
-    if (FS != BFS || FS != (int)Bytes) return false;
-    return MFI.getObjectOffset(FI) == (MFI.getObjectOffset(BFI) + Dist*Bytes);
-  }
 
-  // Handle X + C.
-  if (isBaseWithConstantOffset(Loc)) {
-    int64_t LocOffset = cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue();
-    if (Loc.getOperand(0) == BaseLoc) {
-      // If the base location is a simple address with no offset itself, then
-      // the second load's first add operand should be the base address.
-      if (LocOffset == Dist * (int)Bytes)
-        return true;
-    } else if (isBaseWithConstantOffset(BaseLoc)) {
-      // The base location itself has an offset, so subtract that value from the
-      // second load's offset before comparing to distance * size.
-      int64_t BOffset =
-        cast<ConstantSDNode>(BaseLoc.getOperand(1))->getSExtValue();
-      if (Loc.getOperand(0) == BaseLoc.getOperand(0)) {
-        if ((LocOffset - BOffset) == Dist * (int)Bytes)
-          return true;
-      }
-    }
-  }
-  const GlobalValue *GV1 = nullptr;
-  const GlobalValue *GV2 = nullptr;
-  int64_t Offset1 = 0;
-  int64_t Offset2 = 0;
-  bool isGA1 = TLI->isGAPlusOffset(Loc.getNode(), GV1, Offset1);
-  bool isGA2 = TLI->isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2);
-  if (isGA1 && isGA2 && GV1 == GV2)
-    return Offset1 == (Offset2 + Dist*Bytes);
+  auto BaseLocDecomp = BaseIndexOffset::match(BaseLoc, *this);
+  auto LocDecomp = BaseIndexOffset::match(Loc, *this);
+
+  int64_t Offset = 0;
+  if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))
+    return (Dist * Bytes == Offset);
   return false;
 }
 
 /// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if
 /// it cannot be inferred.
 unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
   // If this is a GlobalAddress + cst, return the alignment.
   const GlobalValue *GV;

lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp

 //===-- llvm/CodeGen/SelectionDAGAddressAnalysis.cpp ------- DAG Address
 //Analysis ---*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 
 #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
+#include "llvm/Target/TargetLowering.h"
 
 namespace llvm {
 
 bool BaseIndexOffset::equalBaseIndex(BaseIndexOffset &Other,
                                      const SelectionDAG &DAG, int64_t &Off) {
   // Obvious equivalent
   Off = Other.Offset - Offset;
   if (Other.Base == Base && Other.Index == Index &&
       Other.IsIndexSignExt == IsIndexSignExt)
     return true;
 
   // Match GlobalAddresses
   if (Index == Other.Index)
-    if (GlobalAddressSDNode *A = dyn_cast<GlobalAddressSDNode>(Base))
-      if (GlobalAddressSDNode *B = dyn_cast<GlobalAddressSDNode>(Other.Base))
+    if (auto *A = dyn_cast<GlobalAddressSDNode>(Base))
+      if (auto *B = dyn_cast<GlobalAddressSDNode>(Other.Base))
         if (A->getGlobal() == B->getGlobal()) {
           Off += B->getOffset() - A->getOffset();
           return true;
         }
 
-  // TODO: we should be able to add FrameIndex analysis improvements here.
+  // Match FrameIndexes
+  if (Index == Other.Index)
+    if (auto *A = dyn_cast<FrameIndexSDNode>(Base))
+      if (auto *B = dyn_cast<FrameIndexSDNode>(Other.Base)) {
+        const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
+        Off += MFI.getObjectOffset(B->getIndex()) -
+               MFI.getObjectOffset(A->getIndex());
+        return true;
+      }
 
   return false;
 }
 
 /// Parses tree in Ptr for base, index, offset addresses.
-BaseIndexOffset BaseIndexOffset::match(SDValue Ptr) {
+BaseIndexOffset BaseIndexOffset::match(SDValue Ptr, const SelectionDAG &DAG) {
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
   // (((B + I*M) + c)) + c ...
-  SDValue Base = Ptr;
+  SDValue Base = TLI.unwrapAddress(Ptr);
   SDValue Index = SDValue();
   int64_t Offset = 0;
   bool IsIndexSignExt = false;
 
-  // Consume constant adds
-  while (Base->getOpcode() == ISD::ADD &&
-         isa<ConstantSDNode>(Base->getOperand(1))) {
-    int64_t POffset = cast<ConstantSDNode>(Base->getOperand(1))->getSExtValue();
-    Offset += POffset;
+  // Consume constant adds & ors with appropriate masking.
+  while (Base->getOpcode() == ISD::ADD || Base->getOpcode() == ISD::OR) {
+    if (auto *C = dyn_cast<ConstantSDNode>(Base->getOperand(1))) {
+      if (Base->getOpcode() == ISD::ADD ||
+          (Base->getOpcode() == ISD::OR &&
+           DAG.MaskedValueIsZero(Base->getOperand(0), C->getAPIntValue()))) {
+        Offset += C->getSExtValue();
         Base = Base->getOperand(0);
+        continue;
+      }
+    }
+    if (auto *C = dyn_cast<ConstantSDNode>(Base->getOperand(0))) {
+      if (Base->getOpcode() == ISD::ADD ||
+          (Base->getOpcode() == ISD::OR &&
+           DAG.MaskedValueIsZero(Base->getOperand(1), C->getAPIntValue()))) {
+        Offset += C->getSExtValue();
+        Base = Base->getOperand(1);
+        continue;
+      }
+    }
+    break;
   }
 
   if (Base->getOpcode() == ISD::ADD) {
     // TODO: The following code appears to be needless as it just
     //       bails on some Ptrs early, reducing the cases where we
     //       find equivalence. We should be able to remove this.
     // Inside a loop the current BASE pointer is calculated using an ADD and a
     // MUL instruction. In this case Base is the actual BASE pointer.

lib/Target/X86/X86ISelLowering.h

public:
 
     MachineBasicBlock *
     EmitInstrWithCustomInserter(MachineInstr &MI,
                                 MachineBasicBlock *MBB) const override;
 
     /// This method returns the name of a target specific DAG node.
     const char *getTargetNodeName(unsigned Opcode) const override;
 
+    bool mergeStoresAfterLegalization() const override { return true; }
+
     bool isCheapToSpeculateCttz() const override;
 
     bool isCheapToSpeculateCtlz() const override;
 
     bool isCtlzFast() const override;
 
     bool hasBitPreservingFPLogic(EVT VT) const override {
       return VT == MVT::f32 || VT == MVT::f64 || VT.isVector();
void computeKnownBitsForTargetNode(const SDValue Op,
                                        unsigned Depth = 0) const override;
 
     /// Determine the number of bits in the operation that are sign bits.
     unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
                                              const APInt &DemandedElts,
                                              const SelectionDAG &DAG,
                                              unsigned Depth) const override;
 
+    SDValue unwrapAddress(SDValue N) const override;
+
     bool isGAPlusOffset(SDNode *N, const GlobalValue* &GA,
                         int64_t &Offset) const override;
 
     SDValue getReturnAddressFrameIndex(SelectionDAG &DAG) const;
 
     bool ExpandInlineAsm(CallInst *CI) const override;
 
     ConstraintType getConstraintType(StringRef Constraint) const override;

lib/Target/X86/X86ISelLowering.cpp

case X86ISD::VPCOMU:
     // Vector compares return zero/all-bits result values.
     return VTBits;
   }
 
   // Fallback case.
   return 1;
 }
 
+SDValue X86TargetLowering::unwrapAddress(SDValue N) const {
+  if (N->getOpcode() == X86ISD::Wrapper || N->getOpcode() == X86ISD::WrapperRIP)
+    return N->getOperand(0);
+  return N;
+}
+
 /// Returns true (and the GlobalValue and the offset) if the node is a
 /// GlobalAddress + offset.
 bool X86TargetLowering::isGAPlusOffset(SDNode *N,
                                        const GlobalValue* &GA,
                                        int64_t &Offset) const {
   if (N->getOpcode() == X86ISD::Wrapper) {
     if (isa<GlobalAddressSDNode>(N->getOperand(0))) {
       GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal();

test/CodeGen/BPF/undef.ll

 ; RUN: not llc < %s -march=bpf | FileCheck %s
 
 %struct.bpf_map_def = type { i32, i32, i32, i32 }
 %struct.__sk_buff = type opaque
 %struct.routing_key_2 = type { [6 x i8] }
 
 @routing = global %struct.bpf_map_def { i32 1, i32 6, i32 12, i32 1024 }, section "maps", align 4
 @routing_miss_0 = global %struct.bpf_map_def { i32 1, i32 1, i32 12, i32 1 }, section "maps", align 4
 @test1 = global %struct.bpf_map_def { i32 2, i32 4, i32 8, i32 1024 }, section "maps", align 4
 @test1_miss_4 = global %struct.bpf_map_def { i32 2, i32 1, i32 8, i32 1 }, section "maps", align 4
 @_license = global [4 x i8] c"GPL\00", section "license", align 1
 @llvm.used = appending global [6 x i8*] [i8* getelementptr inbounds ([4 x i8], [4 x i8]* @_license, i32 0, i32 0), i8* bitcast (i32 (%struct.__sk_buff*)* @ebpf_filter to i8*), i8* bitcast (%struct.bpf_map_def* @routing to i8*), i8* bitcast (%struct.bpf_map_def* @routing_miss_0 to i8*), i8* bitcast (%struct.bpf_map_def* @test1 to i8*), i8* bitcast (%struct.bpf_map_def* @test1_miss_4 to i8*)], section "llvm.metadata"
 
 ; Function Attrs: nounwind uwtable
 define i32 @ebpf_filter(%struct.__sk_buff* nocapture readnone %ebpf_packet) #0 section "socket1" {
-; CHECK: r2 = r10
-; CHECK: r2 += -2
-; CHECK: r1 = 0
-; CHECK: *(u16 *)(r2 + 6) = r1
-; CHECK: *(u16 *)(r2 + 4) = r1
-; CHECK: *(u16 *)(r2 + 2) = r1
-; CHECK: r2 = 6
-; CHECK: *(u8 *)(r10 - 7) = r2
-; CHECK: r2 = 5
-; CHECK: *(u8 *)(r10 - 8) = r2
-; CHECK: r2 = 7
-; CHECK: *(u8 *)(r10 - 6) = r2
-; CHECK: r2 = 8
-; CHECK: *(u8 *)(r10 - 5) = r2
-; CHECK: r2 = 9
-; CHECK: *(u8 *)(r10 - 4) = r2
-; CHECK: r2 = 10
-; CHECK: *(u8 *)(r10 - 3) = r2
-; CHECK: *(u16 *)(r10 + 24) = r1
-; CHECK: *(u16 *)(r10 + 22) = r1
-; CHECK: *(u16 *)(r10 + 20) = r1
-; CHECK: *(u16 *)(r10 + 18) = r1
-; CHECK: *(u16 *)(r10 + 16) = r1
-; CHECK: *(u16 *)(r10 + 14) = r1
-; CHECK: *(u16 *)(r10 + 12) = r1
-; CHECK: *(u16 *)(r10 + 10) = r1
-; CHECK: *(u16 *)(r10 + 8) = r1
-; CHECK: *(u16 *)(r10 + 6) = r1
-; CHECK: *(u16 *)(r10 - 2) = r1
-; CHECK: *(u16 *)(r10 + 26) = r1
+; CHECK: r1 = r10
+; CHECK: r1 += -2
+; CHECK: r2 = 0
+; CHECK: *(u16 *)(r1 + 6) = r2
+; CHECK: *(u16 *)(r1 + 4) = r2
+; CHECK: *(u16 *)(r1 + 2) = r2
+; CHECK: r1 = 134678021
+; CHECK: *(u32 *)(r10 - 8) = r1
+; CHECK: r1 = 9
+; CHECK: *(u8 *)(r10 - 4) = r1
+; CHECK: r1 = 10
+; CHECK: *(u8 *)(r10 - 3) = r1
+; CHECK: *(u16 *)(r10 + 24) = r2
+; CHECK: *(u16 *)(r10 + 22) = r2
+; CHECK: *(u16 *)(r10 + 20) = r2
+; CHECK: *(u16 *)(r10 + 18) = r2
+; CHECK: *(u16 *)(r10 + 16) = r2
+; CHECK: *(u16 *)(r10 + 14) = r2
+; CHECK: *(u16 *)(r10 + 12) = r2
+; CHECK: *(u16 *)(r10 + 10) = r2
+; CHECK: *(u16 *)(r10 + 8) = r2
+; CHECK: *(u16 *)(r10 + 6) = r2
+; CHECK: *(u16 *)(r10 - 2) = r2
+; CHECK: *(u16 *)(r10 + 26) = r2
 ; CHECK: r2 = r10
 ; CHECK: r2 += -8
 ; CHECK: r1 = <MCOperand Expr:(routing)>ll
 ; CHECK: call bpf_map_lookup_elem
 ; CHECK: exit
   %key = alloca %struct.routing_key_2, align 1
   %1 = getelementptr inbounds %struct.routing_key_2, %struct.routing_key_2* %key, i64 0, i32 0, i64 0
   store i8 5, i8* %1, align 1

test/CodeGen/MSP430/Inst16mm.ll

  %y = alloca i32, align 2                        ; <i32*> [#uses=1]
  store i16 0, i16* %retval
  %tmp = load i32, i32* %y                             ; <i32> [#uses=1]
  store i32 %tmp, i32* %x
  store i16 0, i16* %retval
  %0 = load i16, i16* %retval                          ; <i16> [#uses=1]
  ret i16 %0
 ; CHECK-LABEL: mov2:
-; CHECK:  mov.w 2(r1), 6(r1)
-; CHECK:  mov.w 0(r1), 4(r1)
+; CHECK-DAG:  mov.w 2(r1), 6(r1)
+; CHECK-DAG:  mov.w 0(r1), 4(r1)
 }

test/CodeGen/PowerPC/complex-return.ll

 ; RUN: llc -verify-machineinstrs -mcpu=ppc64 -O0 < %s | FileCheck %s
 
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
 define { ppc_fp128, ppc_fp128 } @foo() nounwind {
 entry:
   %retval = alloca { ppc_fp128, ppc_fp128 }, align 16
   %x = alloca { ppc_fp128, ppc_fp128 }, align 16
   %real = getelementptr inbounds { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %x, i32 0, i32 0
   %imag = getelementptr inbounds { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %x, i32 0, i32 1
-  store ppc_fp128 0xM400C0000000000000000000000000000, ppc_fp128* %real
+  store ppc_fp128 0xM400C0000000000300000000010000000, ppc_fp128* %real
   store ppc_fp128 0xMC00547AE147AE1483CA47AE147AE147A, ppc_fp128* %imag
   %x.realp = getelementptr inbounds { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %x, i32 0, i32 0
   %x.real = load ppc_fp128, ppc_fp128* %x.realp
   %x.imagp = getelementptr inbounds { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %x, i32 0, i32 1
   %x.imag = load ppc_fp128, ppc_fp128* %x.imagp
   %real1 = getelementptr inbounds { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %retval, i32 0, i32 0
   %imag2 = getelementptr inbounds { ppc_fp128, ppc_fp128 }, { ppc_fp128, ppc_fp128 }* %retval, i32 0, i32 1
   store ppc_fp128 %x.real, ppc_fp128* %real1

test/CodeGen/X86/MergeConsecutiveStores.ll

 ; CHECK-LABEL: merge_vec_stores_of_constants
 ; CHECK:      vxorps
 ; CHECK-NEXT: vmovaps
 ; CHECK-NEXT: vmovaps
 ; CHECK-NEXT: retq
 }
 
 ; This is a minimized test based on real code that was failing.
-; We could merge stores (and loads) like this...
-
+; This should now be merged.
 define void @merge_vec_element_and_scalar_load([6 x i64]* %array) {
   %idx0 = getelementptr inbounds [6 x i64], [6 x i64]* %array, i64 0, i64 0
   %idx1 = getelementptr inbounds [6 x i64], [6 x i64]* %array, i64 0, i64 1
   %idx4 = getelementptr inbounds [6 x i64], [6 x i64]* %array, i64 0, i64 4
   %idx5 = getelementptr inbounds [6 x i64], [6 x i64]* %array, i64 0, i64 5
 
   %a0 = load i64, i64* %idx0, align 8
   store i64 %a0, i64* %idx4, align 8
 
   %b = bitcast i64* %idx1 to <2 x i64>*
   %v = load <2 x i64>, <2 x i64>* %b, align 8
   %a1 = extractelement <2 x i64> %v, i32 0
   store i64 %a1, i64* %idx5, align 8
   ret void
 
 ; CHECK-LABEL: merge_vec_element_and_scalar_load
-; CHECK:      movq  (%rdi), %rax
-; CHECK-NEXT: movq  8(%rdi), %rcx
-; CHECK-NEXT: movq  %rax, 32(%rdi)
-; CHECK-NEXT: movq  %rcx, 40(%rdi)
+; CHECK:      vmovups (%rdi), %xmm0
+; CHECK-NEXT: vmovups %xmm0, 32(%rdi)
 ; CHECK-NEXT: retq
 }
 
-
-
 ; Don't let a non-consecutive store thwart merging of the last two.
 define void @almost_consecutive_stores(i8* %p) {
   store i8 0, i8* %p
   %p1 = getelementptr i8, i8* %p, i64 42
   store i8 1, i8* %p1
   %p2 = getelementptr i8, i8* %p, i64 2
   store i8 2, i8* %p2
   %p3 = getelementptr i8, i8* %p, i64 3
   store i8 3, i8* %p3
   ret void
 ; CHECK-LABEL: almost_consecutive_stores
 ; CHECK-DAG: movb $0, (%rdi)
 ; CHECK-DAG: movb $1, 42(%rdi)
 ; CHECK-DAG: movw $770, 2(%rdi)
 ; CHECK: retq
 }
+
+; We should be able to merge these.
+define void @merge_bitcast(<4 x i32> %v, float* %ptr) {
+  %fv = bitcast <4 x i32> %v to <4 x float>
+
+  %vecext1 = extractelement <4 x i32> %v, i32 1
+  %vecext2 = extractelement <4 x i32> %v, i32 2
+  %vecext3 = extractelement <4 x i32> %v, i32 3
+  %f0 = extractelement <4 x float> %fv, i32 0
+  %f1 = bitcast i32 %vecext1 to float
+  %f2 = bitcast i32 %vecext2 to float
+  %f3 = bitcast i32 %vecext3 to float
+  %idx0 = getelementptr inbounds float, float* %ptr, i64 0
+  %idx1 = getelementptr inbounds float, float* %ptr, i64 1
+  %idx2 = getelementptr inbounds float, float* %ptr, i64 2
+  %idx3 = getelementptr inbounds float, float* %ptr, i64 3
+  store float %f0, float* %idx0, align 4
+  store float %f1, float* %idx1, align 4
+  store float %f2, float* %idx2, align 4
+  store float %f3, float* %idx3, align 4
+  ret void
+
+; CHECK-LABEL: merge_bitcast
+; CHECK:      vmovups %xmm0, (%rdi)
+; CHECK-NEXT: retq
+}

test/CodeGen/X86/bigstructret.ll

 ; RUN: llc < %s -march=x86 | FileCheck %s
 
 %0 = type { i32, i32, i32, i32 }
 %1 = type { i1, i1, i1, i32 }
 
 ; CHECK: ReturnBigStruct
 ; CHECK: movl $24601, 12(%ecx)
 ; CHECK: movl $48, 8(%ecx)
 ; CHECK: movl $24, 4(%ecx)
 ; CHECK: movl $12, (%ecx)
 
 define fastcc %0 @ReturnBigStruct() nounwind readnone {
 entry:
   %0 = insertvalue %0 zeroinitializer, i32 12, 0
   %1 = insertvalue %0 %0, i32 24, 1
   %2 = insertvalue %0 %1, i32 48, 2
   %3 = insertvalue %0 %2, i32 24601, 3
   ret %0 %3
 }
 
 ; CHECK: ReturnBigStruct2
-; CHECK: movl $48, 4(%ecx)
-; CHECK: movb $1, 2(%ecx)
-; CHECK: movb $1, 1(%ecx)
-; CHECK: movb $0, (%ecx)
+; CHECK-DAG: movl $48, 4(%ecx)
+; CHECK-DAG: movb $1, 2(%ecx)
+; CHECK-DAG: movw   $256, (%ecx)
 
 define fastcc %1 @ReturnBigStruct2() nounwind readnone {
 entry:
   %0 = insertvalue %1 zeroinitializer, i1 false, 0
   %1 = insertvalue %1 %0, i1 true, 1
   %2 = insertvalue %1 %1, i1 true, 2
   %3 = insertvalue %1 %2, i32 48, 3
   ret %1 %3
 }

test/CodeGen/X86/bitcast-i256.ll

 ; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=core-avx-i < %s | FileCheck %s
 
 define i256 @foo(<8 x i32> %a) {
   %r = bitcast <8 x i32> %a to i256
   ret i256 %r
 ; CHECK: foo
 ; CHECK: vextractf128
-; CHECK: vpextrq
-; CHECK: vpextrq
+; CHECK: vmovups
 ; CHECK: ret
 }

test/CodeGen/X86/build-vector-128.ll

; AVX-64-NEXT:    retq
   %ins0 = insertelement <4 x float> undef, float %a0, i32 0
   %ins1 = insertelement <4 x float> %ins0, float %a1, i32 1
   %ins2 = insertelement <4 x float> %ins1, float %a2, i32 2
   %ins3 = insertelement <4 x float> %ins2, float %a3, i32 3
   ret <4 x float> %ins3
 }
 
 define <2 x i64> @test_buildvector_v2i64(i64 %a0, i64 %a1) {
-; SSE2-32-LABEL: test_buildvector_v2i64:
-; SSE2-32:       # BB#0:
-; SSE2-32-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; SSE2-32-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE2-32-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-32-NEXT:    retl
+; SSE-32-LABEL: test_buildvector_v2i64:
+; SSE-32:       # BB#0:
+; SSE-32-NEXT:    movups {{[0-9]+}}(%esp), %xmm0
+; SSE-32-NEXT:    retl
 ;
 ; SSE-64-LABEL: test_buildvector_v2i64:
 ; SSE-64:       # BB#0:
 ; SSE-64-NEXT:    movq %rsi, %xmm1
 ; SSE-64-NEXT:    movq %rdi, %xmm0
 ; SSE-64-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; SSE-64-NEXT:    retq
 ;
-; SSE41-32-LABEL: test_buildvector_v2i64:
-; SSE41-32:       # BB#0:
-; SSE41-32-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE41-32-NEXT:    pinsrd $1, {{[0-9]+}}(%esp), %xmm0
-; SSE41-32-NEXT:    pinsrd $2, {{[0-9]+}}(%esp), %xmm0
-; SSE41-32-NEXT:    pinsrd $3, {{[0-9]+}}(%esp), %xmm0
-; SSE41-32-NEXT:    retl
-;
 ; AVX-32-LABEL: test_buildvector_v2i64:
 ; AVX-32:       # BB#0:
-; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; AVX-32-NEXT:    vmovups {{[0-9]+}}(%esp), %xmm0
 ; AVX-32-NEXT:    retl
 ;
 ; AVX-64-LABEL: test_buildvector_v2i64:
 ; AVX-64:       # BB#0:
 ; AVX-64-NEXT:    vmovq %rsi, %xmm0
 ; AVX-64-NEXT:    vmovq %rdi, %xmm1
 ; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; AVX-64-NEXT:    retq

test/CodeGen/X86/build-vector-256.ll

; AVX-64-NEXT:    retq
   %ins4 = insertelement <8 x float> %ins3, float %a4, i32 4
   %ins5 = insertelement <8 x float> %ins4, float %a5, i32 5
   %ins6 = insertelement <8 x float> %ins5, float %a6, i32 6
   %ins7 = insertelement <8 x float> %ins6, float %a7, i32 7
   ret <8 x float> %ins7
 }
 
 define <4 x i64> @test_buildvector_v4i64(i64 %a0, i64 %a1, i64 %a2, i64 %a3) {
-; AVX1-32-LABEL: test_buildvector_v4i64:
-; AVX1-32:       # BB#0:
-; AVX1-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX1-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX1-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX1-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX1-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX1-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX1-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX1-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX1-32-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-32-NEXT:    retl
+; AVX-32-LABEL: test_buildvector_v4i64:
+; AVX-32:       # BB#0:
+; AVX-32-NEXT:    vmovups {{[0-9]+}}(%esp), %ymm0
+; AVX-32-NEXT:    retl
 ;
 ; AVX1-64-LABEL: test_buildvector_v4i64:
 ; AVX1-64:       # BB#0:
 ; AVX1-64-NEXT:    vmovq %rcx, %xmm0
 ; AVX1-64-NEXT:    vmovq %rdx, %xmm1
 ; AVX1-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; AVX1-64-NEXT:    vmovq %rsi, %xmm1
 ; AVX1-64-NEXT:    vmovq %rdi, %xmm2
 ; AVX1-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; AVX1-64-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-64-NEXT:    retq
 ;
-; AVX2-32-LABEL: test_buildvector_v4i64:
-; AVX2-32:       # BB#0:
-; AVX2-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX2-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX2-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX2-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX2-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX2-32-NEXT:    retl
-;
 ; AVX2-64-LABEL: test_buildvector_v4i64:
 ; AVX2-64:       # BB#0:
 ; AVX2-64-NEXT:    vmovq %rcx, %xmm0
 ; AVX2-64-NEXT:    vmovq %rdx, %xmm1
 ; AVX2-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; AVX2-64-NEXT:    vmovq %rsi, %xmm1
 ; AVX2-64-NEXT:    vmovq %rdi, %xmm2
 ; AVX2-64-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]

test/CodeGen/X86/build-vector-512.ll

; AVX-64-NEXT:    retq
   %ins14 = insertelement <16 x float> %ins13, float %a14, i32 14
   %ins15 = insertelement <16 x float> %ins14, float %a15, i32 15
   ret <16 x float> %ins15
 }
 
 define <8 x i64> @test_buildvector_v8i64(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7) {
 ; AVX-32-LABEL: test_buildvector_v8i64:
 ; AVX-32:       # BB#0:
-; AVX-32-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm0, %xmm0
-; AVX-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX-32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; AVX-32-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX-32-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT:    vpinsrd $2, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT:    vpinsrd $3, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; AVX-32-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX-32-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX-32-NEXT:    vmovups {{[0-9]+}}(%esp), %zmm0
 ; AVX-32-NEXT:    retl
 ;
 ; AVX-64-LABEL: test_buildvector_v8i64:
 ; AVX-64:       # BB#0:
 ; AVX-64-NEXT:    vmovq %rcx, %xmm0
 ; AVX-64-NEXT:    vmovq %rdx, %xmm1
 ; AVX-64-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; AVX-64-NEXT:    vmovq %rsi, %xmm1

test/CodeGen/X86/clear_upper_vector_element_bits.ll

 ; SSE-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
 ; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; SSE-NEXT:    popq %rbx
 ; SSE-NEXT:    popq %r14
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: _clearupper32xi8b:
 ; AVX1:       # BB#0:
+; AVX1-NEXT:    pushq %rbp
+; AVX1-NEXT:    pushq %r15
 ; AVX1-NEXT:    pushq %r14
+; AVX1-NEXT:    pushq %r13
+; AVX1-NEXT:    pushq %r12
 ; AVX1-NEXT:    pushq %rbx
-; AVX1-NEXT:    vpextrq $1, %xmm0, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    vmovq %xmm0, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    movq -{{[0-9]+}}(%rsp), %r14
+; AVX1-NEXT:    vmovq %xmm0, %rcx
+; AVX1-NEXT:    movq %rcx, %r8
+; AVX1-NEXT:    movq %rcx, %r9
+; AVX1-NEXT:    movq %rcx, %r10
+; AVX1-NEXT:    movq %rcx, %r11
+; AVX1-NEXT:    movq %rcx, %r14
+; AVX1-NEXT:    movq %rcx, %r15
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rdx
-; AVX1-NEXT:    movq %rdx, %r8
-; AVX1-NEXT:    movq %rdx, %r9
-; AVX1-NEXT:    movq %rdx, %r11
-; AVX1-NEXT:    movq %rdx, %rsi
-; AVX1-NEXT:    movq %rdx, %rdi
-; AVX1-NEXT:    movq %rdx, %rcx
+; AVX1-NEXT:    movq %rdx, %r12
+; AVX1-NEXT:    movq %rdx, %r13
+; AVX1-NEXT:    movq %rdx, %rbx
 ; AVX1-NEXT:    movq %rdx, %rax
+; AVX1-NEXT:    movq %rdx, %rdi
+; AVX1-NEXT:    movq %rdx, %rsi
+; AVX1-NEXT:    movq %rdx, %rbp
 ; AVX1-NEXT:    andb $15, %dl
 ; AVX1-NEXT:    movb %dl, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    shrq $56, %rax
-; AVX1-NEXT:    andb $15, %al
-; AVX1-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    movq %r14, %r10
-; AVX1-NEXT:    shrq $48, %rcx
+; AVX1-NEXT:    movq %rcx, %rdx
 ; AVX1-NEXT:    andb $15, %cl
 ; AVX1-NEXT:    movb %cl, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    movq %r14, %rdx
-; AVX1-NEXT:    shrq $40, %rdi
-; AVX1-NEXT:    andb $15, %dil
-; AVX1-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    movq %r14, %rax
-; AVX1-NEXT:    shrq $32, %rsi
+; AVX1-NEXT:    shrq $56, %rbp
+; AVX1-NEXT:    andb $15, %bpl
+; AVX1-NEXT:    movb %bpl, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    shrq $48, %rsi
 ; AVX1-NEXT:    andb $15, %sil
 ; AVX1-NEXT:    movb %sil, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    movq %r14, %rcx
-; AVX1-NEXT:    shrq $24, %r11
-; AVX1-NEXT:    andb $15, %r11b
-; AVX1-NEXT:    movb %r11b, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    movq %r14, %rsi
-; AVX1-NEXT:    shrq $16, %r9
-; AVX1-NEXT:    andb $15, %r9b
-; AVX1-NEXT:    movb %r9b, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    movq %r14, %rdi
-; AVX1-NEXT:    shrq $8, %r8
-; AVX1-NEXT:    andb $15, %r8b
-; AVX1-NEXT:    movb %r8b, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    movq %r14, %rbx
-; AVX1-NEXT:    andb $15, %r14b
-; AVX1-NEXT:    movb %r14b, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    shrq $8, %r10
-; AVX1-NEXT:    shrq $16, %rdx
-; AVX1-NEXT:    shrq $24, %rax
-; AVX1-NEXT:    shrq $32, %rcx
-; AVX1-NEXT:    shrq $40, %rsi
-; AVX1-NEXT:    shrq $48, %rdi
-; AVX1-NEXT:    shrq $56, %rbx
-; AVX1-NEXT:    andb $15, %bl
-; AVX1-NEXT:    movb %bl, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    shrq $40, %rdi
 ; AVX1-NEXT:    andb $15, %dil
 ; AVX1-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    andb $15, %sil
-; AVX1-NEXT:    movb %sil, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    andb $15, %cl
-; AVX1-NEXT:    movb %cl, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    shrq $32, %rax
 ; AVX1-NEXT:    andb $15, %al
 ; AVX1-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    shrq $24, %rbx
+; AVX1-NEXT:    andb $15, %bl
+; AVX1-NEXT:    movb %bl, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    shrq $16, %r13
+; AVX1-NEXT:    andb $15, %r13b
+; AVX1-NEXT:    movb %r13b, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    shrq $8, %r12
+; AVX1-NEXT:    andb $15, %r12b
+; AVX1-NEXT:    movb %r12b, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    shrq $8, %r8
+; AVX1-NEXT:    shrq $16, %r9
+; AVX1-NEXT:    shrq $24, %r10
+; AVX1-NEXT:    shrq $32, %r11
+; AVX1-NEXT:    shrq $40, %r14
+; AVX1-NEXT:    shrq $48, %r15
+; AVX1-NEXT:    shrq $56, %rdx
 ; AVX1-NEXT:    andb $15, %dl
 ; AVX1-NEXT:    movb %dl, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    andb $15, %r15b
+; AVX1-NEXT:    movb %r15b, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    andb $15, %r14b
+; AVX1-NEXT:    movb %r14b, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    andb $15, %r11b
+; AVX1-NEXT:    movb %r11b, -{{[0-9]+}}(%rsp)
 ; AVX1-NEXT:    andb $15, %r10b
 ; AVX1-NEXT:    movb %r10b, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    andb $15, %r9b
+; AVX1-NEXT:    movb %r9b, -{{[0-9]+}}(%rsp)
+; AVX1-NEXT:    andb $15, %r8b
+; AVX1-NEXT:    movb %r8b, -{{[0-9]+}}(%rsp)
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    movq %rax, %r8
+; AVX1-NEXT:    movq %rax, %rcx
 ; AVX1-NEXT:    movq %rax, %rdx
 ; AVX1-NEXT:    movq %rax, %rsi
 ; AVX1-NEXT:    movq %rax, %rdi
+; AVX1-NEXT:    movl %eax, %ebp
 ; AVX1-NEXT:    movl %eax, %ebx
-; AVX1-NEXT:    movl %eax, %ecx
 ; AVX1-NEXT:    vmovd %eax, %xmm1
 ; AVX1-NEXT:    shrl $8, %eax
 ; AVX1-NEXT:    vpinsrb $1, %eax, %xmm1, %xmm1
-; AVX1-NEXT:    shrl $16, %ecx
-; AVX1-NEXT:    vpinsrb $2, %ecx, %xmm1, %xmm1
-; AVX1-NEXT:    shrl $24, %ebx
-; AVX1-NEXT:    vpinsrb $3, %ebx, %xmm1, %xmm1
+; AVX1-NEXT:    shrl $16, %ebx
+; AVX1-NEXT:    vpinsrb $2, %ebx, %xmm1, %xmm1
+; AVX1-NEXT:    shrl $24, %ebp
+; AVX1-NEXT:    vpinsrb $3, %ebp, %xmm1, %xmm1
 ; AVX1-NEXT:    shrq $32, %rdi
 ; AVX1-NEXT:    vpinsrb $4, %edi, %xmm1, %xmm1
 ; AVX1-NEXT:    shrq $40, %rsi
 ; AVX1-NEXT:    vpinsrb $5, %esi, %xmm1, %xmm1
 ; AVX1-NEXT:    movb $0, -{{[0-9]+}}(%rsp)
 ; AVX1-NEXT:    vmovdqa -{{[0-9]+}}(%rsp), %xmm2
 ; AVX1-NEXT:    shrq $48, %rdx
 ; AVX1-NEXT:    vpinsrb $6, %edx, %xmm1, %xmm1
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX1-NEXT:    shrq $56, %r8
-; AVX1-NEXT:    vpinsrb $7, %r8d, %xmm1, %xmm0
+; AVX1-NEXT:    shrq $56, %rcx
+; AVX1-NEXT:    vpinsrb $7, %ecx, %xmm1, %xmm0
 ; AVX1-NEXT:    movl %eax, %ecx
 ; AVX1-NEXT:    shrl $8, %ecx
 ; AVX1-NEXT:    vpinsrb $8, %eax, %xmm0, %xmm0
 ; AVX1-NEXT:    vpinsrb $9, %ecx, %xmm0, %xmm0
 ; AVX1-NEXT:    movl %eax, %ecx
 ; AVX1-NEXT:    shrl $16, %ecx
 ; AVX1-NEXT:    vpinsrb $10, %ecx, %xmm0, %xmm0
 ; AVX1-NEXT:    movl %eax, %ecx

 ; AVX1-NEXT:    vpinsrb $13, %ecx, %xmm1, %xmm1
 ; AVX1-NEXT:    movq %rax, %rcx
 ; AVX1-NEXT:    shrq $48, %rcx
 ; AVX1-NEXT:    vpinsrb $14, %ecx, %xmm1, %xmm1
 ; AVX1-NEXT:    shrq $56, %rax
 ; AVX1-NEXT:    vpinsrb $15, %eax, %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    popq %rbx
+; AVX1-NEXT:    popq %r12
+; AVX1-NEXT:    popq %r13
 ; AVX1-NEXT:    popq %r14
+; AVX1-NEXT:    popq %r15
+; AVX1-NEXT:    popq %rbp
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: _clearupper32xi8b:
 ; AVX2:       # BB#0:
+; AVX2-NEXT:    pushq %rbp
+; AVX2-NEXT:    pushq %r15
 ; AVX2-NEXT:    pushq %r14
+; AVX2-NEXT:    pushq %r13
+; AVX2-NEXT:    pushq %r12
 ; AVX2-NEXT:    pushq %rbx
-; AVX2-NEXT:    vpextrq $1, %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    vmovq %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movq -{{[0-9]+}}(%rsp), %r14
+; AVX2-NEXT:    vmovq %xmm0, %rcx
+; AVX2-NEXT:    movq %rcx, %r8
+; AVX2-NEXT:    movq %rcx, %r9
+; AVX2-NEXT:    movq %rcx, %r10
+; AVX2-NEXT:    movq %rcx, %r11
+; AVX2-NEXT:    movq %rcx, %r14
+; AVX2-NEXT:    movq %rcx, %r15
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rdx
-; AVX2-NEXT:    movq %rdx, %r8
-; AVX2-NEXT:    movq %rdx, %r9
-; AVX2-NEXT:    movq %rdx, %r11
-; AVX2-NEXT:    movq %rdx, %rsi
-; AVX2-NEXT:    movq %rdx, %rdi
-; AVX2-NEXT:    movq %rdx, %rcx
+; AVX2-NEXT:    movq %rdx, %r12
+; AVX2-NEXT:    movq %rdx, %r13
+; AVX2-NEXT:    movq %rdx, %rbx
 ; AVX2-NEXT:    movq %rdx, %rax
+; AVX2-NEXT:    movq %rdx, %rdi
+; AVX2-NEXT:    movq %rdx, %rsi
+; AVX2-NEXT:    movq %rdx, %rbp
 ; AVX2-NEXT:    andb $15, %dl
 ; AVX2-NEXT:    movb %dl, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    shrq $56, %rax
-; AVX2-NEXT:    andb $15, %al
-; AVX2-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movq %r14, %r10
-; AVX2-NEXT:    shrq $48, %rcx
+; AVX2-NEXT:    movq %rcx, %rdx
 ; AVX2-NEXT:    andb $15, %cl
 ; AVX2-NEXT:    movb %cl, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movq %r14, %rdx
-; AVX2-NEXT:    shrq $40, %rdi
-; AVX2-NEXT:    andb $15, %dil
-; AVX2-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movq %r14, %rax
-; AVX2-NEXT:    shrq $32, %rsi
+; AVX2-NEXT:    shrq $56, %rbp
+; AVX2-NEXT:    andb $15, %bpl
+; AVX2-NEXT:    movb %bpl, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    shrq $48, %rsi
 ; AVX2-NEXT:    andb $15, %sil
 ; AVX2-NEXT:    movb %sil, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movq %r14, %rcx
-; AVX2-NEXT:    shrq $24, %r11
-; AVX2-NEXT:    andb $15, %r11b
-; AVX2-NEXT:    movb %r11b, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movq %r14, %rsi
-; AVX2-NEXT:    shrq $16, %r9
-; AVX2-NEXT:    andb $15, %r9b
-; AVX2-NEXT:    movb %r9b, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movq %r14, %rdi
-; AVX2-NEXT:    shrq $8, %r8
-; AVX2-NEXT:    andb $15, %r8b
-; AVX2-NEXT:    movb %r8b, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movq %r14, %rbx
-; AVX2-NEXT:    andb $15, %r14b
-; AVX2-NEXT:    movb %r14b, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    shrq $8, %r10
-; AVX2-NEXT:    shrq $16, %rdx
-; AVX2-NEXT:    shrq $24, %rax
-; AVX2-NEXT:    shrq $32, %rcx
-; AVX2-NEXT:    shrq $40, %rsi
-; AVX2-NEXT:    shrq $48, %rdi
-; AVX2-NEXT:    shrq $56, %rbx
-; AVX2-NEXT:    andb $15, %bl
-; AVX2-NEXT:    movb %bl, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    shrq $40, %rdi
 ; AVX2-NEXT:    andb $15, %dil
 ; AVX2-NEXT:    movb %dil, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    andb $15, %sil
-; AVX2-NEXT:    movb %sil, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    andb $15, %cl
-; AVX2-NEXT:    movb %cl, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    shrq $32, %rax
 ; AVX2-NEXT:    andb $15, %al
 ; AVX2-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    shrq $24, %rbx
+; AVX2-NEXT:    andb $15, %bl
+; AVX2-NEXT:    movb %bl, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    shrq $16, %r13
+; AVX2-NEXT:    andb $15, %r13b
+; AVX2-NEXT:    movb %r13b, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    shrq $8, %r12
+; AVX2-NEXT:    andb $15, %r12b
+; AVX2-NEXT:    movb %r12b, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    shrq $8, %r8
+; AVX2-NEXT:    shrq $16, %r9
+; AVX2-NEXT:    shrq $24, %r10
+; AVX2-NEXT:    shrq $32, %r11
+; AVX2-NEXT:    shrq $40, %r14
+; AVX2-NEXT:    shrq $48, %r15
+; AVX2-NEXT:    shrq $56, %rdx
 ; AVX2-NEXT:    andb $15, %dl
 ; AVX2-NEXT:    movb %dl, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    andb $15, %r15b
+; AVX2-NEXT:    movb %r15b, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    andb $15, %r14b
+; AVX2-NEXT:    movb %r14b, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    andb $15, %r11b
+; AVX2-NEXT:    movb %r11b, -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    andb $15, %r10b
 ; AVX2-NEXT:    movb %r10b, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    andb $15, %r9b
+; AVX2-NEXT:    movb %r9b, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT:    andb $15, %r8b
+; AVX2-NEXT:    movb %r8b, -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    movq %rax, %r8
+; AVX2-NEXT:    movq %rax, %rcx
 ; AVX2-NEXT:    movq %rax, %rdx
 ; AVX2-NEXT:    movq %rax, %rsi
 ; AVX2-NEXT:    movq %rax, %rdi
+; AVX2-NEXT:    movl %eax, %ebp
 ; AVX2-NEXT:    movl %eax, %ebx
-; AVX2-NEXT:    movl %eax, %ecx
 ; AVX2-NEXT:    vmovd %eax, %xmm1
 ; AVX2-NEXT:    shrl $8, %eax
 ; AVX2-NEXT:    vpinsrb $1, %eax, %xmm1, %xmm1
-; AVX2-NEXT:    shrl $16, %ecx
-; AVX2-NEXT:    vpinsrb $2, %ecx, %xmm1, %xmm1
-; AVX2-NEXT:    shrl $24, %ebx
-; AVX2-NEXT:    vpinsrb $3, %ebx, %xmm1, %xmm1
+; AVX2-NEXT:    shrl $16, %ebx
+; AVX2-NEXT:    vpinsrb $2, %ebx, %xmm1, %xmm1
+; AVX2-NEXT:    shrl $24, %ebp
+; AVX2-NEXT:    vpinsrb $3, %ebp, %xmm1, %xmm1
 ; AVX2-NEXT:    shrq $32, %rdi
 ; AVX2-NEXT:    vpinsrb $4, %edi, %xmm1, %xmm1
 ; AVX2-NEXT:    shrq $40, %rsi
 ; AVX2-NEXT:    vpinsrb $5, %esi, %xmm1, %xmm1
 ; AVX2-NEXT:    movb $0, -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    vmovdqa -{{[0-9]+}}(%rsp), %xmm2
 ; AVX2-NEXT:    shrq $48, %rdx
 ; AVX2-NEXT:    vpinsrb $6, %edx, %xmm1, %xmm1
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
-; AVX2-NEXT:    shrq $56, %r8
-; AVX2-NEXT:    vpinsrb $7, %r8d, %xmm1, %xmm0
+; AVX2-NEXT:    shrq $56, %rcx
+; AVX2-NEXT:    vpinsrb $7, %ecx, %xmm1, %xmm0
 ; AVX2-NEXT:    movl %eax, %ecx
 ; AVX2-NEXT:    shrl $8, %ecx
 ; AVX2-NEXT:    vpinsrb $8, %eax, %xmm0, %xmm0
 ; AVX2-NEXT:    vpinsrb $9, %ecx, %xmm0, %xmm0
 ; AVX2-NEXT:    movl %eax, %ecx
 ; AVX2-NEXT:    shrl $16, %ecx
 ; AVX2-NEXT:    vpinsrb $10, %ecx, %xmm0, %xmm0
 ; AVX2-NEXT:    movl %eax, %ecx

 ; AVX2-NEXT:    vpinsrb $13, %ecx, %xmm1, %xmm1
 ; AVX2-NEXT:    movq %rax, %rcx
 ; AVX2-NEXT:    shrq $48, %rcx
 ; AVX2-NEXT:    vpinsrb $14, %ecx, %xmm1, %xmm1
 ; AVX2-NEXT:    shrq $56, %rax
 ; AVX2-NEXT:    vpinsrb $15, %eax, %xmm1, %xmm1
 ; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
 ; AVX2-NEXT:    popq %rbx
+; AVX2-NEXT:    popq %r12
+; AVX2-NEXT:    popq %r13
 ; AVX2-NEXT:    popq %r14
+; AVX2-NEXT:    popq %r15
+; AVX2-NEXT:    popq %rbp
 ; AVX2-NEXT:    retq
   %x4  = bitcast <32 x i8> %0 to <64 x i4>
   %r0  = insertelement <64 x i4> %x4,  i4 zeroinitializer, i32 1
   %r1  = insertelement <64 x i4> %r0,  i4 zeroinitializer, i32 3
   %r2  = insertelement <64 x i4> %r1,  i4 zeroinitializer, i32 5
   %r3  = insertelement <64 x i4> %r2,  i4 zeroinitializer, i32 7
   %r4  = insertelement <64 x i4> %r3,  i4 zeroinitializer, i32 9
   %r5  = insertelement <64 x i4> %r4,  i4 zeroinitializer, i32 11

test/CodeGen/X86/constant-combines.ll

 ; The DAG combiner at one point contained bugs that given enough permutations
 ; would incorrectly form an illegal operation for the last of these stores when
 ; it folded it to a zero too late to legalize the zero store operation. If this
 ; ever starts forming a zero store instead of movss, the test case has stopped
 ; being useful.
 ;
 ; CHECK-LABEL: PR22524:
 ; CHECK:       # BB#0: # %entry
-; CHECK-NEXT:    movl $0, 4(%rdi)
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    movd %eax, %xmm0
 ; CHECK-NEXT:    xorps %xmm1, %xmm1
 ; CHECK-NEXT:    mulss %xmm0, %xmm1
-; CHECK-NEXT:    movl $0, (%rdi)
+; CHECK-NEXT:    movq $0, (%rdi)
 ; CHECK-NEXT:    movss %xmm1, 4(%rdi)
 ; CHECK-NEXT:    retq
 entry:
   %0 = getelementptr inbounds { float, float }, { float, float }* %arg,  i32 0, i32 1
   store float 0.000000e+00, float* %0, align 4
   %1 = getelementptr inbounds { float, float }, { float, float }* %arg, i64 0,  i32 0
   %2 = bitcast float* %1 to i64*
   %3 = load i64, i64* %2, align 8

test/CodeGen/X86/fold-vector-sext-crash2.ll

 }
 
 define <2 x i256> @test_zext1() {
   %Se = zext <2 x i8> <i8 -1, i8 -2> to <2 x i256>
   %Shuff = shufflevector <2 x i256> zeroinitializer, <2 x i256> %Se, <2 x i32> <i32 1, i32 3>
   ret <2 x i256> %Shuff
 
   ; X64-LABEL: test_zext1
-  ; X64:       movq $0
-  ; X64-NEXT:  movq $0
+  ; X64: xorps   %xmm0, %xmm0
+  ; X64: movaps  %xmm0
+  ; X64: movaps  %xmm0
+  ; X64: movaps  %xmm0
   ; X64-NEXT:  movq $0
   ; X64-NEXT:  movq $254
 
   ; X32-LABEL: test_zext1
   ; X32:       movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $254
 }
 
 define <2 x i256> @test_zext2() {
   %Se = zext <2 x i128> <i128 -1, i128 -2> to <2 x i256>
   %Shuff = shufflevector <2 x i256> zeroinitializer, <2 x i256> %Se, <2 x i32> <i32 1, i32 3>
   ret <2 x i256> %Shuff
 
   ; X64-LABEL: test_zext2
-  ; X64:       movq $0
-  ; X64-NEXT:  movq $0
+  ; X64: xorps   %xmm0, %xmm0
+  ; X64-NEXT: movaps  %xmm0
+  ; X64-NEXT: movaps  %xmm0
+  ; X64-NEXT: movaps  %xmm0  
   ; X64-NEXT:  movq $-1
   ; X64-NEXT:  movq $-2
 
   ; X32-LABEL: test_zext2
   ; X32:       movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $0
   ; X32-NEXT:  movl $-1
   ; X32-NEXT:  movl $-1
   ; X32-NEXT:  movl $-1
   ; X32-NEXT:  movl $-2
 }

test/CodeGen/X86/legalize-shl-vec.ll

 ; X32-NEXT:    movl $0, 12(%eax)
 ; X32-NEXT:    movl $0, 8(%eax)
 ; X32-NEXT:    movl $0, 4(%eax)
 ; X32-NEXT:    movl $0, (%eax)
 ; X32-NEXT:    retl $4
 ;
 ; X64-LABEL: test_shl:
 ; X64:       # BB#0:
-; X64-NEXT:    movq $0, 56(%rdi)
-; X64-NEXT:    movq $0, 48(%rdi)
-; X64-NEXT:    movq $0, 40(%rdi)
-; X64-NEXT:    movq $0, 32(%rdi)
-; X64-NEXT:    movq $0, 24(%rdi)
-; X64-NEXT:    movq $0, 16(%rdi)
-; X64-NEXT:    movq $0, 8(%rdi)
-; X64-NEXT:    movq $0, (%rdi)
+; X64-NEXT:    xorps %xmm0, %xmm0
+; X64-NEXT:    movaps %xmm0, 48(%rdi)
+; X64-NEXT:    movaps %xmm0, 32(%rdi)
+; X64-NEXT:    movaps %xmm0, 16(%rdi)
+; X64-NEXT:    movaps %xmm0, (%rdi)
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    retq
   %Amt = insertelement <2 x i256> undef, i256 -1, i32 0
   %Out = shl <2 x i256> %In, %Amt
   ret <2 x i256> %Out
 }
 
 define <2 x i256> @test_srl(<2 x i256> %In) {

 ; X32-NEXT:    movl $0, 12(%eax)
 ; X32-NEXT:    movl $0, 8(%eax)
 ; X32-NEXT:    movl $0, 4(%eax)
 ; X32-NEXT:    movl $0, (%eax)
 ; X32-NEXT:    retl $4
 ;
 ; X64-LABEL: test_srl:
 ; X64:       # BB#0:
-; X64-NEXT:    movq $0, 56(%rdi)
-; X64-NEXT:    movq $0, 48(%rdi)
-; X64-NEXT:    movq $0, 40(%rdi)
-; X64-NEXT:    movq $0, 32(%rdi)
-; X64-NEXT:    movq $0, 24(%rdi)
-; X64-NEXT:    movq $0, 16(%rdi)
-; X64-NEXT:    movq $0, 8(%rdi)
-; X64-NEXT:    movq $0, (%rdi)
+; X64-NEXT:    xorps %xmm0, %xmm0
+; X64-NEXT:    movaps %xmm0, 48(%rdi)
+; X64-NEXT:    movaps %xmm0, 32(%rdi)
+; X64-NEXT:    movaps %xmm0, 16(%rdi)
+; X64-NEXT:    movaps %xmm0, (%rdi)
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    retq
   %Amt = insertelement <2 x i256> undef, i256 -1, i32 0
   %Out = lshr <2 x i256> %In, %Amt
   ret <2 x i256> %Out
 }
 
 define <2 x i256> @test_sra(<2 x i256> %In) {

test/CodeGen/X86/merge-consecutive-loads-128.ll

 ;
 ; AVX-LABEL: merge_8i16_i16_23u567u9:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovups 4(%rdi), %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE1-LABEL: merge_8i16_i16_23u567u9:
 ; X32-SSE1:       # BB#0:
-; X32-SSE1-NEXT:    pushl %ebp
+; X32-SSE1-NEXT:    pushl %edi
 ; X32-SSE1-NEXT:  .Lcfi6:
 ; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    pushl %ebx
+; X32-SSE1-NEXT:    pushl %esi
 ; X32-SSE1-NEXT:  .Lcfi7:
 ; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
-; X32-SSE1-NEXT:    pushl %edi
 ; X32-SSE1-NEXT:  .Lcfi8:
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 16
-; X32-SSE1-NEXT:    pushl %esi
+; X32-SSE1-NEXT:    .cfi_offset %esi, -12
 ; X32-SSE1-NEXT:  .Lcfi9:
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 20
-; X32-SSE1-NEXT:  .Lcfi10:
-; X32-SSE1-NEXT:    .cfi_offset %esi, -20
-; X32-SSE1-NEXT:  .Lcfi11:
-; X32-SSE1-NEXT:    .cfi_offset %edi, -16
-; X32-SSE1-NEXT:  .Lcfi12:
-; X32-SSE1-NEXT:    .cfi_offset %ebx, -12
-; X32-SSE1-NEXT:  .Lcfi13:
-; X32-SSE1-NEXT:    .cfi_offset %ebp, -8
+; X32-SSE1-NEXT:    .cfi_offset %edi, -8
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movzwl 4(%ecx), %edx
-; X32-SSE1-NEXT:    movzwl 6(%ecx), %esi
-; X32-SSE1-NEXT:    movzwl 10(%ecx), %edi
-; X32-SSE1-NEXT:    movzwl 12(%ecx), %ebx
-; X32-SSE1-NEXT:    movzwl 14(%ecx), %ebp
+; X32-SSE1-NEXT:    movl 4(%ecx), %edx
+; X32-SSE1-NEXT:    movl 10(%ecx), %esi
+; X32-SSE1-NEXT:    movzwl 14(%ecx), %edi
 ; X32-SSE1-NEXT:    movzwl 18(%ecx), %ecx
-; X32-SSE1-NEXT:    movw %bp, 10(%eax)
-; X32-SSE1-NEXT:    movw %bx, 8(%eax)
+; X32-SSE1-NEXT:    movw %di, 10(%eax)
 ; X32-SSE1-NEXT:    movw %cx, 14(%eax)
-; X32-SSE1-NEXT:    movw %si, 2(%eax)
-; X32-SSE1-NEXT:    movw %dx, (%eax)
-; X32-SSE1-NEXT:    movw %di, 6(%eax)
+; X32-SSE1-NEXT:    movl %edx, (%eax)
+; X32-SSE1-NEXT:    movl %esi, 6(%eax)
 ; X32-SSE1-NEXT:    popl %esi
 ; X32-SSE1-NEXT:    popl %edi
-; X32-SSE1-NEXT:    popl %ebx
-; X32-SSE1-NEXT:    popl %ebp
 ; X32-SSE1-NEXT:    retl $4
 ;
 ; X32-SSE41-LABEL: merge_8i16_i16_23u567u9:
 ; X32-SSE41:       # BB#0:
 ; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE41-NEXT:    movups 4(%eax), %xmm0
 ; X32-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 2

 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE1-LABEL: merge_8i16_i16_34uuuuuu:
 ; X32-SSE1:       # BB#0:
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movzwl 6(%ecx), %edx
-; X32-SSE1-NEXT:    movzwl 8(%ecx), %ecx
-; X32-SSE1-NEXT:    movw %cx, 2(%eax)
-; X32-SSE1-NEXT:    movw %dx, (%eax)
+; X32-SSE1-NEXT:    movl 6(%ecx), %ecx
+; X32-SSE1-NEXT:    movl %ecx, (%eax)
 ; X32-SSE1-NEXT:    retl $4
 ;
 ; X32-SSE41-LABEL: merge_8i16_i16_34uuuuuu:
 ; X32-SSE41:       # BB#0:
 ; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X32-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 3

 ;
 ; AVX-LABEL: merge_8i16_i16_45u7zzzz:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE1-LABEL: merge_8i16_i16_45u7zzzz:
 ; X32-SSE1:       # BB#0:
-; X32-SSE1-NEXT:    pushl %esi
-; X32-SSE1-NEXT:  .Lcfi14:
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:  .Lcfi15:
-; X32-SSE1-NEXT:    .cfi_offset %esi, -8
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movzwl 8(%ecx), %edx
-; X32-SSE1-NEXT:    movzwl 10(%ecx), %esi
+; X32-SSE1-NEXT:    movl 8(%ecx), %edx
 ; X32-SSE1-NEXT:    movzwl 14(%ecx), %ecx
-; X32-SSE1-NEXT:    movw %si, 2(%eax)
-; X32-SSE1-NEXT:    movw %dx, (%eax)
+; X32-SSE1-NEXT:    movl %edx, (%eax)
 ; X32-SSE1-NEXT:    movw %cx, 6(%eax)
-; X32-SSE1-NEXT:    movw $0, 14(%eax)
-; X32-SSE1-NEXT:    movw $0, 12(%eax)
-; X32-SSE1-NEXT:    movw $0, 10(%eax)
-; X32-SSE1-NEXT:    movw $0, 8(%eax)
-; X32-SSE1-NEXT:    popl %esi
+; X32-SSE1-NEXT:    movl $0, 12(%eax)
+; X32-SSE1-NEXT:    movl $0, 8(%eax)
 ; X32-SSE1-NEXT:    retl $4
 ;
 ; X32-SSE41-LABEL: merge_8i16_i16_45u7zzzz:
 ; X32-SSE41:       # BB#0:
 ; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X32-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 4

 ;
 ; AVX-LABEL: merge_16i8_i8_01u3456789ABCDuF:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovups (%rdi), %xmm0
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE1-LABEL: merge_16i8_i8_01u3456789ABCDuF:
 ; X32-SSE1:       # BB#0:
+; X32-SSE1-NEXT:    pushl %ebp
+; X32-SSE1-NEXT:  .Lcfi10:
+; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
 ; X32-SSE1-NEXT:    pushl %ebx
+; X32-SSE1-NEXT:  .Lcfi11:
+; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
+; X32-SSE1-NEXT:    pushl %edi
+; X32-SSE1-NEXT:  .Lcfi12:
+; X32-SSE1-NEXT:    .cfi_def_cfa_offset 16
+; X32-SSE1-NEXT:    pushl %esi
+; X32-SSE1-NEXT:  .Lcfi13:
+; X32-SSE1-NEXT:    .cfi_def_cfa_offset 20
+; X32-SSE1-NEXT:  .Lcfi14:
+; X32-SSE1-NEXT:    .cfi_offset %esi, -20
+; X32-SSE1-NEXT:  .Lcfi15:
+; X32-SSE1-NEXT:    .cfi_offset %edi, -16
 ; X32-SSE1-NEXT:  .Lcfi16:
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    subl $12, %esp
+; X32-SSE1-NEXT:    .cfi_offset %ebx, -12
 ; X32-SSE1-NEXT:  .Lcfi17:
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 20
-; X32-SSE1-NEXT:  .Lcfi18:
-; X32-SSE1-NEXT:    .cfi_offset %ebx, -8
+; X32-SSE1-NEXT:    .cfi_offset %ebp, -8
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movb (%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 1(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 3(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 4(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 5(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 6(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 7(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 8(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 9(%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 10(%ecx), %bh
-; X32-SSE1-NEXT:    movb 11(%ecx), %bl
-; X32-SSE1-NEXT:    movb 12(%ecx), %dh
+; X32-SSE1-NEXT:    movzwl (%ecx), %ebp
+; X32-SSE1-NEXT:    movl 3(%ecx), %esi
+; X32-SSE1-NEXT:    movl 7(%ecx), %edi
+; X32-SSE1-NEXT:    movzwl 11(%ecx), %ebx
 ; X32-SSE1-NEXT:    movb 13(%ecx), %dl
 ; X32-SSE1-NEXT:    movb 15(%ecx), %cl
 ; X32-SSE1-NEXT:    movb %dl, 13(%eax)
-; X32-SSE1-NEXT:    movb %dh, 12(%eax)
 ; X32-SSE1-NEXT:    movb %cl, 15(%eax)
-; X32-SSE1-NEXT:    movb %bl, 11(%eax)
-; X32-SSE1-NEXT:    movb %bh, 10(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 9(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 8(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 7(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 6(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 5(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 4(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 1(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, (%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, 3(%eax)
-; X32-SSE1-NEXT:    addl $12, %esp
+; X32-SSE1-NEXT:    movw %bx, 11(%eax)
+; X32-SSE1-NEXT:    movl %edi, 7(%eax)
+; X32-SSE1-NEXT:    movw %bp, (%eax)
+; X32-SSE1-NEXT:    movl %esi, 3(%eax)
+; X32-SSE1-NEXT:    popl %esi
+; X32-SSE1-NEXT:    popl %edi
 ; X32-SSE1-NEXT:    popl %ebx
+; X32-SSE1-NEXT:    popl %ebp
 ; X32-SSE1-NEXT:    retl $4
 ;
 ; X32-SSE41-LABEL: merge_16i8_i8_01u3456789ABCDuF:
 ; X32-SSE41:       # BB#0:
 ; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE41-NEXT:    movups (%eax), %xmm0
 ; X32-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 0

 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE1-LABEL: merge_16i8_i8_01u3uuzzuuuuuzzz:
 ; X32-SSE1:       # BB#0:
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movb (%ecx), %dl
-; X32-SSE1-NEXT:    movb 1(%ecx), %dh
+; X32-SSE1-NEXT:    movzwl (%ecx), %edx
 ; X32-SSE1-NEXT:    movb 3(%ecx), %cl
-; X32-SSE1-NEXT:    movb %dh, 1(%eax)
-; X32-SSE1-NEXT:    movb %dl, (%eax)
+; X32-SSE1-NEXT:    movw %dx, (%eax)
 ; X32-SSE1-NEXT:    movb %cl, 3(%eax)
 ; X32-SSE1-NEXT:    movb $0, 15(%eax)
-; X32-SSE1-NEXT:    movb $0, 14(%eax)
-; X32-SSE1-NEXT:    movb $0, 13(%eax)
-; X32-SSE1-NEXT:    movb $0, 7(%eax)
-; X32-SSE1-NEXT:    movb $0, 6(%eax)
+; X32-SSE1-NEXT:    movw $0, 13(%eax)
+; X32-SSE1-NEXT:    movw $0, 6(%eax)
 ; X32-SSE1-NEXT:    retl $4
 ;
 ; X32-SSE41-LABEL: merge_16i8_i8_01u3uuzzuuuuuzzz:
 ; X32-SSE41:       # BB#0:
 ; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X32-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 0

 ;
 ; AVX-LABEL: merge_16i8_i8_0123uu67uuuuuzzz:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE1-LABEL: merge_16i8_i8_0123uu67uuuuuzzz:
 ; X32-SSE1:       # BB#0:
-; X32-SSE1-NEXT:    pushl %ebx
-; X32-SSE1-NEXT:  .Lcfi19:
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    pushl %eax
-; X32-SSE1-NEXT:  .Lcfi20:
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
-; X32-SSE1-NEXT:  .Lcfi21:
-; X32-SSE1-NEXT:    .cfi_offset %ebx, -8
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movb (%ecx), %dl
-; X32-SSE1-NEXT:    movb %dl, {{[0-9]+}}(%esp) # 1-byte Spill
-; X32-SSE1-NEXT:    movb 1(%ecx), %dh
-; X32-SSE1-NEXT:    movb 2(%ecx), %bl
-; X32-SSE1-NEXT:    movb 3(%ecx), %bh
-; X32-SSE1-NEXT:    movb 6(%ecx), %dl
-; X32-SSE1-NEXT:    movb 7(%ecx), %cl
-; X32-SSE1-NEXT:    movb %cl, 7(%eax)
-; X32-SSE1-NEXT:    movb %dl, 6(%eax)
-; X32-SSE1-NEXT:    movb %bh, 3(%eax)
-; X32-SSE1-NEXT:    movb %bl, 2(%eax)
-; X32-SSE1-NEXT:    movb %dh, 1(%eax)
-; X32-SSE1-NEXT:    movb {{[0-9]+}}(%esp), %cl # 1-byte Reload
-; X32-SSE1-NEXT:    movb %cl, (%eax)
+; X32-SSE1-NEXT:    movl (%ecx), %edx
+; X32-SSE1-NEXT:    movzwl 6(%ecx), %ecx
+; X32-SSE1-NEXT:    movw %cx, 6(%eax)
+; X32-SSE1-NEXT:    movl %edx, (%eax)
 ; X32-SSE1-NEXT:    movb $0, 15(%eax)
-; X32-SSE1-NEXT:    movb $0, 14(%eax)
-; X32-SSE1-NEXT:    movb $0, 13(%eax)
-; X32-SSE1-NEXT:    addl $4, %esp
-; X32-SSE1-NEXT:    popl %ebx
+; X32-SSE1-NEXT:    movw $0, 13(%eax)
 ; X32-SSE1-NEXT:    retl $4
 ;
 ; X32-SSE41-LABEL: merge_16i8_i8_0123uu67uuuuuzzz:
 ; X32-SSE41:       # BB#0:
 ; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; X32-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 0

 ; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX-NEXT:    retq
 ;
 ; X32-SSE1-LABEL: merge_2i64_i64_12_volatile:
 ; X32-SSE1:       # BB#0:
 ; X32-SSE1-NEXT:    pushl %edi
-; X32-SSE1-NEXT:  .Lcfi22:
+; X32-SSE1-NEXT:  .Lcfi18:
 ; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
 ; X32-SSE1-NEXT:    pushl %esi
-; X32-SSE1-NEXT:  .Lcfi23:
+; X32-SSE1-NEXT:  .Lcfi19:
 ; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
-; X32-SSE1-NEXT:  .Lcfi24:
+; X32-SSE1-NEXT:  .Lcfi20:
 ; X32-SSE1-NEXT:    .cfi_offset %esi, -12
-; X32-SSE1-NEXT:  .Lcfi25:
+; X32-SSE1-NEXT:  .Lcfi21:
 ; X32-SSE1-NEXT:    .cfi_offset %edi, -8
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-SSE1-NEXT:    movl 8(%ecx), %edx
 ; X32-SSE1-NEXT:    movl 12(%ecx), %esi
 ; X32-SSE1-NEXT:    movl 16(%ecx), %edi
 ; X32-SSE1-NEXT:    movl 20(%ecx), %ecx
 ; X32-SSE1-NEXT:    movl %ecx, 12(%eax)

test/CodeGen/X86/merge-store-partially-alias-loads.ll

 ; REQUIRES: asserts
 ; RUN: llc -march=x86-64 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck -check-prefix=X86 %s
 ; RUN: llc -march=x86-64 -mtriple=x86_64-unknown-linux-gnu -debug-only=isel < %s 2>&1 | FileCheck -check-prefix=DBGDAG %s
 
 ; It's OK to merge the load / store of the first 2 components, but
 ; they must not be placed on the same chain after merging.
 
 ; X86-LABEL: {{^}}merge_store_partial_overlap_load:
 ; X86-DAG: movzwl ([[BASEREG:%[a-z]+]]), %e[[LO2:[a-z]+]]
 ; X86-DAG: movb 2([[BASEREG]]), [[HI1:%[a-z]+]]
 
 ; X86-NEXT: movw %[[LO2]], 1([[BASEREG]])
 ; X86-NEXT: movb [[HI1]], 3([[BASEREG]])
 ; X86-NEXT: retq
 
-; DBGDAG-LABEL: Optimized lowered selection DAG: BB#0 'merge_store_partial_overlap_load:'
+; DBGDAG-LABEL: Optimized legalized selection DAG: BB#0 'merge_store_partial_overlap_load:'
 ; DBGDAG: [[ENTRYTOKEN:t[0-9]+]]: ch = EntryToken
 ; DBGDAG-DAG: [[BASEPTR:t[0-9]+]]: i64,ch = CopyFromReg [[ENTRYTOKEN]],
 ; DBGDAG-DAG: [[ADDPTR:t[0-9]+]]: i64 = add [[BASEPTR]], Constant:i64<2>
 
 ; DBGDAG-DAG: [[LD2:t[0-9]+]]: i16,ch = load<LD2[%tmp81](align=1)> [[ENTRYTOKEN]], [[BASEPTR]], undef:i64
 ; DBGDAG-DAG: [[LD1:t[0-9]+]]: i8,ch = load<LD1[%tmp12]> [[ENTRYTOKEN]], [[ADDPTR]], undef:i64
 
 ; DBGDAG-DAG: [[ST1:t[0-9]+]]: ch = store<ST1[%tmp14]> [[ENTRYTOKEN]], [[LD1]], t{{[0-9]+}}, undef:i64
 ; DBGDAG-DAG: [[LOADTOKEN:t[0-9]+]]: ch = TokenFactor [[LD2]]:1, [[LD1]]:1
 ; DBGDAG-DAG: [[ST2:t[0-9]+]]: ch = store<ST2[%tmp10](align=1)> [[LOADTOKEN]], [[LD2]], t{{[0-9]+}}, undef:i64
 
 ; DBGDAG: X86ISD::RET_FLAG t{{[0-9]+}},
 
-; DBGDAG: Type-legalized selection DAG: BB#0 'merge_store_partial_overlap_load:'
+; DBGDAG-LABEL: Instruction selection begins
 define void @merge_store_partial_overlap_load([4 x i8]* %tmp) {
   %tmp8 = getelementptr inbounds [4 x i8], [4 x i8]* %tmp, i32 0, i8 0
   %tmp10 = getelementptr inbounds [4 x i8], [4 x i8]* %tmp, i32 0, i8 1
   %tmp12 = getelementptr inbounds [4 x i8], [4 x i8]* %tmp, i32 0, i8 2
   %tmp14 = getelementptr [4 x i8], [4 x i8]* %tmp, i32 0, i8 3
 
   %tmp9 = load i8, i8* %tmp8, align 1   ; base + 0
   %tmp11 = load i8, i8* %tmp10, align 1 ; base + 1

test/CodeGen/X86/no-sse2-avg.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; REQUIRES: asserts
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-sse2 | FileCheck %s
 
 define <16 x i8> @PR27973() {
 ; CHECK-LABEL: PR27973:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    movb $0, 15(%rdi)
-; CHECK-NEXT:    movb $0, 14(%rdi)
-; CHECK-NEXT:    movb $0, 13(%rdi)
-; CHECK-NEXT:    movb $0, 12(%rdi)
-; CHECK-NEXT:    movb $0, 11(%rdi)
-; CHECK-NEXT:    movb $0, 10(%rdi)
-; CHECK-NEXT:    movb $0, 9(%rdi)
-; CHECK-NEXT:    movb $0, 8(%rdi)
-; CHECK-NEXT:    movb $0, 7(%rdi)
-; CHECK-NEXT:    movb $0, 6(%rdi)
-; CHECK-NEXT:    movb $0, 5(%rdi)
-; CHECK-NEXT:    movb $0, 4(%rdi)
-; CHECK-NEXT:    movb $0, 3(%rdi)
-; CHECK-NEXT:    movb $0, 2(%rdi)
-; CHECK-NEXT:    movb $0, 1(%rdi)
-; CHECK-NEXT:    movb $0, (%rdi)
+; CHECK-NEXT:    movq $0, 8(%rdi)
+; CHECK-NEXT:    movq $0, (%rdi)
 ; CHECK-NEXT:    movq %rdi, %rax
 ; CHECK-NEXT:    retq
   %t0 = zext <16 x i8> zeroinitializer to <16 x i32>
   %t1 = add nuw nsw <16 x i32> %t0, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %t2 = lshr <16 x i32> %t1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %t3 = trunc <16 x i32> %t2 to <16 x i8>
   ret <16 x i8> %t3
 }

test/CodeGen/X86/oddshuffles.ll

 ; SSE2-NEXT:    movd %xmm0, 8(%rdi)
 ; SSE2-NEXT:    movq %xmm2, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: v3i32:
 ; SSE42:       # BB#0:
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
 ; SSE42-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
-; SSE42-NEXT:    pextrd $2, %xmm0, 8(%rdi)
-; SSE42-NEXT:    movq %xmm1, (%rdi)
+; SSE42-NEXT:    movdqa %xmm1, (%rdi)
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: v3i32:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
-; AVX1-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
-; AVX1-NEXT:    vmovq %xmm1, (%rdi)
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
+; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v3i32:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2,3]
-; AVX2-NEXT:    vpextrd $2, %xmm0, 8(%rdi)
-; AVX2-NEXT:    vmovq %xmm1, (%rdi)
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
 ; AVX2-NEXT:    retq
   %r = shufflevector <2 x i32> %a, <2 x i32> %b, <3 x i32> <i32 0, i32 2, i32 1>
   store <3 x i32> %r, <3 x i32>* %p
   ret void
 }
 
 define void @v5i16(<4 x i16> %a, <4 x i16> %b, <5 x i16>* %p) nounwind {
 ; SSE2-LABEL: v5i16:

test/CodeGen/X86/stdarg.ll

 ; RUN: llc -stack-symbol-ordering=0 < %s -mtriple=x86_64-linux | FileCheck %s
 
 %struct.__va_list_tag = type { i32, i32, i8*, i8* }
 
 define void @foo(i32 %x, ...) nounwind {
 entry:
   %ap = alloca [1 x %struct.__va_list_tag], align 8; <[1 x %struct.__va_list_tag]*> [#uses=2]
   %ap12 = bitcast [1 x %struct.__va_list_tag]* %ap to i8*; <i8*> [#uses=2]
   call void @llvm.va_start(i8* %ap12)
 ; CHECK: testb %al, %al
 
 ; These test for specific offsets, which is very fragile. Still, the test needs
 ; to ensure that va_list has the correct element types.
 ;
 ; CHECK-DAG: movq {{.*}}, 192(%rsp)
 ; CHECK-DAG: movq {{.*}}, 184(%rsp)
-; CHECK-DAG: movl {{.*}}, 180(%rsp)
-; CHECK-DAG: movl {{.*}}, 176(%rsp)
+; CHECK-DAG: movq {{.*}}, 176(%rsp)
   %ap3 = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %ap, i64 0, i64 0; <%struct.__va_list_tag*> [#uses=1]
   call void @bar(%struct.__va_list_tag* %ap3) nounwind
   call void @llvm.va_end(i8* %ap12)
   ret void
 }
 
 declare void @llvm.va_start(i8*) nounwind
 
 declare void @bar(%struct.__va_list_tag*)
 
 declare void @llvm.va_end(i8*) nounwind

test/CodeGen/X86/stores-merging.ll

 ; RUN: llc < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 %structTy = type { i8, i32, i32 }
 
 @e = common global %structTy zeroinitializer, align 4
 
 ;; Ensure that MergeConsecutiveStores doesn't incorrectly reorder
 ;; store operations.  The first test stores in increasing address
 ;; order, the second in decreasing -- but in both cases should have
 ;; the same result in memory in the end.
 
 ; CHECK-LABEL: redundant_stores_merging:
-; CHECK:   movabsq $528280977409, %rax
-; CHECK:   movq    %rax, e+4(%rip)
-; CHECK:   movl    $456, e+8(%rip)
+; CHECK: movabsq $1958505086977, %rax
+; CHECK: movq    %rax, e+4(%rip)
 define void @redundant_stores_merging() {
 entry:
   store i32 1, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 1), align 4
   store i32 123, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 2), align 4
   store i32 456, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 2), align 4
   ret void
 }
 
 ;; This variant tests PR25154.
 ; CHECK-LABEL: redundant_stores_merging_reverse:
-; CHECK:   movabsq $528280977409, %rax
-; CHECK:   movq    %rax, e+4(%rip)
-; CHECK:   movl    $456, e+8(%rip)
+; CHECK: movabsq $1958505086977, %rax
+; CHECK: movq  %rax, e+4(%rip)
 define void @redundant_stores_merging_reverse() {
 entry:
   store i32 123, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 2), align 4
   store i32 456, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 2), align 4
   store i32 1, i32* getelementptr inbounds (%structTy, %structTy* @e, i64 0, i32 1), align 4
   ret void
 }
 

test/CodeGen/X86/vector-shuffle-256-v4.ll

 ; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 0, i32 1, i32 4, i32 2>
   ret <4 x i64> %shuffle
 }
 
 define <4 x i64> @shuffle_v4i64_0412(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_0412:
 ; AVX1:       # BB#0:
+; AVX1-NEXT:    vmovddup {{.*#+}} xmm1 = xmm1[0,0]
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpalignr {{.*#+}} xmm2 = xmm0[8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vmovddup {{.*#+}} xmm1 = xmm1[0,0]
 ; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4i64_0412:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,2]
 ; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]

test/CodeGen/X86/wide-integer-cmp.ll

 ; CHECK-NEXT:  .Lcfi1:
 ; CHECK-NEXT:    .cfi_offset %esi, -8
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; CHECK-NEXT:    cmpl {{[0-9]+}}(%esp), %edx
 ; CHECK-NEXT:    sbbl {{[0-9]+}}(%esp), %esi
-; CHECK-NEXT:    sbbl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    sbbl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT:    sbbl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    jge .LBB4_2
 ; CHECK-NEXT:  # BB#1: # %bb1
 ; CHECK-NEXT:    movl $1, %eax
 ; CHECK-NEXT:    popl %esi
 ; CHECK-NEXT:    retl
 ; CHECK-NEXT:  .LBB4_2: # %bb2
 ; CHECK-NEXT:    movl $2, %eax
 ; CHECK-NEXT:    popl %esi

test/CodeGen/X86/widen_arith-6.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.2 | FileCheck %s
 
 ; widen a v3f32 to vfi32 to do a vector multiple and an add
 
 define void @update(<3 x float>* %dst, <3 x float>* %src, i32 %n) nounwind {
 ; CHECK-LABEL: update:
 ; CHECK:       # BB#0: # %entry
 ; CHECK-NEXT:    pushl %ebp
 ; CHECK-NEXT:    movl %esp, %ebp
 ; CHECK-NEXT:    andl $-16, %esp
 ; CHECK-NEXT:    subl $48, %esp
+; CHECK-NEXT:    movl $1065353216, {{[0-9]+}}(%esp) # imm = 0x3F800000
 ; CHECK-NEXT:    movl $1077936128, {{[0-9]+}}(%esp) # imm = 0x40400000
 ; CHECK-NEXT:    movl $1073741824, {{[0-9]+}}(%esp) # imm = 0x40000000
-; CHECK-NEXT:    movl $1065353216, {{[0-9]+}}(%esp) # imm = 0x3F800000
 ; CHECK-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movaps {{.*#+}} xmm0 = <1976.04004,1976.04004,1976.04004,u>
 ; CHECK-NEXT:    jmp .LBB0_1
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB0_2: # %forbody
 ; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl 8(%ebp), %ecx