DAG: Avoid OOB when legalizing vector indexing

[arsenm]

Summary

If a vector index is out of bounds, the result is supposed to be
undefined but is not undefined behavior. Change the legalization
for indexing the vector on the stack so that an out of bounds
index does not create an out of bounds memory access.

[efriedma]

This approach looks right.

lib/CodeGen/SelectionDAG/TargetLowering.cpp

3714

I'm pretty sure this special-case isn't safe; just because it's a constant doesn't mean it's in range.

3726

It would be nice if we could round the number of elements up to a power of two... I guess it's not important.

[arsenm]

lib/CodeGen/SelectionDAG/TargetLowering.cpp

3714

I was wondering about this, however I think it's OK. getNode for EXTRACT_VECTOR_ELT turns this into undef for out of bounds. For INSERT_VECTOR_ELT it seems to not, but I also haven't been able to come up with a testcase where this gets hit

3726

This should probably be an assert for now since I think this only happens after vector type legalization. When 3-vectors are eventually added this will need to be handled

[efriedma]

lib/CodeGen/SelectionDAG/TargetLowering.cpp

3714

If getNode() is actually enforcing the restriction, fine... otherwise, we're likely to run into issues which are extremely difficult to reproduce because the constant is getting produced by splitting an integer in legalization or something like that.

[arsenm]

lib/CodeGen/SelectionDAG/TargetLowering.cpp

3714

Should I fix getNode not doing this for insert_vector_elt to match instead? It seems broken to me that it would for one but not the other

[efriedma]

lib/CodeGen/SelectionDAG/TargetLowering.cpp

3714

Actually, thinking about it a bit more, I would prefer not to leave this issue around as a trap if we start using getVectorElementPointer outside of insert/extract legalization, where we might not enforce a "constants are in range" rule. The check isn't doing anything useful anyway.

(Of course, fixing getNode would be fine.)

[arsenm]

Rebase

[arsenm]

Rebase

[arsenm]

Regenerate x86 changed test

[efriedma]

LGTM.

[arsenm]

r291604

include/llvm/Target/TargetLowering.h

public:
   /// the stored elements. This number is equal to the number of '1's bits
   /// in the \p Mask.
   /// \p DataVT is a vector type. \p Mask is a vector value.
   /// \p DataVT and \p Mask have the same number of vector elements.
   SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL,
                                  EVT DataVT, SelectionDAG &DAG,
                                  bool IsCompressedMemory) const;
 
+  /// Get a pointer to vector element \p Idx located in memory for a vector of
+  /// type \p VecVT starting at a base address of \p VecPtr. If \p Idx is out of
+  /// bounds the returned pointer is unspecified, but will be within the vector
+  /// bounds.
+  SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT,
+                                  SDValue Idx) const;
+
   //===--------------------------------------------------------------------===//
   // Instruction Emitting Hooks
   //
 
   /// This method should be implemented by targets that mark instructions with
   /// the 'usesCustomInserter' flag.  These instructions are special in various
   /// ways, which require special support to insert.  The specified MachineInstr
   /// is created but not inserted into any basic blocks, and this method is

lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
   // If the target doesn't support this, we have to spill the input vector
   // to a temporary stack slot, update the element, then reload it.  This is
   // badness.  We could also load the value into a vector register (either
   // with a "move to register" or "extload into register" instruction, then
   // permute it into place, if the idx is a constant and if the idx is
   // supported by the target.
   EVT VT    = Tmp1.getValueType();
   EVT EltVT = VT.getVectorElementType();
-  EVT IdxVT = Tmp3.getValueType();
-  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
   SDValue StackPtr = DAG.CreateStackTemporary(VT);
 
   int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
 
   // Store the vector.
   SDValue Ch = DAG.getStore(
       DAG.getEntryNode(), dl, Tmp1, StackPtr,
       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
 
-  // Truncate or zero extend offset to target pointer type.
-  Tmp3 = DAG.getZExtOrTrunc(Tmp3, dl, PtrVT);
-  // Add the offset to the index.
-  unsigned EltSize = EltVT.getSizeInBits()/8;
-  Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,
-                     DAG.getConstant(EltSize, dl, IdxVT));
-  SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr);
+  SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
+
   // Store the scalar value.
   Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
   // Load the updated vector.
   return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
                                                DAG.getMachineFunction(), SPFI));
 }
 
 SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
         continue;
 
       StackPtr = ST->getBasePtr();
       Ch = SDValue(ST, 0);
       break;
     }
   }
 
+  EVT VecVT = Vec.getValueType();
+
   if (!Ch.getNode()) {
     // Store the value to a temporary stack slot, then LOAD the returned part.
-    StackPtr = DAG.CreateStackTemporary(Vec.getValueType());
+    StackPtr = DAG.CreateStackTemporary(VecVT);
     Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
                       MachinePointerInfo());
   }
 
-  // Add the offset to the index.
-  unsigned EltSize = Vec.getScalarValueSizeInBits() / 8;
-  Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx,
-                    DAG.getConstant(EltSize, SDLoc(Vec), Idx.getValueType()));
-
-  Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy(DAG.getDataLayout()));
-  StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr);
+  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
 
   SDValue NewLoad;
 
   if (Op.getValueType().isVector())
     NewLoad =
         DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
   else
     NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
                              MachinePointerInfo(),
-                             Vec.getValueType().getVectorElementType());
+                             VecVT.getVectorElementType());
 
   // Replace the chain going out of the store, by the one out of the load.
   DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
 
   // We introduced a cycle though, so update the loads operands, making sure
   // to use the original store's chain as an incoming chain.
   SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
                                           NewLoad->op_end());
   NewLoadOperands[0] = Ch;
   NewLoad =
       SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
   return NewLoad;
 }
 
 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
   assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
 
   SDValue Vec  = Op.getOperand(0);
   SDValue Part = Op.getOperand(1);
   SDValue Idx  = Op.getOperand(2);
   SDLoc dl(Op);
 
   // Store the value to a temporary stack slot, then LOAD the returned part.
-
-  SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType());
+  EVT VecVT = Vec.getValueType();
+  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
   int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
   MachinePointerInfo PtrInfo =
       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
 
   // First store the whole vector.
   SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
 
   // Then store the inserted part.
-
-  // Add the offset to the index.
-  unsigned EltSize = Vec.getScalarValueSizeInBits() / 8;
-
-  Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx,
-                    DAG.getConstant(EltSize, SDLoc(Vec), Idx.getValueType()));
-  Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy(DAG.getDataLayout()));
-
-  SDValue SubStackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
-                                    StackPtr);
+  SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
 
   // Store the subvector.
   Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
 
   // Finally, load the updated vector.
   return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
 }
 

lib/CodeGen/SelectionDAG/LegalizeTypes.h

private:
   bool CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult);
   bool CustomWidenLowerNode(SDNode *N, EVT VT);
 
   /// Replace each result of the given MERGE_VALUES node with the corresponding
   /// input operand, except for the result 'ResNo', for which the corresponding
   /// input operand is returned.
   SDValue DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo);
 
-  SDValue GetVectorElementPointer(SDValue VecPtr, EVT EltVT, SDValue Index);
   SDValue JoinIntegers(SDValue Lo, SDValue Hi);
   SDValue LibCallify(RTLIB::Libcall LC, SDNode *N, bool isSigned);
 
   std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
                                                  SDNode *Node, bool isSigned);
   std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node);
 
   SDValue PromoteTargetBoolean(SDValue Bool, EVT ValVT);

lib/CodeGen/SelectionDAG/LegalizeTypes.cpp

void DAGTypeLegalizer::GetPairElements(SDValue Pair,
   SDLoc dl(Pair);
   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Pair.getValueType());
   Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, NVT, Pair,
                    DAG.getIntPtrConstant(0, dl));
   Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, NVT, Pair,
                    DAG.getIntPtrConstant(1, dl));
 }
 
-SDValue DAGTypeLegalizer::GetVectorElementPointer(SDValue VecPtr, EVT EltVT,
-                                                  SDValue Index) {
-  SDLoc dl(Index);
-  // Make sure the index type is big enough to compute in.
-  Index = DAG.getZExtOrTrunc(Index, dl, TLI.getPointerTy(DAG.getDataLayout()));
-
-  // Calculate the element offset and add it to the pointer.
-  unsigned EltSize = EltVT.getSizeInBits() / 8; // FIXME: should be ABI size.
-  assert(EltSize * 8 == EltVT.getSizeInBits() &&
-         "Converting bits to bytes lost precision");
-
-  Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
-                      DAG.getConstant(EltSize, dl, Index.getValueType()));
-  return DAG.getNode(ISD::ADD, dl, Index.getValueType(), Index, VecPtr);
-}
-
 /// Build an integer with low bits Lo and high bits Hi.
 SDValue DAGTypeLegalizer::JoinIntegers(SDValue Lo, SDValue Hi) {
   // Arbitrarily use dlHi for result SDLoc
   SDLoc dlHi(Hi);
   SDLoc dlLo(Lo);
   EVT LVT = Lo.getValueType();
   EVT HVT = Hi.getValueType();
   EVT NVT = EVT::getIntegerVT(*DAG.getContext(),

lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
                                                     SDValue &Hi) {
   SDValue Vec = N->getOperand(0);
   SDValue SubVec = N->getOperand(1);
   SDValue Idx = N->getOperand(2);
   SDLoc dl(N);
   GetSplitVector(Vec, Lo, Hi);
 
   EVT VecVT = Vec.getValueType();
-  EVT VecElemVT = VecVT.getVectorElementType();
   unsigned VecElems = VecVT.getVectorNumElements();
   unsigned SubElems = SubVec.getValueType().getVectorNumElements();
 
   // If we know the index is 0, and we know the subvector doesn't cross the
   // boundary between the halves, we can avoid spilling the vector, and insert
   // into the lower half of the split vector directly.
   // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
   // the index is constant and there is no boundary crossing. But those cases
void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
   }
 
   // Spill the vector to the stack.
   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
   SDValue Store =
       DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
 
   // Store the new subvector into the specified index.
-  SDValue SubVecPtr = GetVectorElementPointer(StackPtr, VecElemVT, Idx);
+  SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
   Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
   unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
   Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
 
   // Load the Lo part from the stack slot.
   Lo =
       DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
 
void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
   EVT VecVT = Vec.getValueType();
   EVT EltVT = VecVT.getVectorElementType();
   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
   SDValue Store =
       DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
 
   // Store the new element.  This may be larger than the vector element type,
   // so use a truncating store.
-  SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx);
+  SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
   Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
   unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
   Store =
       DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT);
 
   // Load the Lo part from the stack slot.
   Lo =
       DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
   }
 
   // Store the vector to the stack.
   SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
   SDValue Store =
       DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
 
   // Load back the required element.
-  StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx);
+  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
   return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
                         MachinePointerInfo(), EltVT);
 }
 
 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
                                              unsigned OpNo) {
   EVT LoVT, HiVT;
   SDLoc dl(MGT);

lib/CodeGen/SelectionDAG/TargetLowering.cpp

Store2 = DAG.getTruncStore(
       ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT, Alignment,
       ST->getMemOperand()->getFlags(), ST->getAAInfo());
 
   SDValue Result =
     DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
   return Result;
 }
 
-SDValue 
+SDValue
 TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,
                                        const SDLoc &DL, EVT DataVT,
                                        SelectionDAG &DAG,
                                        bool IsCompressedMemory) const {
   SDValue Increment;
   EVT AddrVT = Addr.getValueType();
   EVT MaskVT = Mask.getValueType();
   assert(DataVT.getVectorNumElements() == MaskVT.getVectorNumElements() &&
          "Incompatible types of Data and Mask");
   if (IsCompressedMemory) {
     // Incrementing the pointer according to number of '1's in the mask.
     EVT MaskIntVT = EVT::getIntegerVT(*DAG.getContext(), MaskVT.getSizeInBits());
     SDValue MaskInIntReg = DAG.getBitcast(MaskIntVT, Mask);
     if (MaskIntVT.getSizeInBits() < 32) {
       MaskInIntReg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg);
       MaskIntVT = MVT::i32;
     }
 
     // Count '1's with POPCNT.
     Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg);
     Increment = DAG.getZExtOrTrunc(Increment, DL, AddrVT);
     // Scale is an element size in bytes.
     SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL,
                                     AddrVT);
     Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale);
   } else
     Increment = DAG.getConstant(DataVT.getSizeInBits() / 8, DL, AddrVT);
 
   return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment);
 }
 
+static SDValue clampDynamicVectorIndex(SelectionDAG &DAG,
+                                       SDValue Idx,
+                                       EVT VecVT,
+                                       const SDLoc &dl) {
+  if (isa<ConstantSDNode>(Idx))
+    return Idx;
+
+  EVT IdxVT = Idx.getValueType();
+  unsigned NElts = VecVT.getVectorNumElements();
+  if (isPowerOf2_32(NElts)) {
+    APInt Imm = APInt::getLowBitsSet(IdxVT.getSizeInBits(),
+                                     Log2_32(NElts));
+    return DAG.getNode(ISD::AND, dl, IdxVT, Idx,
+                       DAG.getConstant(Imm, dl, IdxVT));
+  }
+
+  return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx,
+                     DAG.getConstant(NElts - 1, dl, IdxVT));
+}
+
+SDValue TargetLowering::getVectorElementPointer(SelectionDAG &DAG,
+                                                SDValue VecPtr, EVT VecVT,
+                                                SDValue Index) const {
+  SDLoc dl(Index);
+  // Make sure the index type is big enough to compute in.
+  Index = DAG.getZExtOrTrunc(Index, dl, getPointerTy(DAG.getDataLayout()));
+
+  EVT EltVT = VecVT.getVectorElementType();
+
+  // Calculate the element offset and add it to the pointer.
+  unsigned EltSize = EltVT.getSizeInBits() / 8; // FIXME: should be ABI size.
+  assert(EltSize * 8 == EltVT.getSizeInBits() &&
+         "Converting bits to bytes lost precision");
+
+  Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl);
+
+  EVT IdxVT = Index.getValueType();
+
+  Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index,
+                      DAG.getConstant(EltSize, dl, IdxVT));
+  return DAG.getNode(ISD::ADD, dl, IdxVT, Index, VecPtr);
+}
+
 //===----------------------------------------------------------------------===//
 // Implementation of Emulated TLS Model
 //===----------------------------------------------------------------------===//
 
 SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
                                                 SelectionDAG &DAG) const {
   // Access to address of TLS varialbe xyz is lowered to a function call:
   //   __emutls_get_address( address of global variable named "__emutls_v.xyz" )

test/CodeGen/AArch64/arm64-neon-copy.ll

; CHECK: ret
   %vecinit12 = insertelement <8 x i8> %vecinit10, i8 %vecext11, i32 6
   %vecext13 = extractelement <16 x i8> %x, i32 7
   %vecinit14 = insertelement <8 x i8> %vecinit12, i8 %vecext13, i32 7
   ret <8 x i8> %vecinit14
 }
 
 ; CHECK-LABEL: test_extracts_inserts_varidx_extract:
 ; CHECK: str q0
-; CHECK: add x[[PTR:[0-9]+]], {{.*}}, w0, sxtw #1
-; CHECK-DAG: ld1 { v[[R:[0-9]+]].h }[0], [x[[PTR]]]
+; CHECK-DAG: and [[MASKED_IDX:x[0-9]+]], x0, #0x7
+; CHECK: bfi [[PTR:x[0-9]+]], [[MASKED_IDX]], #1, #3
+; CHECK-DAG: ld1 { v[[R:[0-9]+]].h }[0], {{\[}}[[PTR]]{{\]}}
 ; CHECK-DAG: ins v[[R]].h[1], v0.h[1]
 ; CHECK-DAG: ins v[[R]].h[2], v0.h[2]
 ; CHECK-DAG: ins v[[R]].h[3], v0.h[3]
 define <4 x i16> @test_extracts_inserts_varidx_extract(<8 x i16> %x, i32 %idx) {
   %tmp = extractelement <8 x i16> %x, i32 %idx
   %tmp2 = insertelement <4 x i16> undef, i16 %tmp, i32 0
   %tmp3 = extractelement <8 x i16> %x, i32 1
   %tmp4 = insertelement <4 x i16> %tmp2, i16 %tmp3, i32 1
   %tmp5 = extractelement <8 x i16> %x, i32 2
   %tmp6 = insertelement <4 x i16> %tmp4, i16 %tmp5, i32 2
   %tmp7 = extractelement <8 x i16> %x, i32 3
   %tmp8 = insertelement <4 x i16> %tmp6, i16 %tmp7, i32 3
   ret <4 x i16> %tmp8
 }
 
 ; CHECK-LABEL: test_extracts_inserts_varidx_insert:
-; CHECK: str h0, [{{.*}}, w0, sxtw #1]
+; CHECK: and [[MASKED_IDX:x[0-9]+]], x0, #0x3
+; CHECK: bfi x9, [[MASKED_IDX]], #1, #2
+; CHECK: st1 { v0.h }[0], [x9]
 ; CHECK-DAG: ldr d[[R:[0-9]+]]
 ; CHECK-DAG: ins v[[R]].h[1], v0.h[1]
 ; CHECK-DAG: ins v[[R]].h[2], v0.h[2]
 ; CHECK-DAG: ins v[[R]].h[3], v0.h[3]
 define <4 x i16> @test_extracts_inserts_varidx_insert(<8 x i16> %x, i32 %idx) {
   %tmp = extractelement <8 x i16> %x, i32 0
   %tmp2 = insertelement <4 x i16> undef, i16 %tmp, i32 %idx
   %tmp3 = extractelement <8 x i16> %x, i32 1

test/CodeGen/AArch64/arm64-nvcast.ll

 ; RUN: llc < %s -mtriple=arm64-apple-ios | FileCheck %s
 
 ; CHECK-LABEL: _test:
-; CHECK:  fmov.2d v0, #2.00000000
-; CHECK:  str  q0, [sp, #-16]!
-; CHECK:  mov  x8, sp
-; CHECK:  ldr s0, [x8, w1, sxtw #2]
+; CHECK-DAG:  fmov.2d v0, #2.00000000
+; CHECK-DAG: and [[MASK_IDX:x[0-9]+]], x1, #0x3
+; CHECK-DAG:  mov  x9, sp
+; CHECK-DAG:  str  q0, [sp], #16
+; CHECK-DAG:  bfi [[PTR:x[0-9]+]], [[MASK_IDX]], #2, #2
+; CHECK:  ldr s0, {{\[}}[[PTR]]{{\]}}
 ; CHECK:  str  s0, [x0]
 
 define void @test(float * %p1, i32 %v1) {
 entry:
   %v2 = extractelement <3 x float> <float 0.000000e+00, float 2.000000e+00, float 0.000000e+00>, i32 %v1
   store float %v2, float* %p1, align 4
   ret void
 }
 
 ; CHECK-LABEL: _test2
 ; CHECK: movi.16b  v0, #63
-; CHECK: str  q0, [sp, #-16]!
-; CHECK: mov  x8, sp
-; CHECK: ldr s0, [x8, w1, sxtw #2]
+; CHECK-DAG: and [[MASK_IDX:x[0-9]+]], x1, #0x3
+; CHECK-DAG: str  q0, [sp], #16
+; CHECK-DAG: mov  x9, sp
+; CHECK-DAG:  bfi [[PTR:x[0-9]+]], [[MASK_IDX]], #2, #2
+; CHECK: ldr s0, {{\[}}[[PTR]]{{\]}}
 ; CHECK: str  s0, [x0]
 
 define void @test2(float * %p1, i32 %v1) {
 entry:
   %v2 = extractelement <3 x float> <float 0.7470588088035583, float 0.7470588088035583, float 0.7470588088035583>, i32 %v1
   store float %v2, float* %p1, align 4
   ret void
 }

test/CodeGen/AMDGPU/insert_vector_elt.ll

 }
 
 ; GCN-LABEL: {{^}}dynamic_insertelement_v4i16:
 ; GCN: buffer_load_ushort v{{[0-9]+}}, off
 ; GCN: buffer_load_ushort v{{[0-9]+}}, off
 ; GCN: buffer_load_ushort v{{[0-9]+}}, off
 ; GCN: buffer_load_ushort v{{[0-9]+}}, off
 
+; GCN-DAG: v_mov_b32_e32 [[BASE_FI:v[0-9]+]], 0{{$}}
+; GCN-DAG: s_and_b32 [[MASK_IDX:s[0-9]+]], s{{[0-9]+}}, 3{{$}}
+; GCN-DAG: v_or_b32_e32 [[IDX:v[0-9]+]], [[MASK_IDX]], [[BASE_FI]]{{$}}
+
 ; GCN-DAG: buffer_store_short v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:6
 ; GCN-DAG: buffer_store_short v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:4
 ; GCN-DAG: buffer_store_short v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:2
-; GCN-DAG: buffer_store_short v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}{{$}}
-; GCN: buffer_store_short v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}}
+; GCN-DAG: buffer_store_short v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+$}}
+; GCN: buffer_store_short v{{[0-9]+}}, [[IDX]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}}
 
 ; GCN: s_waitcnt
 
 ; GCN: buffer_load_ushort
 ; GCN: buffer_load_ushort
 ; GCN: buffer_load_ushort
 ; GCN: buffer_load_ushort
 

test/CodeGen/AMDGPU/local-stack-slot-bug.ll

 ; RUN: llc -march=amdgcn -mcpu=verde -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck %s
 
 ; This used to fail due to a v_add_i32 instruction with an illegal immediate
 ; operand that was created during Local Stack Slot Allocation. Test case derived
 ; from https://bugs.freedesktop.org/show_bug.cgi?id=96602
 ;
 ; CHECK-LABEL: {{^}}main:
 
+; CHECK-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x200
+; CHECK-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
 ; CHECK-DAG: v_lshlrev_b32_e32 [[BYTES:v[0-9]+]], 2, v0
-; CHECK-DAG: v_add_i32_e32 [[HI_OFF:v[0-9]+]], vcc, 0x200, [[BYTES]]
+; CHECK-DAG: v_and_b32_e32 [[CLAMP_IDX:v[0-9]+]], 0x1fc, [[BYTES]]
 
 ; TODO: add 0?
-; CHECK-DAG: v_add_i32_e32 [[LO_OFF:v[0-9]+]], vcc, 0, [[BYTES]]
+; CHECK-DAG: v_or_b32_e32 [[LO_OFF:v[0-9]+]], [[CLAMP_IDX]], [[ZERO]]
+; CHECK-DAG: v_or_b32_e32 [[HI_OFF:v[0-9]+]], [[CLAMP_IDX]], [[K]]
 
 ; CHECK: buffer_load_dword {{v[0-9]+}}, [[LO_OFF]], {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen
 ; CHECK: buffer_load_dword {{v[0-9]+}}, [[HI_OFF]], {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen
 define amdgpu_ps float @main(i32 %idx) {
 main_body:
   %v1 = extractelement <81 x float> <float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0xBFEA477C60000000, float 0xBFEBE5DC60000000, float 0xBFEC71C720000000, float 0xBFEBE5DC60000000, float 0xBFEA477C60000000, float 0xBFE7A693C0000000, float 0xBFE41CFEA0000000, float 0x3FDF9B13E0000000, float 0x3FDF9B1380000000, float 0x3FD5C53B80000000, float 0x3FD5C53B00000000, float 0x3FC6326AC0000000, float 0x3FC63269E0000000, float 0xBEE05CEB00000000, float 0xBEE086A320000000, float 0xBFC63269E0000000, float 0xBFC6326AC0000000, float 0xBFD5C53B80000000, float 0xBFD5C53B80000000, float 0xBFDF9B13E0000000, float 0xBFDF9B1460000000, float 0xBFE41CFE80000000, float 0x3FE7A693C0000000, float 0x3FEA477C20000000, float 0x3FEBE5DC40000000, float 0x3FEC71C6E0000000, float 0x3FEBE5DC40000000, float 0x3FEA477C20000000, float 0x3FE7A693C0000000, float 0xBFE41CFE80000000>, i32 %idx
   %v2 = extractelement <81 x float> <float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float undef, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFEA0000000, float 0xBFE7A693C0000000, float 0x3FE7A693C0000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFEBE5DC40000000, float 0x3FEBE5DC40000000, float 0xBFEC71C720000000, float 0x3FEC71C6E0000000, float 0xBFEBE5DC60000000, float 0x3FEBE5DC40000000, float 0xBFEA477C20000000, float 0x3FEA477C20000000, float 0xBFE7A693C0000000, float 0x3FE7A69380000000, float 0xBFE41CFEA0000000, float 0xBFDF9B13E0000000, float 0xBFD5C53B80000000, float 0xBFC6326AC0000000, float 0x3EE0789320000000, float 0x3FC6326AC0000000, float 0x3FD5C53B80000000, float 0x3FDF9B13E0000000, float 0x3FE41CFE80000000>, i32 %idx
   %r = fadd float %v1, %v2
   ret float %r
 }

test/CodeGen/ARM/fp16-promote.ll

 ; CHECK-ALL: ldrh
 ; CHECK-ALL: strh
 ; CHECK-ALL: ldrh
 ; CHECK-ALL: strh
 ; CHECK-ALL: ldrh
 ; CHECK-ALL: strh
 ; CHECK-ALL: mov
 ; CHECK-ALL-DAG: ldrh
-; CHECK-ALL-DAG: add
+; CHECK-ALL-DAG: orr
 ; CHECK-ALL: strh
 ; CHECK-ALL: ldrh
 ; CHECK-ALL: strh
 ; CHECK-ALL: ldrh
 ; CHECK-ALL: strh
 ; CHECK-ALL: ldrh
 ; CHECK-ALL: strh
 ; CHECK-ALL: ldrh

 ; CHECK-VFP: ldrh
 ; CHECK-VFP: orr
 ; CHECK-VFP: str
 ; CHECK-VFP: ldrh
 ; CHECK-VFP: ldrh
 ; CHECK-VFP: orr
 ; CHECK-VFP: str
 ; CHECK-VFP: mov
-; CHECK-VFP: add
+; CHECK-VFP: orr
 ; CHECK-VFP: ldrh
 ; CHECK-VFP: strh
 ; CHECK-VFP: add sp, sp, #8
 ; CHECK-VFP: bx lr
 ; CHECK-NOVFP: ldrh
 ; CHECK-NOVFP: strh
 ; CHECK-NOVFP: ldrh
 ; CHECK-NOVFP: strh

test/CodeGen/ARM/vdup.ll

 ; Also make sure we don't barf on variable-index extractelts, where we almost
 ; could have generated a vdup.
 
 define <8 x i8> @check_i8_varidx(<16 x i8> %v, i32 %idx) {
 ; CHECK-LABEL: check_i8_varidx:
 ; CHECK: mov r[[FP:[0-9]+]], sp
 ; CHECK: ldr r[[IDX:[0-9]+]], [r[[FP]], #4]
 ; CHECK: mov r[[SPCOPY:[0-9]+]], sp
-; CHECK: vst1.64 {d{{.*}}, d{{.*}}}, [r[[SPCOPY]]:128], r[[IDX]]
+; CHECK: and r[[MASKED_IDX:[0-9]+]], r[[IDX]], #15
+; CHECK: vst1.64 {d{{.*}}, d{{.*}}}, [r[[SPCOPY]]:128], r[[MASKED_IDX]]
 ; CHECK: vld1.8 {d{{.*}}[]}, [r[[SPCOPY]]]
   %x = extractelement <16 x i8> %v, i32 %idx
   %1 = insertelement  <8 x i8> undef, i8 %x, i32 0
   %2 = insertelement  <8 x i8> %1, i8 %x, i32 1
   ret <8 x i8> %2
 }

test/CodeGen/Mips/llvm-ir/extractelement.ll

 ; RUN: llc < %s -march=mips -mcpu=mips2 | FileCheck %s -check-prefix=ALL
 
 ; This test triggered a bug in the vector splitting where the type legalizer
 ; attempted to extract the element with by storing the vector, then reading
 ; an element back. However, the address calculation was:
 ;   Base + Index * (EltSizeInBits / 8)
 ; and EltSizeInBits was 1. This caused the index to be forgotten.
 define i1 @via_stack_bug(i8 signext %idx) {
   %1 = extractelement <2 x i1> <i1 false, i1 true>, i8 %idx
   ret i1 %1
 }
 
 ; ALL-LABEL: via_stack_bug:
 ; ALL-DAG:       addiu  [[ONE:\$[0-9]+]], $zero, 1
 ; ALL-DAG:       sb     [[ONE]], 7($sp)
 ; ALL-DAG:       sb     $zero, 6($sp)
+; ALL-DAG:       andi   [[MASKED_IDX:\$[0-9]+]], $4, 1
 ; ALL-DAG:       addiu  [[VPTR:\$[0-9]+]], $sp, 6
-; ALL-DAG:       addu   [[EPTR:\$[0-9]+]], $4, [[VPTR]]
+; ALL-DAG:       or   [[EPTR:\$[0-9]+]], [[MASKED_IDX]], [[VPTR]]
 ; ALL:           lbu    $2, 0([[EPTR]])

test/CodeGen/PowerPC/variable_elem_vec_extracts.ll

 ; CHECK-DAG: lvsl [[SHMSKREG:[0-9]+]], 0, [[SHIFTREG]]
 ; CHECK-DAG: vperm [[PERMVEC:[0-9]+]], 2, 2, [[SHMSKREG]]
 ; CHECK-DAG: li [[ONEREG:[0-9]+]], 1
 ; CHECK-DAG: and [[ELEMSREG:[0-9]+]], [[ONEREG]], 5
 ; CHECK-DAG: sldi [[SHAMREG:[0-9]+]], [[ELEMSREG]], 5
 ; CHECK: mfvsrd [[TOGPR:[0-9]+]],
 ; CHECK: srd [[RSHREG:[0-9]+]], [[TOGPR]], [[SHAMREG]]
 ; CHECK: extsw 3, [[RSHREG]]
-; CHECK-P7-DAG: sldi [[ELEMOFFREG:[0-9]+]], 5, 2
+; CHECK-P7-DAG: rlwinm [[ELEMOFFREG:[0-9]+]], 5, 2, 28, 29
 ; CHECK-P7-DAG: stxvw4x 34,
 ; CHECK-P7: lwax 3, [[ELEMOFFREG]],
 ; CHECK-BE-DAG: andi. [[ANDREG:[0-9]+]], 5, 2
 ; CHECK-BE-DAG: sldi [[SLREG:[0-9]+]], [[ANDREG]], 2
 ; CHECK-BE-DAG: lvsl [[SHMSKREG:[0-9]+]], 0, [[SLREG]]
 ; CHECK-BE-DAG: vperm {{[0-9]+}}, 2, 2, [[SHMSKREG]]
 ; CHECK-BE-DAG: li [[IMMREG:[0-9]+]], 1
 ; CHECK-BE-DAG: andc [[ANDCREG:[0-9]+]], [[IMMREG]], 5

 ; CHECK-P7-LABEL: @getl
 ; CHECK-BE-LABEL: @getl
 ; CHECK-DAG: li [[TRUNCREG:[0-9]+]], 1
 ; CHECK-DAG: andc [[MASKREG:[0-9]+]], [[TRUNCREG]], 5
 ; CHECK-DAG: sldi [[SHIFTREG:[0-9]+]], [[MASKREG]], 3
 ; CHECK-DAG: lvsl [[SHMSKREG:[0-9]+]], 0, [[SHIFTREG]]
 ; CHECK-DAG: vperm [[PERMVEC:[0-9]+]], 2, 2, [[SHMSKREG]]
 ; CHECK: mfvsrd 3,
-; CHECK-P7-DAG: sldi [[ELEMOFFREG:[0-9]+]], 5, 3
+; CHECK-P7-DAG: rlwinm [[ELEMOFFREG:[0-9]+]], 5, 3, 28, 28
 ; CHECK-P7-DAG: stxvd2x 34,
 ; CHECK-P7: ldx 3, [[ELEMOFFREG]],
 ; CHECK-BE-DAG: andi. [[ANDREG:[0-9]+]], 5, 1
 ; CHECK-BE-DAG: sldi [[SLREG:[0-9]+]], [[ANDREG]], 3
 ; CHECK-BE-DAG: lvsl [[SHMSKREG:[0-9]+]], 0, [[SLREG]]
 ; CHECK-BE-DAG: vperm {{[0-9]+}}, 2, 2, [[SHMSKREG]]
 ; CHECK-BE: mfvsrd 3,
 }
 
 ; Function Attrs: norecurse nounwind readnone
 define float @getf(<4 x float> %a, i32 signext %b) {
 entry:
   %vecext = extractelement <4 x float> %a, i32 %b
   ret float %vecext
 ; CHECK-LABEL: @getf
 ; CHECK-P7-LABEL: @getf
 ; CHECK-BE-LABEL: @getf
 ; CHECK: li [[IMMREG:[0-9]+]], 3
 ; CHECK: xor [[TRUNCREG:[0-9]+]], [[IMMREG]], 5
 ; CHECK: lvsl [[SHMSKREG:[0-9]+]], 0, [[TRUNCREG]]
 ; CHECK: vperm {{[0-9]+}}, 2, 2, [[SHMSKREG]]
 ; CHECK: xscvspdpn 1,
-; CHECK-P7-DAG: sldi [[ELEMOFFREG:[0-9]+]], 5, 2
+; CHECK-P7-DAG: rlwinm [[ELEMOFFREG:[0-9]+]], 5, 2, 28, 29
 ; CHECK-P7-DAG: stxvw4x 34,
 ; CHECK-P7: lfsx 1, [[ELEMOFFREG]],
 ; CHECK-BE: sldi [[ELNOREG:[0-9]+]], 5, 2
 ; CHECK-BE: lvsl [[SHMSKREG:[0-9]+]], 0, [[ELNOREG]]
 ; CHECK-BE: vperm {{[0-9]+}}, 2, 2, [[SHMSKREG]]
 ; CHECK-BE: xscvspdpn 1,
 }
 

test/CodeGen/X86/extractelement-index.ll

 
 ;
 ; ExtractElement - Variable Index
 ;
 
 define i8 @extractelement_v16i8_var(<16 x i8> %a, i256 %i) nounwind {
 ; SSE-LABEL: extractelement_v16i8_var:
 ; SSE:       # BB#0:
+; SSE-NEXT:    andl $15, %edi
 ; SSE-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
 ; SSE-NEXT:    leaq -{{[0-9]+}}(%rsp), %rax
 ; SSE-NEXT:    movb (%rdi,%rax), %al
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: extractelement_v16i8_var:
 ; AVX:       # BB#0:
+; AVX-NEXT:    andl $15, %edi
 ; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
 ; AVX-NEXT:    leaq -{{[0-9]+}}(%rsp), %rax
 ; AVX-NEXT:    movb (%rdi,%rax), %al
 ; AVX-NEXT:    retq
   %b = extractelement <16 x i8> %a, i256 %i
   ret i8 %b
 }
 
 define i8 @extractelement_v32i8_var(<32 x i8> %a, i256 %i) nounwind {
 ; SSE-LABEL: extractelement_v32i8_var:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    pushq %rbp
 ; SSE-NEXT:    movq %rsp, %rbp
 ; SSE-NEXT:    andq $-32, %rsp
 ; SSE-NEXT:    subq $64, %rsp
+; SSE-NEXT:    andl $31, %edi
 ; SSE-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
 ; SSE-NEXT:    movaps %xmm0, (%rsp)
 ; SSE-NEXT:    movq %rsp, %rax
 ; SSE-NEXT:    movb (%rdi,%rax), %al
 ; SSE-NEXT:    movq %rbp, %rsp
 ; SSE-NEXT:    popq %rbp
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: extractelement_v32i8_var:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    pushq %rbp
 ; AVX-NEXT:    movq %rsp, %rbp
 ; AVX-NEXT:    andq $-32, %rsp
 ; AVX-NEXT:    subq $64, %rsp
+; AVX-NEXT:    andl $31, %edi
 ; AVX-NEXT:    vmovaps %ymm0, (%rsp)
 ; AVX-NEXT:    movq %rsp, %rax
 ; AVX-NEXT:    movb (%rdi,%rax), %al
 ; AVX-NEXT:    movq %rbp, %rsp
 ; AVX-NEXT:    popq %rbp
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
   %b = extractelement <32 x i8> %a, i256 %i
   ret i8 %b
 }
 
 define i16 @extractelement_v8i16_var(<8 x i16> %a, i256 %i) nounwind {
 ; SSE-LABEL: extractelement_v8i16_var:
 ; SSE:       # BB#0:
+; SSE-NEXT:    andl $7, %edi
 ; SSE-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
 ; SSE-NEXT:    movzwl -24(%rsp,%rdi,2), %eax
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: extractelement_v8i16_var:
 ; AVX:       # BB#0:
+; AVX-NEXT:    andl $7, %edi
 ; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
 ; AVX-NEXT:    movzwl -24(%rsp,%rdi,2), %eax
 ; AVX-NEXT:    retq
   %b = extractelement <8 x i16> %a, i256 %i
   ret i16 %b
 }
 
 define i16 @extractelement_v16i16_var(<16 x i16> %a, i256 %i) nounwind {
 ; SSE-LABEL: extractelement_v16i16_var:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    pushq %rbp
 ; SSE-NEXT:    movq %rsp, %rbp
 ; SSE-NEXT:    andq $-32, %rsp
 ; SSE-NEXT:    subq $64, %rsp
+; SSE-NEXT:    andl $15, %edi
 ; SSE-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
 ; SSE-NEXT:    movaps %xmm0, (%rsp)
 ; SSE-NEXT:    movzwl (%rsp,%rdi,2), %eax
 ; SSE-NEXT:    movq %rbp, %rsp
 ; SSE-NEXT:    popq %rbp
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: extractelement_v16i16_var:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    pushq %rbp
 ; AVX-NEXT:    movq %rsp, %rbp
 ; AVX-NEXT:    andq $-32, %rsp
 ; AVX-NEXT:    subq $64, %rsp
+; AVX-NEXT:    andl $15, %edi
 ; AVX-NEXT:    vmovaps %ymm0, (%rsp)
 ; AVX-NEXT:    movzwl (%rsp,%rdi,2), %eax
 ; AVX-NEXT:    movq %rbp, %rsp
 ; AVX-NEXT:    popq %rbp
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
   %b = extractelement <16 x i16> %a, i256 %i
   ret i16 %b
 }
 
 define i32 @extractelement_v4i32_var(<4 x i32> %a, i256 %i) nounwind {
 ; SSE-LABEL: extractelement_v4i32_var:
 ; SSE:       # BB#0:
+; SSE-NEXT:    andl $3, %edi
 ; SSE-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
 ; SSE-NEXT:    movl -24(%rsp,%rdi,4), %eax
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: extractelement_v4i32_var:
 ; AVX:       # BB#0:
+; AVX-NEXT:    andl $3, %edi
 ; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
 ; AVX-NEXT:    movl -24(%rsp,%rdi,4), %eax
 ; AVX-NEXT:    retq
   %b = extractelement <4 x i32> %a, i256 %i
   ret i32 %b
 }
 
 define i32 @extractelement_v8i32_var(<8 x i32> %a, i256 %i) nounwind {
 ; SSE-LABEL: extractelement_v8i32_var:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    pushq %rbp
 ; SSE-NEXT:    movq %rsp, %rbp
 ; SSE-NEXT:    andq $-32, %rsp
 ; SSE-NEXT:    subq $64, %rsp
+; SSE-NEXT:    andl $7, %edi
 ; SSE-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
 ; SSE-NEXT:    movaps %xmm0, (%rsp)
 ; SSE-NEXT:    movl (%rsp,%rdi,4), %eax
 ; SSE-NEXT:    movq %rbp, %rsp
 ; SSE-NEXT:    popq %rbp
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: extractelement_v8i32_var:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    pushq %rbp
 ; AVX1-NEXT:    movq %rsp, %rbp
 ; AVX1-NEXT:    andq $-32, %rsp
 ; AVX1-NEXT:    subq $64, %rsp
+; AVX1-NEXT:    andl $7, %edi
 ; AVX1-NEXT:    vmovaps %ymm0, (%rsp)
 ; AVX1-NEXT:    movl (%rsp,%rdi,4), %eax
 ; AVX1-NEXT:    movq %rbp, %rsp
 ; AVX1-NEXT:    popq %rbp
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: extractelement_v8i32_var:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    vmovd %edi, %xmm1
 ; AVX2-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
   %b = extractelement <8 x i32> %a, i256 %i
   ret i32 %b
 }
 
 define i64 @extractelement_v2i64_var(<2 x i64> %a, i256 %i) nounwind {
 ; SSE-LABEL: extractelement_v2i64_var:
 ; SSE:       # BB#0:
+; SSE-NEXT:    andl $1, %edi
 ; SSE-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
 ; SSE-NEXT:    movq -24(%rsp,%rdi,8), %rax
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: extractelement_v2i64_var:
 ; AVX:       # BB#0:
+; AVX-NEXT:    andl $1, %edi
 ; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
 ; AVX-NEXT:    movq -24(%rsp,%rdi,8), %rax
 ; AVX-NEXT:    retq
   %b = extractelement <2 x i64> %a, i256 %i
   ret i64 %b
 }
 
 define i64 @extractelement_v4i64_var(<4 x i64> %a, i256 %i) nounwind {
 ; SSE-LABEL: extractelement_v4i64_var:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    pushq %rbp
 ; SSE-NEXT:    movq %rsp, %rbp
 ; SSE-NEXT:    andq $-32, %rsp
 ; SSE-NEXT:    subq $64, %rsp
+; SSE-NEXT:    andl $3, %edi
 ; SSE-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
 ; SSE-NEXT:    movaps %xmm0, (%rsp)
 ; SSE-NEXT:    movq (%rsp,%rdi,8), %rax
 ; SSE-NEXT:    movq %rbp, %rsp
 ; SSE-NEXT:    popq %rbp
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: extractelement_v4i64_var:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    pushq %rbp
 ; AVX-NEXT:    movq %rsp, %rbp
 ; AVX-NEXT:    andq $-32, %rsp
 ; AVX-NEXT:    subq $64, %rsp
+; AVX-NEXT:    andl $3, %edi
 ; AVX-NEXT:    vmovaps %ymm0, (%rsp)
 ; AVX-NEXT:    movq (%rsp,%rdi,8), %rax
 ; AVX-NEXT:    movq %rbp, %rsp
 ; AVX-NEXT:    popq %rbp
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
   %b = extractelement <4 x i64> %a, i256 %i
   ret i64 %b

test/CodeGen/X86/extractelement-legalization-store-ordering.ll

 ; CHECK-NEXT: pushl %ebx
 ; CHECK-NEXT: pushl %edi
 ; CHECK-NEXT: pushl %esi
 ; CHECK-NEXT: movl 16(%esp), %eax
 ; CHECK-NEXT: movl 24(%esp), %ecx
 ; CHECK-NEXT: movl 20(%esp), %edx
 ; CHECK-NEXT: paddd (%edx), %xmm0
 ; CHECK-NEXT: movdqa %xmm0, (%edx)
-; CHECK-NEXT: shll $4, %ecx
-; CHECK-NEXT: movl (%ecx,%edx), %esi
-; CHECK-NEXT: movl 12(%ecx,%edx), %edi
-; CHECK-NEXT: movl 8(%ecx,%edx), %ebx
-; CHECK-NEXT: movl 4(%ecx,%edx), %edx
+; CHECK-NEXT: movl (%edx), %esi
+; CHECK-NEXT: movl 12(%edx), %edi
+; CHECK-NEXT: movl 8(%edx), %ebx
+; CHECK-NEXT: movl 4(%edx), %edx
+; CHECK-NEXT: shll  $4, %ecx
 ; CHECK-NEXT: movl %esi, 12(%eax,%ecx)
 ; CHECK-NEXT: movl %edx, (%eax,%ecx)
 ; CHECK-NEXT: movl %ebx, 8(%eax,%ecx)
 ; CHECK-NEXT: movl %edi, 4(%eax,%ecx)
 ; CHECK-NEXT: popl %esi
 ; CHECK-NEXT: popl %edi
 ; CHECK-NEXT: popl %ebx
 ; CHECK-NEXT: retl

test/CodeGen/X86/i64-mem-copy.ll

; X32AVX-NEXT:    retl
   ret void
 }
 
 ; PR23476
 ; Handle extraction from a non-simple / pre-legalization type.
 
 define void @PR23476(<5 x i64> %in, i64* %out, i32 %index) {
 ; X32-LABEL: PR23476:
+; X32: andl $7, %eax
 ; X32:         movsd {{.*#+}} xmm0 = mem[0],zero
 ; X32:         movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    movsd %xmm0, (%eax)
+; X32-NEXT:    movsd %xmm0, (%ecx)
   %ext = extractelement <5 x i64> %in, i32 %index
   store i64 %ext, i64* %out, align 8
   ret void
 }

test/CodeGen/X86/vec_ins_extract-1.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i386-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=X32
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=X64
 
 ; Inserts and extracts with variable indices must be lowered
 ; to memory accesses.
 
 define i32 @t0(i32 inreg %t7, <4 x i32> inreg %t8) nounwind {
 ; X32-LABEL: t0:
 ; X32:       # BB#0:
 ; X32-NEXT:    pushl %ebp
 ; X32-NEXT:    movl %esp, %ebp
 ; X32-NEXT:    andl $-16, %esp
 ; X32-NEXT:    subl $32, %esp
+; X32-NEXT:    andl $3, %eax
 ; X32-NEXT:    movaps %xmm0, (%esp)
 ; X32-NEXT:    movl $76, (%esp,%eax,4)
 ; X32-NEXT:    movl (%esp), %eax
 ; X32-NEXT:    movl %ebp, %esp
 ; X32-NEXT:    popl %ebp
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: t0:
 ; X64:       # BB#0:
+; X64-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
 ; X64-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movslq %edi, %rax
-; X64-NEXT:    movl $76, -24(%rsp,%rax,4)
+; X64-NEXT:    andl $3, %edi
+; X64-NEXT:    movl $76, -24(%rsp,%rdi,4)
 ; X64-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
 ; X64-NEXT:    retq
   %t13 = insertelement <4 x i32> %t8, i32 76, i32 %t7
   %t9 = extractelement <4 x i32> %t13, i32 0
   ret i32 %t9
 }
 
 define i32 @t1(i32 inreg %t7, <4 x i32> inreg %t8) nounwind {
 ; X32-LABEL: t1:
 ; X32:       # BB#0:
 ; X32-NEXT:    pushl %ebp
 ; X32-NEXT:    movl %esp, %ebp
 ; X32-NEXT:    andl $-16, %esp
 ; X32-NEXT:    subl $32, %esp
+; X32-NEXT:    andl $3, %eax
 ; X32-NEXT:    movl $76, %ecx
 ; X32-NEXT:    pinsrd $0, %ecx, %xmm0
 ; X32-NEXT:    movdqa %xmm0, (%esp)
 ; X32-NEXT:    movl (%esp,%eax,4), %eax
 ; X32-NEXT:    movl %ebp, %esp
 ; X32-NEXT:    popl %ebp
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: t1:
 ; X64:       # BB#0:
+; X64-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
 ; X64-NEXT:    movl $76, %eax
 ; X64-NEXT:    pinsrd $0, %eax, %xmm0
 ; X64-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movslq %edi, %rax
-; X64-NEXT:    movl -24(%rsp,%rax,4), %eax
+; X64-NEXT:    andl $3, %edi
+; X64-NEXT:    movl -24(%rsp,%rdi,4), %eax
 ; X64-NEXT:    retq
   %t13 = insertelement <4 x i32> %t8, i32 76, i32 0
   %t9 = extractelement <4 x i32> %t13, i32 %t7
   ret i32 %t9
 }
 
 define <4 x i32> @t2(i32 inreg %t7, <4 x i32> inreg %t8) nounwind {
 ; X32-LABEL: t2:
 ; X32:       # BB#0:
 ; X32-NEXT:    pushl %ebp
 ; X32-NEXT:    movl %esp, %ebp
 ; X32-NEXT:    andl $-16, %esp
 ; X32-NEXT:    subl $32, %esp
+; X32-NEXT:    andl $3, %eax
 ; X32-NEXT:    movdqa %xmm0, (%esp)
 ; X32-NEXT:    pinsrd $0, (%esp,%eax,4), %xmm0
 ; X32-NEXT:    movl %ebp, %esp
 ; X32-NEXT:    popl %ebp
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: t2:
 ; X64:       # BB#0:
+; X64-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
 ; X64-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movslq %edi, %rax
-; X64-NEXT:    pinsrd $0, -24(%rsp,%rax,4), %xmm0
+; X64-NEXT:    andl $3, %edi
+; X64-NEXT:    pinsrd $0, -24(%rsp,%rdi,4), %xmm0
 ; X64-NEXT:    retq
   %t9 = extractelement <4 x i32> %t8, i32 %t7
   %t13 = insertelement <4 x i32> %t8, i32 %t9, i32 0
   ret <4 x i32> %t13
 }
 
 define <4 x i32> @t3(i32 inreg %t7, <4 x i32> inreg %t8) nounwind {
 ; X32-LABEL: t3:
 ; X32:       # BB#0:
 ; X32-NEXT:    pushl %ebp
 ; X32-NEXT:    movl %esp, %ebp
 ; X32-NEXT:    andl $-16, %esp
 ; X32-NEXT:    subl $32, %esp
+; X32-NEXT:    andl $3, %eax
 ; X32-NEXT:    movaps %xmm0, (%esp)
 ; X32-NEXT:    movss %xmm0, (%esp,%eax,4)
 ; X32-NEXT:    movaps (%esp), %xmm0
 ; X32-NEXT:    movl %ebp, %esp
 ; X32-NEXT:    popl %ebp
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: t3:
 ; X64:       # BB#0:
+; X64-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
 ; X64-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movslq %edi, %rax
-; X64-NEXT:    movss %xmm0, -24(%rsp,%rax,4)
+; X64-NEXT:    andl $3, %edi
+; X64-NEXT:    movss %xmm0, -24(%rsp,%rdi,4)
 ; X64-NEXT:    movaps -{{[0-9]+}}(%rsp), %xmm0
 ; X64-NEXT:    retq
   %t9 = extractelement <4 x i32> %t8, i32 0
   %t13 = insertelement <4 x i32> %t8, i32 %t9, i32 %t7
   ret <4 x i32> %t13
 }

test/CodeGen/X86/vec_insert-4.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-apple-darwin9.2.2 -mattr=+sse2,-sse4.1 | FileCheck %s --check-prefix=X32
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin9.2.2 -mattr=+sse2,-sse4.1 | FileCheck %s --check-prefix=X64
 
 define <8 x float> @f(<8 x float> %a, i32 %b) nounwind  {
 ; X32-LABEL: f:
 ; X32:       ## BB#0: ## %entry
 ; X32-NEXT:    pushl %ebp
 ; X32-NEXT:    movl %esp, %ebp
 ; X32-NEXT:    andl $-32, %esp
 ; X32-NEXT:    subl $64, %esp
 ; X32-NEXT:    movl 8(%ebp), %eax
+; X32-NEXT:    andl $7, %eax
 ; X32-NEXT:    movaps %xmm1, {{[0-9]+}}(%esp)
 ; X32-NEXT:    movaps %xmm0, (%esp)
 ; X32-NEXT:    movl $1084227584, (%esp,%eax,4) ## imm = 0x40A00000
 ; X32-NEXT:    movaps (%esp), %xmm0
 ; X32-NEXT:    movaps {{[0-9]+}}(%esp), %xmm1
 ; X32-NEXT:    movl %ebp, %esp
 ; X32-NEXT:    popl %ebp
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: f:
 ; X64:       ## BB#0: ## %entry
 ; X64-NEXT:    pushq %rbp
 ; X64-NEXT:    movq %rsp, %rbp
 ; X64-NEXT:    andq $-32, %rsp
 ; X64-NEXT:    subq $64, %rsp
+; X64-NEXT:    ## kill: %EDI<def> %EDI<kill> %RDI<def>
 ; X64-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
 ; X64-NEXT:    movaps %xmm0, (%rsp)
-; X64-NEXT:    movslq %edi, %rax
-; X64-NEXT:    movl $1084227584, (%rsp,%rax,4) ## imm = 0x40A00000
+; X64-NEXT:    andl $7, %edi
+; X64-NEXT:    movl $1084227584, (%rsp,%rdi,4) ## imm = 0x40A00000
 ; X64-NEXT:    movaps (%rsp), %xmm0
 ; X64-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
 ; X64-NEXT:    movq %rbp, %rsp
 ; X64-NEXT:    popq %rbp
 ; X64-NEXT:    retq
 entry:
   %vecins = insertelement <8 x float> %a, float 5.000000e+00, i32 %b
   ret <8 x float> %vecins
 }

test/CodeGen/X86/vec_insert-8.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i386-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=X32
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=X64
 
 ; tests variable insert and extract of a 4 x i32
 
 define <4 x i32> @var_insert(<4 x i32> %x, i32 %val, i32 %idx) nounwind {
 ; X32-LABEL: var_insert:
 ; X32:       # BB#0: # %entry
 ; X32-NEXT:    pushl %ebp
 ; X32-NEXT:    movl %esp, %ebp
 ; X32-NEXT:    andl $-16, %esp
 ; X32-NEXT:    subl $32, %esp
-; X32-NEXT:    movl 8(%ebp), %eax
-; X32-NEXT:    movl 12(%ebp), %ecx
+; X32-NEXT:    movl 12(%ebp), %eax
+; X32-NEXT:    andl $3, %eax
+; X32-NEXT:    movl 8(%ebp), %ecx
 ; X32-NEXT:    movaps %xmm0, (%esp)
-; X32-NEXT:    movl %eax, (%esp,%ecx,4)
+; X32-NEXT:    movl %ecx, (%esp,%eax,4)
 ; X32-NEXT:    movaps (%esp), %xmm0
 ; X32-NEXT:    movl %ebp, %esp
 ; X32-NEXT:    popl %ebp
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: var_insert:
 ; X64:       # BB#0: # %entry
+; X64-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
 ; X64-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movslq %esi, %rax
-; X64-NEXT:    movl %edi, -24(%rsp,%rax,4)
+; X64-NEXT:    andl $3, %esi
+; X64-NEXT:    movl %edi, -24(%rsp,%rsi,4)
 ; X64-NEXT:    movaps -{{[0-9]+}}(%rsp), %xmm0
 ; X64-NEXT:    retq
 entry:
   %tmp3 = insertelement <4 x i32> %x, i32 %val, i32 %idx
   ret <4 x i32> %tmp3
 }
 
 define i32 @var_extract(<4 x i32> %x, i32 %idx) nounwind {
 ; X32-LABEL: var_extract:
 ; X32:       # BB#0: # %entry
 ; X32-NEXT:    pushl %ebp
 ; X32-NEXT:    movl %esp, %ebp
 ; X32-NEXT:    andl $-16, %esp
 ; X32-NEXT:    subl $32, %esp
 ; X32-NEXT:    movl 8(%ebp), %eax
+; X32-NEXT:    andl $3, %eax
 ; X32-NEXT:    movaps %xmm0, (%esp)
 ; X32-NEXT:    movl (%esp,%eax,4), %eax
 ; X32-NEXT:    movl %ebp, %esp
 ; X32-NEXT:    popl %ebp
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: var_extract:
 ; X64:       # BB#0: # %entry
+; X64-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
 ; X64-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movslq %edi, %rax
-; X64-NEXT:    movl -24(%rsp,%rax,4), %eax
+; X64-NEXT:    andl $3, %edi
+; X64-NEXT:    movl -24(%rsp,%rdi,4), %eax
 ; X64-NEXT:    retq
 entry:
   %tmp3 = extractelement <4 x i32> %x, i32 %idx
   ret i32 %tmp3
 }

test/CodeGen/X86/vector-shuffle-variable-128.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
 
 ;
 ; Unary shuffle indices from registers
 ;
 
 define <2 x double> @var_shuffle_v2f64_v2f64_xx_i64(<2 x double> %x, i64 %i0, i64 %i1) nounwind {
 ; SSE-LABEL: var_shuffle_v2f64_v2f64_xx_i64:
 ; SSE:       # BB#0:
+; SSE-NEXT:    andl $1, %esi
+; SSE-NEXT:    andl $1, %edi
 ; SSE-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
 ; SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; SSE-NEXT:    movhpd {{.*#+}} xmm0 = xmm0[0],mem[0]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: var_shuffle_v2f64_v2f64_xx_i64:
 ; AVX:       # BB#0:
+; AVX-NEXT:    andl $1, %esi
+; AVX-NEXT:    andl $1, %edi
 ; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    vmovhpd {{.*#+}} xmm0 = xmm0[0],mem[0]
 ; AVX-NEXT:    retq
   %x0 = extractelement <2 x double> %x, i64 %i0
   %x1 = extractelement <2 x double> %x, i64 %i1
   %r0 = insertelement <2 x double> undef, double %x0, i32 0
   %r1 = insertelement <2 x double>   %r0, double %x1, i32 1
   ret <2 x double> %r1
 }
 
 define <2 x i64> @var_shuffle_v2i64_v2i64_xx_i64(<2 x i64> %x, i32 %i0, i32 %i1) nounwind {
 ; SSE-LABEL: var_shuffle_v2i64_v2i64_xx_i64:
 ; SSE:       # BB#0:
-; SSE-NEXT:    movslq %edi, %rax
+; SSE-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; SSE-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
+; SSE-NEXT:    andl $1, %edi
 ; SSE-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE-NEXT:    movslq %esi, %rcx
+; SSE-NEXT:    andl $1, %esi
 ; SSE-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
 ; SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: var_shuffle_v2i64_v2i64_xx_i64:
 ; AVX:       # BB#0:
-; AVX-NEXT:    movslq %edi, %rax
+; AVX-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; AVX-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
+; AVX-NEXT:    andl $1, %edi
 ; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX-NEXT:    movslq %esi, %rcx
+; AVX-NEXT:    andl $1, %esi
 ; AVX-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX-NEXT:    retq
   %x0 = extractelement <2 x i64> %x, i32 %i0
   %x1 = extractelement <2 x i64> %x, i32 %i1
   %r0 = insertelement <2 x i64> undef, i64 %x0, i32 0
   %r1 = insertelement <2 x i64>   %r0, i64 %x1, i32 1
   ret <2 x i64> %r1
 }
 
 define <4 x float> @var_shuffle_v4f32_v4f32_xxxx_i32(<4 x float> %x, i32 %i0, i32 %i1, i32 %i2, i32 %i3) nounwind {
 ; SSE2-LABEL: var_shuffle_v4f32_v4f32_xxxx_i32:
 ; SSE2:       # BB#0:
-; SSE2-NEXT:    movslq %edi, %rax
-; SSE2-NEXT:    movslq %esi, %rsi
-; SSE2-NEXT:    movslq %edx, %rdx
+; SSE2-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; SSE2-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; SSE2-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; SSE2-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
+; SSE2-NEXT:    andl $3, %edi
+; SSE2-NEXT:    andl $3, %esi
+; SSE2-NEXT:    andl $3, %edx
 ; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movslq %ecx, %rcx
+; SSE2-NEXT:    andl $3, %ecx
 ; SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSE2-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: var_shuffle_v4f32_v4f32_xxxx_i32:
 ; SSSE3:       # BB#0:
-; SSSE3-NEXT:    movslq %edi, %rax
-; SSSE3-NEXT:    movslq %esi, %rsi
-; SSSE3-NEXT:    movslq %edx, %rdx
+; SSSE3-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; SSSE3-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; SSSE3-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; SSSE3-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
+; SSSE3-NEXT:    andl $3, %edi
+; SSSE3-NEXT:    andl $3, %esi
+; SSSE3-NEXT:    andl $3, %edx
 ; SSSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSSE3-NEXT:    movslq %ecx, %rcx
+; SSSE3-NEXT:    andl $3, %ecx
 ; SSSE3-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSSE3-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; SSSE3-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: var_shuffle_v4f32_v4f32_xxxx_i32:
 ; SSE41:       # BB#0:
-; SSE41-NEXT:    movslq %edi, %rax
-; SSE41-NEXT:    movslq %esi, %rsi
-; SSE41-NEXT:    movslq %edx, %rdx
+; SSE41-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; SSE41-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; SSE41-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; SSE41-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
+; SSE41-NEXT:    andl $3, %edi
+; SSE41-NEXT:    andl $3, %esi
+; SSE41-NEXT:    andl $3, %edx
 ; SSE41-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE41-NEXT:    movslq %ecx, %rcx
+; SSE41-NEXT:    andl $3, %ecx
 ; SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
 ; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
 ; SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: var_shuffle_v4f32_v4f32_xxxx_i32:
 ; AVX:       # BB#0:
-; AVX-NEXT:    movslq %edi, %rax
-; AVX-NEXT:    movslq %esi, %rsi
-; AVX-NEXT:    movslq %edx, %rdx
+; AVX-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; AVX-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; AVX-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; AVX-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
+; AVX-NEXT:    andl $3, %edi
+; AVX-NEXT:    andl $3, %esi
+; AVX-NEXT:    andl $3, %edx
 ; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX-NEXT:    movslq %ecx, %rcx
+; AVX-NEXT:    andl $3, %ecx
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
 ; AVX-NEXT:    retq
   %x0 = extractelement <4 x float> %x, i32 %i0
   %x1 = extractelement <4 x float> %x, i32 %i1
   %x2 = extractelement <4 x float> %x, i32 %i2
   %x3 = extractelement <4 x float> %x, i32 %i3
   %r0 = insertelement <4 x float> undef, float %x0, i32 0
   %r1 = insertelement <4 x float>   %r0, float %x1, i32 1
   %r2 = insertelement <4 x float>   %r1, float %x2, i32 2
   %r3 = insertelement <4 x float>   %r2, float %x3, i32 3
   ret <4 x float> %r3
 }
 
 define <4 x i32> @var_shuffle_v4i32_v4i32_xxxx_i32(<4 x i32> %x, i32 %i0, i32 %i1, i32 %i2, i32 %i3) nounwind {
 ; SSE2-LABEL: var_shuffle_v4i32_v4i32_xxxx_i32:
 ; SSE2:       # BB#0:
-; SSE2-NEXT:    movslq %edi, %rax
-; SSE2-NEXT:    movslq %esi, %rsi
-; SSE2-NEXT:    movslq %edx, %rdx
+; SSE2-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; SSE2-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; SSE2-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; SSE2-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
+; SSE2-NEXT:    andl $3, %edi
+; SSE2-NEXT:    andl $3, %esi
+; SSE2-NEXT:    andl $3, %edx
 ; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movslq %ecx, %rcx
+; SSE2-NEXT:    andl $3, %ecx
 ; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: var_shuffle_v4i32_v4i32_xxxx_i32:
 ; SSSE3:       # BB#0:
-; SSSE3-NEXT:    movslq %edi, %rax
-; SSSE3-NEXT:    movslq %esi, %rsi
-; SSSE3-NEXT:    movslq %edx, %rdx
+; SSSE3-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; SSSE3-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; SSSE3-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; SSSE3-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
+; SSSE3-NEXT:    andl $3, %edi
+; SSSE3-NEXT:    andl $3, %esi
+; SSSE3-NEXT:    andl $3, %edx
 ; SSSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSSE3-NEXT:    movslq %ecx, %rcx
+; SSSE3-NEXT:    andl $3, %ecx
 ; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: var_shuffle_v4i32_v4i32_xxxx_i32:
 ; SSE41:       # BB#0:
-; SSE41-NEXT:    movslq %edi, %rax
-; SSE41-NEXT:    movslq %esi, %rsi
-; SSE41-NEXT:    movslq %edx, %rdx
+; SSE41-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; SSE41-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; SSE41-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; SSE41-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
+; SSE41-NEXT:    andl $3, %edi
+; SSE41-NEXT:    andl $3, %esi
+; SSE41-NEXT:    andl $3, %edx
 ; SSE41-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE41-NEXT:    movslq %ecx, %rcx
+; SSE41-NEXT:    andl $3, %ecx
 ; SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE41-NEXT:    pinsrd $1, -24(%rsp,%rsi,4), %xmm0
 ; SSE41-NEXT:    pinsrd $2, -24(%rsp,%rdx,4), %xmm0
 ; SSE41-NEXT:    pinsrd $3, -24(%rsp,%rcx,4), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: var_shuffle_v4i32_v4i32_xxxx_i32:
 ; AVX:       # BB#0:
-; AVX-NEXT:    movslq %edi, %rax
-; AVX-NEXT:    movslq %esi, %rsi
-; AVX-NEXT:    movslq %edx, %rdx
+; AVX-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; AVX-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; AVX-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; AVX-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
+; AVX-NEXT:    andl $3, %edi
+; AVX-NEXT:    andl $3, %esi
+; AVX-NEXT:    andl $3, %edx
 ; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX-NEXT:    movslq %ecx, %rcx
+; AVX-NEXT:    andl $3, %ecx
 ; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    vpinsrd $1, -24(%rsp,%rsi,4), %xmm0, %xmm0
 ; AVX-NEXT:    vpinsrd $2, -24(%rsp,%rdx,4), %xmm0, %xmm0
 ; AVX-NEXT:    vpinsrd $3, -24(%rsp,%rcx,4), %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %x0 = extractelement <4 x i32> %x, i32 %i0
   %x1 = extractelement <4 x i32> %x, i32 %i1
   %x2 = extractelement <4 x i32> %x, i32 %i2

 ; SSE2-LABEL: var_shuffle_v8i16_v8i16_xxxxxxxx_i16:
 ; SSE2:       # BB#0:
 ; SSE2-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
 ; SSE2-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
 ; SSE2-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
 ; SSE2-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
 ; SSE2-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
 ; SSE2-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; SSE2-NEXT:    movswq %di, %rax
-; SSE2-NEXT:    movswq %si, %rsi
-; SSE2-NEXT:    movswq %dx, %rdx
-; SSE2-NEXT:    movswq %cx, %r10
-; SSE2-NEXT:    movswq %r8w, %r11
+; SSE2-NEXT:    andl $7, %edi
+; SSE2-NEXT:    andl $7, %esi
+; SSE2-NEXT:    andl $7, %edx
+; SSE2-NEXT:    andl $7, %ecx
+; SSE2-NEXT:    andl $7, %r8d
 ; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movswq %r9w, %r8
-; SSE2-NEXT:    movswq {{[0-9]+}}(%rsp), %rcx
-; SSE2-NEXT:    movswq {{[0-9]+}}(%rsp), %rdi
-; SSE2-NEXT:    movzwl -24(%rsp,%rcx,2), %ecx
-; SSE2-NEXT:    movzwl -24(%rsp,%rdi,2), %edi
+; SSE2-NEXT:    andl $7, %r9d
+; SSE2-NEXT:    movzwl {{[0-9]+}}(%rsp), %r10d
+; SSE2-NEXT:    andl $7, %r10d
+; SSE2-NEXT:    movzwl {{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    andl $7, %eax
+; SSE2-NEXT:    movzwl -24(%rsp,%r10,2), %r10d
 ; SSE2-NEXT:    movzwl -24(%rsp,%rax,2), %eax
+; SSE2-NEXT:    movzwl -24(%rsp,%rdi,2), %edi
 ; SSE2-NEXT:    movzwl -24(%rsp,%rsi,2), %esi
-; SSE2-NEXT:    movd %ecx, %xmm0
-; SSE2-NEXT:    movzwl -24(%rsp,%rdx,2), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    movd %r10d, %xmm0
+; SSE2-NEXT:    movzwl -24(%rsp,%rdx,2), %edx
+; SSE2-NEXT:    movd %edx, %xmm1
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT:    movzwl -24(%rsp,%r10,2), %ecx
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movzwl -24(%rsp,%r11,2), %eax
-; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    movzwl -24(%rsp,%rcx,2), %ecx
+; SSE2-NEXT:    movd %edi, %xmm0
+; SSE2-NEXT:    movzwl -24(%rsp,%r8,2), %edx
+; SSE2-NEXT:    movd %edx, %xmm2
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; SSE2-NEXT:    movd %edi, %xmm1
+; SSE2-NEXT:    movd %eax, %xmm1
 ; SSE2-NEXT:    movd %ecx, %xmm2
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; SSE2-NEXT:    movd %esi, %xmm1
-; SSE2-NEXT:    movzwl -24(%rsp,%r8,2), %eax
+; SSE2-NEXT:    movzwl -24(%rsp,%r9,2), %eax
 ; SSE2-NEXT:    movd %eax, %xmm3
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: var_shuffle_v8i16_v8i16_xxxxxxxx_i16:
 ; SSSE3:       # BB#0:
 ; SSSE3-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
 ; SSSE3-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
 ; SSSE3-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
 ; SSSE3-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
 ; SSSE3-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
 ; SSSE3-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; SSSE3-NEXT:    movswq %di, %rax
-; SSSE3-NEXT:    movswq %si, %rsi
-; SSSE3-NEXT:    movswq %dx, %rdx
-; SSSE3-NEXT:    movswq %cx, %r10
-; SSSE3-NEXT:    movswq %r8w, %r11
+; SSSE3-NEXT:    andl $7, %edi
+; SSSE3-NEXT:    andl $7, %esi
+; SSSE3-NEXT:    andl $7, %edx
+; SSSE3-NEXT:    andl $7, %ecx
+; SSSE3-NEXT:    andl $7, %r8d
 ; SSSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSSE3-NEXT:    movswq %r9w, %r8
-; SSSE3-NEXT:    movswq {{[0-9]+}}(%rsp), %rcx
-; SSSE3-NEXT:    movswq {{[0-9]+}}(%rsp), %rdi
-; SSSE3-NEXT:    movzwl -24(%rsp,%rcx,2), %ecx
-; SSSE3-NEXT:    movzwl -24(%rsp,%rdi,2), %edi
+; SSSE3-NEXT:    andl $7, %r9d
+; SSSE3-NEXT:    movzwl {{[0-9]+}}(%rsp), %r10d
+; SSSE3-NEXT:    andl $7, %r10d
+; SSSE3-NEXT:    movzwl {{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    andl $7, %eax
+; SSSE3-NEXT:    movzwl -24(%rsp,%r10,2), %r10d
 ; SSSE3-NEXT:    movzwl -24(%rsp,%rax,2), %eax
+; SSSE3-NEXT:    movzwl -24(%rsp,%rdi,2), %edi
 ; SSSE3-NEXT:    movzwl -24(%rsp,%rsi,2), %esi
-; SSSE3-NEXT:    movd %ecx, %xmm0
-; SSSE3-NEXT:    movzwl -24(%rsp,%rdx,2), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    movd %r10d, %xmm0
+; SSSE3-NEXT:    movzwl -24(%rsp,%rdx,2), %edx
+; SSSE3-NEXT:    movd %edx, %xmm1
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSSE3-NEXT:    movzwl -24(%rsp,%r10,2), %ecx
-; SSSE3-NEXT:    movd %eax, %xmm0
-; SSSE3-NEXT:    movzwl -24(%rsp,%r11,2), %eax
-; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    movzwl -24(%rsp,%rcx,2), %ecx
+; SSSE3-NEXT:    movd %edi, %xmm0
+; SSSE3-NEXT:    movzwl -24(%rsp,%r8,2), %edx
+; SSSE3-NEXT:    movd %edx, %xmm2
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; SSSE3-NEXT:    movd %edi, %xmm1
+; SSSE3-NEXT:    movd %eax, %xmm1
 ; SSSE3-NEXT:    movd %ecx, %xmm2
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; SSSE3-NEXT:    movd %esi, %xmm1
-; SSSE3-NEXT:    movzwl -24(%rsp,%r8,2), %eax
+; SSSE3-NEXT:    movzwl -24(%rsp,%r9,2), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm3
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: var_shuffle_v8i16_v8i16_xxxxxxxx_i16:
 ; SSE41:       # BB#0:
-; SSE41-NEXT:    pushq %rbx
 ; SSE41-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
 ; SSE41-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
 ; SSE41-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
 ; SSE41-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
 ; SSE41-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
 ; SSE41-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; SSE41-NEXT:    movswq %di, %rax
-; SSE41-NEXT:    movswq %si, %rbx
-; SSE41-NEXT:    movswq %dx, %r11
-; SSE41-NEXT:    movswq %cx, %r10
-; SSE41-NEXT:    movswq %r8w, %rdi
+; SSE41-NEXT:    andl $7, %edi
+; SSE41-NEXT:    andl $7, %esi
+; SSE41-NEXT:    andl $7, %edx
+; SSE41-NEXT:    andl $7, %ecx
+; SSE41-NEXT:    andl $7, %r8d
 ; SSE41-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE41-NEXT:    movswq %r9w, %rcx
-; SSE41-NEXT:    movswq {{[0-9]+}}(%rsp), %rdx
-; SSE41-NEXT:    movswq {{[0-9]+}}(%rsp), %rsi
-; SSE41-NEXT:    movzwl -16(%rsp,%rdx,2), %edx
-; SSE41-NEXT:    movzwl -16(%rsp,%rsi,2), %esi
-; SSE41-NEXT:    movzwl -16(%rsp,%rax,2), %eax
-; SSE41-NEXT:    movd %eax, %xmm0
-; SSE41-NEXT:    pinsrw $1, -16(%rsp,%rbx,2), %xmm0
-; SSE41-NEXT:    pinsrw $2, -16(%rsp,%r11,2), %xmm0
-; SSE41-NEXT:    pinsrw $3, -16(%rsp,%r10,2), %xmm0
-; SSE41-NEXT:    pinsrw $4, -16(%rsp,%rdi,2), %xmm0
-; SSE41-NEXT:    pinsrw $5, -16(%rsp,%rcx,2), %xmm0
-; SSE41-NEXT:    pinsrw $6, %edx, %xmm0
-; SSE41-NEXT:    pinsrw $7, %esi, %xmm0
-; SSE41-NEXT:    popq %rbx
+; SSE41-NEXT:    andl $7, %r9d
+; SSE41-NEXT:    movzwl {{[0-9]+}}(%rsp), %r10d
+; SSE41-NEXT:    andl $7, %r10d
+; SSE41-NEXT:    movzwl {{[0-9]+}}(%rsp), %eax
+; SSE41-NEXT:    andl $7, %eax
+; SSE41-NEXT:    movzwl -24(%rsp,%r10,2), %r10d
+; SSE41-NEXT:    movzwl -24(%rsp,%rax,2), %eax
+; SSE41-NEXT:    movzwl -24(%rsp,%rdi,2), %edi
+; SSE41-NEXT:    movd %edi, %xmm0
+; SSE41-NEXT:    pinsrw $1, -24(%rsp,%rsi,2), %xmm0
+; SSE41-NEXT:    pinsrw $2, -24(%rsp,%rdx,2), %xmm0
+; SSE41-NEXT:    pinsrw $3, -24(%rsp,%rcx,2), %xmm0
+; SSE41-NEXT:    pinsrw $4, -24(%rsp,%r8,2), %xmm0
+; SSE41-NEXT:    pinsrw $5, -24(%rsp,%r9,2), %xmm0
+; SSE41-NEXT:    pinsrw $6, %r10d, %xmm0
+; SSE41-NEXT:    pinsrw $7, %eax, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: var_shuffle_v8i16_v8i16_xxxxxxxx_i16:
 ; AVX:       # BB#0:
-; AVX-NEXT:    pushq %r14
-; AVX-NEXT:    pushq %rbx
 ; AVX-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
 ; AVX-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
 ; AVX-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
 ; AVX-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
 ; AVX-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
 ; AVX-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; AVX-NEXT:    movswq %di, %r10
-; AVX-NEXT:    movswq %si, %r11
-; AVX-NEXT:    movswq %dx, %r14
-; AVX-NEXT:    movswq %cx, %rcx
-; AVX-NEXT:    movswq %r8w, %rdi
+; AVX-NEXT:    andl $7, %edi
+; AVX-NEXT:    andl $7, %esi
+; AVX-NEXT:    andl $7, %edx
+; AVX-NEXT:    andl $7, %ecx
+; AVX-NEXT:    andl $7, %r8d
 ; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX-NEXT:    movswq %r9w, %rax
-; AVX-NEXT:    movswq {{[0-9]+}}(%rsp), %rsi
-; AVX-NEXT:    movswq {{[0-9]+}}(%rsp), %rdx
-; AVX-NEXT:    movzwl -24(%rsp,%rsi,2), %esi
-; AVX-NEXT:    movzwl -24(%rsp,%rdx,2), %edx
-; AVX-NEXT:    movzwl -24(%rsp,%r10,2), %ebx
-; AVX-NEXT:    vmovd %ebx, %xmm0
-; AVX-NEXT:    vpinsrw $1, -24(%rsp,%r11,2), %xmm0, %xmm0
-; AVX-NEXT:    vpinsrw $2, -24(%rsp,%r14,2), %xmm0, %xmm0
+; AVX-NEXT:    andl $7, %r9d
+; AVX-NEXT:    movzwl {{[0-9]+}}(%rsp), %r10d
+; AVX-NEXT:    andl $7, %r10d
+; AVX-NEXT:    movzwl {{[0-9]+}}(%rsp), %eax
+; AVX-NEXT:    andl $7, %eax
+; AVX-NEXT:    movzwl -24(%rsp,%r10,2), %r10d
+; AVX-NEXT:    movzwl -24(%rsp,%rax,2), %eax
+; AVX-NEXT:    movzwl -24(%rsp,%rdi,2), %edi
+; AVX-NEXT:    vmovd %edi, %xmm0
+; AVX-NEXT:    vpinsrw $1, -24(%rsp,%rsi,2), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $2, -24(%rsp,%rdx,2), %xmm0, %xmm0
 ; AVX-NEXT:    vpinsrw $3, -24(%rsp,%rcx,2), %xmm0, %xmm0
-; AVX-NEXT:    vpinsrw $4, -24(%rsp,%rdi,2), %xmm0, %xmm0
-; AVX-NEXT:    vpinsrw $5, -24(%rsp,%rax,2), %xmm0, %xmm0
-; AVX-NEXT:    vpinsrw $6, %esi, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrw $7, %edx, %xmm0, %xmm0
-; AVX-NEXT:    popq %rbx
-; AVX-NEXT:    popq %r14
+; AVX-NEXT:    vpinsrw $4, -24(%rsp,%r8,2), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $5, -24(%rsp,%r9,2), %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $6, %r10d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %x0 = extractelement <8 x i16> %x, i16 %i0
   %x1 = extractelement <8 x i16> %x, i16 %i1
   %x2 = extractelement <8 x i16> %x, i16 %i2
   %x3 = extractelement <8 x i16> %x, i16 %i3
   %x4 = extractelement <8 x i16> %x, i16 %i4
   %x5 = extractelement <8 x i16> %x, i16 %i5
   %x6 = extractelement <8 x i16> %x, i16 %i6

 ; SSE2:       # BB#0:
 ; SSE2-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
 ; SSE2-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
 ; SSE2-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
 ; SSE2-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
 ; SSE2-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
 ; SSE2-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
 ; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movsbq {{[0-9]+}}(%rsp), %r10
+; SSE2-NEXT:    movzbl {{[0-9]+}}(%rsp), %r10d
+; SSE2-NEXT:    andl $15, %r10d
 ; SSE2-NEXT:    leaq -{{[0-9]+}}(%rsp), %r11
 ; SSE2-NEXT:    movzbl (%r10,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm15
-; SSE2-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    andl $15, %eax
 ; SSE2-NEXT:    movzbl (%rax,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm8
-; SSE2-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    andl $15, %eax
 ; SSE2-NEXT:    movzbl (%rax,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm9
-; SSE2-NEXT:    movsbq %dl, %rax
-; SSE2-NEXT:    movzbl (%rax,%r11), %eax
+; SSE2-NEXT:    andl $15, %edx
+; SSE2-NEXT:    movzbl (%rdx,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    andl $15, %eax
 ; SSE2-NEXT:    movzbl (%rax,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm10
-; SSE2-NEXT:    movsbq %dil, %rax
-; SSE2-NEXT:    movzbl (%rax,%r11), %eax
+; SSE2-NEXT:    andl $15, %edi
+; SSE2-NEXT:    movzbl (%rdi,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    andl $15, %eax
 ; SSE2-NEXT:    movzbl (%rax,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm11
-; SSE2-NEXT:    movsbq %r8b, %rax
-; SSE2-NEXT:    movzbl (%rax,%r11), %eax
+; SSE2-NEXT:    andl $15, %r8d
+; SSE2-NEXT:    movzbl (%r8,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm7
-; SSE2-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    andl $15, %eax
 ; SSE2-NEXT:    movzbl (%rax,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    andl $15, %eax
 ; SSE2-NEXT:    movzbl (%rax,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm12
-; SSE2-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    andl $15, %eax
 ; SSE2-NEXT:    movzbl (%rax,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm13
-; SSE2-NEXT:    movsbq %cl, %rax
-; SSE2-NEXT:    movzbl (%rax,%r11), %eax
+; SSE2-NEXT:    andl $15, %ecx
+; SSE2-NEXT:    movzbl (%rcx,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm6
-; SSE2-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    andl $15, %eax
 ; SSE2-NEXT:    movzbl (%rax,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm14
-; SSE2-NEXT:    movsbq %sil, %rax
-; SSE2-NEXT:    movzbl (%rax,%r11), %eax
+; SSE2-NEXT:    andl $15, %esi
+; SSE2-NEXT:    movzbl (%rsi,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm5
-; SSE2-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSE2-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    andl $15, %eax
 ; SSE2-NEXT:    movzbl (%rax,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:    movsbq %r9b, %rax
-; SSE2-NEXT:    movzbl (%rax,%r11), %eax
+; SSE2-NEXT:    andl $15, %r9d
+; SSE2-NEXT:    movzbl (%r9,%r11), %eax
 ; SSE2-NEXT:    movd %eax, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3],xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3],xmm3[4],xmm9[4],xmm3[5],xmm9[5],xmm3[6],xmm9[6],xmm3[7],xmm9[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm15[0],xmm3[1],xmm15[1],xmm3[2],xmm15[2],xmm3[3],xmm15[3],xmm3[4],xmm15[4],xmm3[5],xmm15[5],xmm3[6],xmm15[6],xmm3[7],xmm15[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm11[0],xmm7[1],xmm11[1],xmm7[2],xmm11[2],xmm7[3],xmm11[3],xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1],xmm0[2],xmm7[2],xmm0[3],xmm7[3],xmm0[4],xmm7[4],xmm0[5],xmm7[5],xmm0[6],xmm7[6],xmm0[7],xmm7[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]

 ; SSSE3:       # BB#0:
 ; SSSE3-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
 ; SSSE3-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
 ; SSSE3-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
 ; SSSE3-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
 ; SSSE3-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
 ; SSSE3-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
 ; SSSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSSE3-NEXT:    movsbq {{[0-9]+}}(%rsp), %r10
+; SSSE3-NEXT:    movzbl {{[0-9]+}}(%rsp), %r10d
+; SSSE3-NEXT:    andl $15, %r10d
 ; SSSE3-NEXT:    leaq -{{[0-9]+}}(%rsp), %r11
 ; SSSE3-NEXT:    movzbl (%r10,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm15
-; SSSE3-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSSE3-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    andl $15, %eax
 ; SSSE3-NEXT:    movzbl (%rax,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm8
-; SSSE3-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSSE3-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    andl $15, %eax
 ; SSSE3-NEXT:    movzbl (%rax,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm9
-; SSSE3-NEXT:    movsbq %dl, %rax
-; SSSE3-NEXT:    movzbl (%rax,%r11), %eax
+; SSSE3-NEXT:    andl $15, %edx
+; SSSE3-NEXT:    movzbl (%rdx,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm3
-; SSSE3-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSSE3-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    andl $15, %eax
 ; SSSE3-NEXT:    movzbl (%rax,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm10
-; SSSE3-NEXT:    movsbq %dil, %rax
-; SSSE3-NEXT:    movzbl (%rax,%r11), %eax
+; SSSE3-NEXT:    andl $15, %edi
+; SSSE3-NEXT:    movzbl (%rdi,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm0
-; SSSE3-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSSE3-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    andl $15, %eax
 ; SSSE3-NEXT:    movzbl (%rax,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm11
-; SSSE3-NEXT:    movsbq %r8b, %rax
-; SSSE3-NEXT:    movzbl (%rax,%r11), %eax
+; SSSE3-NEXT:    andl $15, %r8d
+; SSSE3-NEXT:    movzbl (%r8,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm7
-; SSSE3-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSSE3-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    andl $15, %eax
 ; SSSE3-NEXT:    movzbl (%rax,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm2
-; SSSE3-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSSE3-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    andl $15, %eax
 ; SSSE3-NEXT:    movzbl (%rax,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm12
-; SSSE3-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSSE3-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    andl $15, %eax
 ; SSSE3-NEXT:    movzbl (%rax,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm13
-; SSSE3-NEXT:    movsbq %cl, %rax
-; SSSE3-NEXT:    movzbl (%rax,%r11), %eax
+; SSSE3-NEXT:    andl $15, %ecx
+; SSSE3-NEXT:    movzbl (%rcx,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm6
-; SSSE3-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSSE3-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    andl $15, %eax
 ; SSSE3-NEXT:    movzbl (%rax,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm14
-; SSSE3-NEXT:    movsbq %sil, %rax
-; SSSE3-NEXT:    movzbl (%rax,%r11), %eax
+; SSSE3-NEXT:    andl $15, %esi
+; SSSE3-NEXT:    movzbl (%rsi,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm5
-; SSSE3-NEXT:    movsbq {{[0-9]+}}(%rsp), %rax
+; SSSE3-NEXT:    movzbl {{[0-9]+}}(%rsp), %eax
+; SSSE3-NEXT:    andl $15, %eax
 ; SSSE3-NEXT:    movzbl (%rax,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm4
-; SSSE3-NEXT:    movsbq %r9b, %rax
-; SSSE3-NEXT:    movzbl (%rax,%r11), %eax
+; SSSE3-NEXT:    andl $15, %r9d
+; SSSE3-NEXT:    movzbl (%r9,%r11), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm1
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3],xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3],xmm3[4],xmm9[4],xmm3[5],xmm9[5],xmm3[6],xmm9[6],xmm3[7],xmm9[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm15[0],xmm3[1],xmm15[1],xmm3[2],xmm15[2],xmm3[3],xmm15[3],xmm3[4],xmm15[4],xmm3[5],xmm15[5],xmm3[6],xmm15[6],xmm3[7],xmm15[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm11[0],xmm7[1],xmm11[1],xmm7[2],xmm11[2],xmm7[3],xmm11[3],xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1],xmm0[2],xmm7[2],xmm0[3],xmm7[3],xmm0[4],xmm7[4],xmm0[5],xmm7[5],xmm0[6],xmm7[6],xmm0[7],xmm7[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm12[0],xmm2[1],xmm12[1],xmm2[2],xmm12[2],xmm2[3],xmm12[3],xmm2[4],xmm12[4],xmm2[5],xmm12[5],xmm2[6],xmm12[6],xmm2[7],xmm12[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm13[0],xmm6[1],xmm13[1],xmm6[2],xmm13[2],xmm6[3],xmm13[3],xmm6[4],xmm13[4],xmm6[5],xmm13[5],xmm6[6],xmm13[6],xmm6[7],xmm13[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm14[0],xmm5[1],xmm14[1],xmm5[2],xmm14[2],xmm5[3],xmm14[3],xmm5[4],xmm14[4],xmm5[5],xmm14[5],xmm5[6],xmm14[6],xmm5[7],xmm14[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3],xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3],xmm5[4],xmm6[4],xmm5[5],xmm6[5],xmm5[6],xmm6[6],xmm5[7],xmm6[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: var_shuffle_v16i8_v16i8_xxxxxxxxxxxxxxxx_i8:
 ; SSE41:       # BB#0:
 ; SSE41-NEXT:    pushq %rbp
 ; SSE41-NEXT:    pushq %r15
 ; SSE41-NEXT:    pushq %r14
-; SSE41-NEXT:    pushq %r13
 ; SSE41-NEXT:    pushq %r12
 ; SSE41-NEXT:    pushq %rbx
 ; SSE41-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
 ; SSE41-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
 ; SSE41-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
 ; SSE41-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
 ; SSE41-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
 ; SSE41-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; SSE41-NEXT:    movsbq %dil, %r15
-; SSE41-NEXT:    movsbq %sil, %r14
-; SSE41-NEXT:    movsbq %dl, %r11
-; SSE41-NEXT:    movsbq %cl, %r10
-; SSE41-NEXT:    movsbq %r8b, %r8
+; SSE41-NEXT:    andl $15, %edi
+; SSE41-NEXT:    andl $15, %esi
+; SSE41-NEXT:    andl $15, %edx
+; SSE41-NEXT:    andl $15, %ecx
+; SSE41-NEXT:    andl $15, %r8d
 ; SSE41-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE41-NEXT:    movsbq %r9b, %r9
-; SSE41-NEXT:    movsbq {{[0-9]+}}(%rsp), %r12
-; SSE41-NEXT:    movsbq {{[0-9]+}}(%rsp), %r13
-; SSE41-NEXT:    movsbq {{[0-9]+}}(%rsp), %rbp
-; SSE41-NEXT:    movsbq {{[0-9]+}}(%rsp), %rbx
+; SSE41-NEXT:    andl $15, %r9d
+; SSE41-NEXT:    movzbl {{[0-9]+}}(%rsp), %r10d
+; SSE41-NEXT:    andl $15, %r10d
+; SSE41-NEXT:    movzbl {{[0-9]+}}(%rsp), %r11d
+; SSE41-NEXT:    andl $15, %r11d
+; SSE41-NEXT:    movzbl {{[0-9]+}}(%rsp), %r14d
+; SSE41-NEXT:    andl $15, %r14d
+; SSE41-NEXT:    movzbl {{[0-9]+}}(%rsp), %r15d
+; SSE41-NEXT:    andl $15, %r15d
 ; SSE41-NEXT:    leaq -{{[0-9]+}}(%rsp), %rax
-; SSE41-NEXT:    movzbl (%r15,%rax), %ecx
-; SSE41-NEXT:    movd %ecx, %xmm0
-; SSE41-NEXT:    movsbq {{[0-9]+}}(%rsp), %r15
-; SSE41-NEXT:    pinsrb $1, (%r14,%rax), %xmm0
-; SSE41-NEXT:    movsbq {{[0-9]+}}(%rsp), %r14
-; SSE41-NEXT:    pinsrb $2, (%r11,%rax), %xmm0
-; SSE41-NEXT:    movsbq {{[0-9]+}}(%rsp), %r11
-; SSE41-NEXT:    pinsrb $3, (%r10,%rax), %xmm0
-; SSE41-NEXT:    movsbq {{[0-9]+}}(%rsp), %r10
+; SSE41-NEXT:    movzbl (%rdi,%rax), %edi
+; SSE41-NEXT:    movd %edi, %xmm0
+; SSE41-NEXT:    movzbl {{[0-9]+}}(%rsp), %r12d
+; SSE41-NEXT:    andl $15, %r12d
+; SSE41-NEXT:    pinsrb $1, (%rsi,%rax), %xmm0
+; SSE41-NEXT:    movzbl {{[0-9]+}}(%rsp), %esi
+; SSE41-NEXT:    andl $15, %esi
+; SSE41-NEXT:    pinsrb $2, (%rdx,%rax), %xmm0
+; SSE41-NEXT:    movzbl {{[0-9]+}}(%rsp), %edx
+; SSE41-NEXT:    andl $15, %edx
+; SSE41-NEXT:    pinsrb $3, (%rcx,%rax), %xmm0
+; SSE41-NEXT:    movzbl {{[0-9]+}}(%rsp), %ecx
+; SSE41-NEXT:    andl $15, %ecx
 ; SSE41-NEXT:    pinsrb $4, (%r8,%rax), %xmm0
-; SSE41-NEXT:    movsbq {{[0-9]+}}(%rsp), %rcx
+; SSE41-NEXT:    movzbl {{[0-9]+}}(%rsp), %ebx
+; SSE41-NEXT:    andl $15, %ebx
 ; SSE41-NEXT:    pinsrb $5, (%r9,%rax), %xmm0
-; SSE41-NEXT:    movsbq {{[0-9]+}}(%rsp), %rdx
-; SSE41-NEXT:    movzbl (%r12,%rax), %esi
-; SSE41-NEXT:    movzbl (%r13,%rax), %edi
-; SSE41-NEXT:    movzbl (%rbp,%rax), %ebp
-; SSE41-NEXT:    movzbl (%rbx,%rax), %ebx
-; SSE41-NEXT:    movzbl (%r15,%rax), %r8d
-; SSE41-NEXT:    movzbl (%r14,%rax), %r9d
-; SSE41-NEXT:    movzbl (%r11,%rax), %r11d
-; SSE41-NEXT:    movzbl (%r10,%rax), %r10d
+; SSE41-NEXT:    movzbl {{[0-9]+}}(%rsp), %edi
+; SSE41-NEXT:    andl $15, %edi
+; SSE41-NEXT:    movzbl (%r10,%rax), %r8d
+; SSE41-NEXT:    movzbl (%r11,%rax), %r9d
+; SSE41-NEXT:    movzbl (%r14,%rax), %r10d
+; SSE41-NEXT:    movzbl (%r15,%rax), %r11d
+; SSE41-NEXT:    movzbl (%r12,%rax), %ebp
+; SSE41-NEXT:    movzbl (%rsi,%rax), %esi
+; SSE41-NEXT:    movzbl (%rdx,%rax), %edx
 ; SSE41-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE41-NEXT:    movzbl (%rdx,%rax), %eax
-; SSE41-NEXT:    pinsrb $6, %esi, %xmm0
-; SSE41-NEXT:    pinsrb $7, %edi, %xmm0
-; SSE41-NEXT:    pinsrb $8, %ebp, %xmm0
-; SSE41-NEXT:    pinsrb $9, %ebx, %xmm0
-; SSE41-NEXT:    pinsrb $10, %r8d, %xmm0
-; SSE41-NEXT:    pinsrb $11, %r9d, %xmm0
-; SSE41-NEXT:    pinsrb $12, %r11d, %xmm0
-; SSE41-NEXT:    pinsrb $13, %r10d, %xmm0
-; SSE41-NEXT:    pinsrb $14, %ecx, %xmm0
+; SSE41-NEXT:    movzbl (%rbx,%rax), %ebx
+; SSE41-NEXT:    movzbl (%rdi,%rax), %eax
+; SSE41-NEXT:    pinsrb $6, %r8d, %xmm0
+; SSE41-NEXT:    pinsrb $7, %r9d, %xmm0
+; SSE41-NEXT:    pinsrb $8, %r10d, %xmm0
+; SSE41-NEXT:    pinsrb $9, %r11d, %xmm0
+; SSE41-NEXT:    pinsrb $10, %ebp, %xmm0
+; SSE41-NEXT:    pinsrb $11, %esi, %xmm0
+; SSE41-NEXT:    pinsrb $12, %edx, %xmm0
+; SSE41-NEXT:    pinsrb $13, %ecx, %xmm0
+; SSE41-NEXT:    pinsrb $14, %ebx, %xmm0
 ; SSE41-NEXT:    pinsrb $15, %eax, %xmm0
 ; SSE41-NEXT:    popq %rbx
 ; SSE41-NEXT:    popq %r12
-; SSE41-NEXT:    popq %r13
 ; SSE41-NEXT:    popq %r14
 ; SSE41-NEXT:    popq %r15
 ; SSE41-NEXT:    popq %rbp
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: var_shuffle_v16i8_v16i8_xxxxxxxxxxxxxxxx_i8:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    pushq %rbp
 ; AVX-NEXT:    pushq %r15
 ; AVX-NEXT:    pushq %r14
-; AVX-NEXT:    pushq %r13
 ; AVX-NEXT:    pushq %r12
 ; AVX-NEXT:    pushq %rbx
 ; AVX-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
 ; AVX-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
 ; AVX-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
 ; AVX-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
 ; AVX-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
 ; AVX-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; AVX-NEXT:    movsbq %dil, %r10
-; AVX-NEXT:    movsbq %sil, %r11
-; AVX-NEXT:    movsbq %dl, %r14
-; AVX-NEXT:    movsbq %cl, %r15
-; AVX-NEXT:    movsbq %r8b, %r8
+; AVX-NEXT:    andl $15, %edi
+; AVX-NEXT:    andl $15, %esi
+; AVX-NEXT:    andl $15, %edx
+; AVX-NEXT:    andl $15, %ecx
+; AVX-NEXT:    andl $15, %r8d
 ; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX-NEXT:    movsbq %r9b, %r9
-; AVX-NEXT:    movsbq {{[0-9]+}}(%rsp), %r12
-; AVX-NEXT:    movsbq {{[0-9]+}}(%rsp), %r13
-; AVX-NEXT:    movsbq {{[0-9]+}}(%rsp), %rbp
-; AVX-NEXT:    movsbq {{[0-9]+}}(%rsp), %rcx
-; AVX-NEXT:    leaq -{{[0-9]+}}(%rsp), %rdi
-; AVX-NEXT:    movzbl (%r10,%rdi), %eax
-; AVX-NEXT:    vmovd %eax, %xmm0
-; AVX-NEXT:    movsbq {{[0-9]+}}(%rsp), %r10
-; AVX-NEXT:    vpinsrb $1, (%r11,%rdi), %xmm0, %xmm0
-; AVX-NEXT:    movsbq {{[0-9]+}}(%rsp), %r11
-; AVX-NEXT:    vpinsrb $2, (%r14,%rdi), %xmm0, %xmm0
-; AVX-NEXT:    movsbq {{[0-9]+}}(%rsp), %r14
-; AVX-NEXT:    vpinsrb $3, (%r15,%rdi), %xmm0, %xmm0
-; AVX-NEXT:    movsbq {{[0-9]+}}(%rsp), %r15
-; AVX-NEXT:    vpinsrb $4, (%r8,%rdi), %xmm0, %xmm0
-; AVX-NEXT:    movsbq {{[0-9]+}}(%rsp), %r8
-; AVX-NEXT:    vpinsrb $5, (%r9,%rdi), %xmm0, %xmm0
-; AVX-NEXT:    movsbq {{[0-9]+}}(%rsp), %rsi
-; AVX-NEXT:    movzbl (%r12,%rdi), %edx
-; AVX-NEXT:    movzbl (%r13,%rdi), %ebx
-; AVX-NEXT:    movzbl (%rbp,%rdi), %ebp
-; AVX-NEXT:    movzbl (%rcx,%rdi), %ecx
-; AVX-NEXT:    movzbl (%r10,%rdi), %eax
-; AVX-NEXT:    movzbl (%r11,%rdi), %r9d
-; AVX-NEXT:    movzbl (%r14,%rdi), %r10d
-; AVX-NEXT:    movzbl (%r15,%rdi), %r11d
-; AVX-NEXT:    movzbl (%r8,%rdi), %r8d
-; AVX-NEXT:    movzbl (%rsi,%rdi), %esi
-; AVX-NEXT:    vpinsrb $6, %edx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $7, %ebx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $8, %ebp, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $9, %ecx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $10, %eax, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $11, %r9d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $12, %r10d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $13, %r11d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $14, %r8d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $15, %esi, %xmm0, %xmm0
+; AVX-NEXT:    andl $15, %r9d
+; AVX-NEXT:    movzbl {{[0-9]+}}(%rsp), %r10d
+; AVX-NEXT:    andl $15, %r10d
+; AVX-NEXT:    movzbl {{[0-9]+}}(%rsp), %r11d
+; AVX-NEXT:    andl $15, %r11d
+; AVX-NEXT:    movzbl {{[0-9]+}}(%rsp), %r14d
+; AVX-NEXT:    andl $15, %r14d
+; AVX-NEXT:    movzbl {{[0-9]+}}(%rsp), %r15d
+; AVX-NEXT:    andl $15, %r15d
+; AVX-NEXT:    leaq -{{[0-9]+}}(%rsp), %rax
+; AVX-NEXT:    movzbl (%rdi,%rax), %edi
+; AVX-NEXT:    vmovd %edi, %xmm0
+; AVX-NEXT:    movzbl {{[0-9]+}}(%rsp), %r12d
+; AVX-NEXT:    andl $15, %r12d
+; AVX-NEXT:    vpinsrb $1, (%rsi,%rax), %xmm0, %xmm0
+; AVX-NEXT:    movzbl {{[0-9]+}}(%rsp), %esi
+; AVX-NEXT:    andl $15, %esi
+; AVX-NEXT:    vpinsrb $2, (%rdx,%rax), %xmm0, %xmm0
+; AVX-NEXT:    movzbl {{[0-9]+}}(%rsp), %edx
+; AVX-NEXT:    andl $15, %edx
+; AVX-NEXT:    vpinsrb $3, (%rcx,%rax), %xmm0, %xmm0
+; AVX-NEXT:    movzbl {{[0-9]+}}(%rsp), %ecx
+; AVX-NEXT:    andl $15, %ecx
+; AVX-NEXT:    vpinsrb $4, (%r8,%rax), %xmm0, %xmm0
+; AVX-NEXT:    movzbl {{[0-9]+}}(%rsp), %ebx
+; AVX-NEXT:    andl $15, %ebx
+; AVX-NEXT:    vpinsrb $5, (%r9,%rax), %xmm0, %xmm0
+; AVX-NEXT:    movzbl {{[0-9]+}}(%rsp), %edi
+; AVX-NEXT:    andl $15, %edi
+; AVX-NEXT:    movzbl (%r10,%rax), %r8d
+; AVX-NEXT:    movzbl (%r11,%rax), %r9d
+; AVX-NEXT:    movzbl (%r14,%rax), %r10d
+; AVX-NEXT:    movzbl (%r15,%rax), %r11d
+; AVX-NEXT:    movzbl (%r12,%rax), %ebp
+; AVX-NEXT:    movzbl (%rsi,%rax), %esi
+; AVX-NEXT:    movzbl (%rdx,%rax), %edx
+; AVX-NEXT:    movzbl (%rcx,%rax), %ecx
+; AVX-NEXT:    movzbl (%rbx,%rax), %ebx
+; AVX-NEXT:    movzbl (%rdi,%rax), %eax
+; AVX-NEXT:    vpinsrb $6, %r8d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $7, %r9d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $8, %r10d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $9, %r11d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $10, %ebp, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $11, %esi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $12, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $13, %ecx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $14, %ebx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $15, %eax, %xmm0, %xmm0
 ; AVX-NEXT:    popq %rbx
 ; AVX-NEXT:    popq %r12
-; AVX-NEXT:    popq %r13
 ; AVX-NEXT:    popq %r14
 ; AVX-NEXT:    popq %r15
 ; AVX-NEXT:    popq %rbp
 ; AVX-NEXT:    retq
   %x0  = extractelement <16 x i8> %x, i8 %i0
   %x1  = extractelement <16 x i8> %x, i8 %i1
   %x2  = extractelement <16 x i8> %x, i8 %i2
   %x3  = extractelement <16 x i8> %x, i8 %i3

 
 ;
 ; Unary shuffle indices from memory
 ;
 
 define <4 x i32> @mem_shuffle_v4i32_v4i32_xxxx_i32(<4 x i32> %x, i32* %i) nounwind {
 ; SSE2-LABEL: mem_shuffle_v4i32_v4i32_xxxx_i32:
 ; SSE2:       # BB#0:
-; SSE2-NEXT:    movslq (%rdi), %rax
+; SSE2-NEXT:    movl (%rdi), %eax
+; SSE2-NEXT:    movl 4(%rdi), %ecx
+; SSE2-NEXT:    andl $3, %eax
 ; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movslq 4(%rdi), %rcx
-; SSE2-NEXT:    movslq 8(%rdi), %rdx
-; SSE2-NEXT:    movslq 12(%rdi), %rsi
+; SSE2-NEXT:    andl $3, %ecx
+; SSE2-NEXT:    movl 8(%rdi), %edx
+; SSE2-NEXT:    andl $3, %edx
+; SSE2-NEXT:    movl 12(%rdi), %esi
+; SSE2-NEXT:    andl $3, %esi
 ; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: mem_shuffle_v4i32_v4i32_xxxx_i32:
 ; SSSE3:       # BB#0:
-; SSSE3-NEXT:    movslq (%rdi), %rax
+; SSSE3-NEXT:    movl (%rdi), %eax
+; SSSE3-NEXT:    movl 4(%rdi), %ecx
+; SSSE3-NEXT:    andl $3, %eax
 ; SSSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSSE3-NEXT:    movslq 4(%rdi), %rcx
-; SSSE3-NEXT:    movslq 8(%rdi), %rdx
-; SSSE3-NEXT:    movslq 12(%rdi), %rsi
+; SSSE3-NEXT:    andl $3, %ecx
+; SSSE3-NEXT:    movl 8(%rdi), %edx
+; SSSE3-NEXT:    andl $3, %edx
+; SSSE3-NEXT:    movl 12(%rdi), %esi
+; SSSE3-NEXT:    andl $3, %esi
 ; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: mem_shuffle_v4i32_v4i32_xxxx_i32:
 ; SSE41:       # BB#0:
-; SSE41-NEXT:    movslq (%rdi), %rax
+; SSE41-NEXT:    movl (%rdi), %eax
+; SSE41-NEXT:    movl 4(%rdi), %ecx
+; SSE41-NEXT:    andl $3, %eax
 ; SSE41-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE41-NEXT:    movslq 4(%rdi), %rcx
-; SSE41-NEXT:    movslq 8(%rdi), %rdx
-; SSE41-NEXT:    movslq 12(%rdi), %rsi
+; SSE41-NEXT:    andl $3, %ecx
+; SSE41-NEXT:    movl 8(%rdi), %edx
+; SSE41-NEXT:    andl $3, %edx
+; SSE41-NEXT:    movl 12(%rdi), %esi
+; SSE41-NEXT:    andl $3, %esi
 ; SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE41-NEXT:    pinsrd $1, -24(%rsp,%rcx,4), %xmm0
 ; SSE41-NEXT:    pinsrd $2, -24(%rsp,%rdx,4), %xmm0
 ; SSE41-NEXT:    pinsrd $3, -24(%rsp,%rsi,4), %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: mem_shuffle_v4i32_v4i32_xxxx_i32:
 ; AVX:       # BB#0:
-; AVX-NEXT:    movslq (%rdi), %rax
+; AVX-NEXT:    movl (%rdi), %eax
+; AVX-NEXT:    movl 4(%rdi), %ecx
+; AVX-NEXT:    andl $3, %eax
 ; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX-NEXT:    movslq 4(%rdi), %rcx
-; AVX-NEXT:    movslq 8(%rdi), %rdx
-; AVX-NEXT:    movslq 12(%rdi), %rsi
+; AVX-NEXT:    andl $3, %ecx
+; AVX-NEXT:    movl 8(%rdi), %edx
+; AVX-NEXT:    andl $3, %edx
+; AVX-NEXT:    movl 12(%rdi), %esi
+; AVX-NEXT:    andl $3, %esi
 ; AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    vpinsrd $1, -24(%rsp,%rcx,4), %xmm0, %xmm0
 ; AVX-NEXT:    vpinsrd $2, -24(%rsp,%rdx,4), %xmm0, %xmm0
 ; AVX-NEXT:    vpinsrd $3, -24(%rsp,%rsi,4), %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %p0  = getelementptr inbounds i32, i32* %i, i64 0
   %p1  = getelementptr inbounds i32, i32* %i, i64 1
   %p2  = getelementptr inbounds i32, i32* %i, i64 2
; AVX-NEXT:    retq
   %r2 = insertelement <4 x i32>   %r1, i32 %x2, i32 2
   %r3 = insertelement <4 x i32>   %r2, i32 %x3, i32 3
   ret <4 x i32> %r3
 }
 
 define <16 x i8> @mem_shuffle_v16i8_v16i8_xxxxxxxxxxxxxxxx_i8(<16 x i8> %x, i8* %i) nounwind {
 ; SSE2-LABEL: mem_shuffle_v16i8_v16i8_xxxxxxxxxxxxxxxx_i8:
 ; SSE2:       # BB#0:
-; SSE2-NEXT:    movsbq (%rdi), %rcx
+; SSE2-NEXT:    movzbl (%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
 ; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    leaq -{{[0-9]+}}(%rsp), %rax
-; SSE2-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm0
-; SSE2-NEXT:    movsbq 8(%rdi), %rcx
-; SSE2-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm8
-; SSE2-NEXT:    movsbq 12(%rdi), %rcx
-; SSE2-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm9
-; SSE2-NEXT:    movsbq 4(%rdi), %rcx
-; SSE2-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    movsbq 14(%rdi), %rcx
-; SSE2-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm10
-; SSE2-NEXT:    movsbq 6(%rdi), %rcx
-; SSE2-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm5
-; SSE2-NEXT:    movsbq 10(%rdi), %rcx
-; SSE2-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm11
-; SSE2-NEXT:    movsbq 2(%rdi), %rcx
-; SSE2-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm7
-; SSE2-NEXT:    movsbq 15(%rdi), %rcx
-; SSE2-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm12
-; SSE2-NEXT:    movsbq 7(%rdi), %rcx
-; SSE2-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm2
-; SSE2-NEXT:    movsbq 11(%rdi), %rcx
-; SSE2-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm13
-; SSE2-NEXT:    movsbq 3(%rdi), %rcx
-; SSE2-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm6
-; SSE2-NEXT:    movsbq 13(%rdi), %rcx
-; SSE2-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm14
-; SSE2-NEXT:    movsbq 5(%rdi), %rcx
-; SSE2-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm4
-; SSE2-NEXT:    movsbq 9(%rdi), %rcx
-; SSE2-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSE2-NEXT:    movd %ecx, %xmm15
-; SSE2-NEXT:    movsbq 1(%rdi), %rcx
-; SSE2-NEXT:    movzbl (%rcx,%rax), %eax
+; SSE2-NEXT:    leaq -{{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    movzbl 8(%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
+; SSE2-NEXT:    movd %eax, %xmm8
+; SSE2-NEXT:    movzbl 12(%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
+; SSE2-NEXT:    movd %eax, %xmm9
+; SSE2-NEXT:    movzbl 4(%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
+; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    movzbl 14(%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
+; SSE2-NEXT:    movd %eax, %xmm10
+; SSE2-NEXT:    movzbl 6(%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
+; SSE2-NEXT:    movd %eax, %xmm5
+; SSE2-NEXT:    movzbl 10(%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
+; SSE2-NEXT:    movd %eax, %xmm11
+; SSE2-NEXT:    movzbl 2(%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
+; SSE2-NEXT:    movd %eax, %xmm7
+; SSE2-NEXT:    movzbl 15(%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
+; SSE2-NEXT:    movd %eax, %xmm12
+; SSE2-NEXT:    movzbl 7(%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    movzbl 11(%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
+; SSE2-NEXT:    movd %eax, %xmm13
+; SSE2-NEXT:    movzbl 3(%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
+; SSE2-NEXT:    movd %eax, %xmm6
+; SSE2-NEXT:    movzbl 13(%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
+; SSE2-NEXT:    movd %eax, %xmm14
+; SSE2-NEXT:    movzbl 5(%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
+; SSE2-NEXT:    movd %eax, %xmm4
+; SSE2-NEXT:    movzbl 9(%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
+; SSE2-NEXT:    movd %eax, %xmm15
+; SSE2-NEXT:    movzbl 1(%rdi), %eax
+; SSE2-NEXT:    andl $15, %eax
+; SSE2-NEXT:    movzbl (%rax,%rcx), %eax
 ; SSE2-NEXT:    movd %eax, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3],xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3],xmm3[4],xmm9[4],xmm3[5],xmm9[5],xmm3[6],xmm9[6],xmm3[7],xmm9[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm10[0],xmm5[1],xmm10[1],xmm5[2],xmm10[2],xmm5[3],xmm10[3],xmm5[4],xmm10[4],xmm5[5],xmm10[5],xmm5[6],xmm10[6],xmm5[7],xmm10[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm11[0],xmm7[1],xmm11[1],xmm7[2],xmm11[2],xmm7[3],xmm11[3],xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3],xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1],xmm0[2],xmm7[2],xmm0[3],xmm7[3],xmm0[4],xmm7[4],xmm0[5],xmm7[5],xmm0[6],xmm7[6],xmm0[7],xmm7[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm12[0],xmm2[1],xmm12[1],xmm2[2],xmm12[2],xmm2[3],xmm12[3],xmm2[4],xmm12[4],xmm2[5],xmm12[5],xmm2[6],xmm12[6],xmm2[7],xmm12[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm13[0],xmm6[1],xmm13[1],xmm6[2],xmm13[2],xmm6[3],xmm13[3],xmm6[4],xmm13[4],xmm6[5],xmm13[5],xmm6[6],xmm13[6],xmm6[7],xmm13[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3],xmm4[4],xmm14[4],xmm4[5],xmm14[5],xmm4[6],xmm14[6],xmm4[7],xmm14[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm15[0],xmm1[1],xmm15[1],xmm1[2],xmm15[2],xmm1[3],xmm15[3],xmm1[4],xmm15[4],xmm1[5],xmm15[5],xmm1[6],xmm15[6],xmm1[7],xmm15[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3],xmm1[4],xmm6[4],xmm1[5],xmm6[5],xmm1[6],xmm6[6],xmm1[7],xmm6[7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: mem_shuffle_v16i8_v16i8_xxxxxxxxxxxxxxxx_i8:
 ; SSSE3:       # BB#0:
-; SSSE3-NEXT:    movsbq (%rdi), %rcx
+; SSSE3-NEXT:    movzbl (%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
 ; SSSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSSE3-NEXT:    leaq -{{[0-9]+}}(%rsp), %rax
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm0
-; SSSE3-NEXT:    movsbq 8(%rdi), %rcx
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm8
-; SSSE3-NEXT:    movsbq 12(%rdi), %rcx
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm9
-; SSSE3-NEXT:    movsbq 4(%rdi), %rcx
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm3
-; SSSE3-NEXT:    movsbq 14(%rdi), %rcx
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm10
-; SSSE3-NEXT:    movsbq 6(%rdi), %rcx
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm5
-; SSSE3-NEXT:    movsbq 10(%rdi), %rcx
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm11
-; SSSE3-NEXT:    movsbq 2(%rdi), %rcx
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm7
-; SSSE3-NEXT:    movsbq 15(%rdi), %rcx
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm12
-; SSSE3-NEXT:    movsbq 7(%rdi), %rcx
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm2
-; SSSE3-NEXT:    movsbq 11(%rdi), %rcx
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm13
-; SSSE3-NEXT:    movsbq 3(%rdi), %rcx
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm6
-; SSSE3-NEXT:    movsbq 13(%rdi), %rcx
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm14
-; SSSE3-NEXT:    movsbq 5(%rdi), %rcx
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm4
-; SSSE3-NEXT:    movsbq 9(%rdi), %rcx
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm15
-; SSSE3-NEXT:    movsbq 1(%rdi), %rcx
-; SSSE3-NEXT:    movzbl (%rcx,%rax), %eax
+; SSSE3-NEXT:    leaq -{{[0-9]+}}(%rsp), %rcx
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    movzbl 8(%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
+; SSSE3-NEXT:    movd %eax, %xmm8
+; SSSE3-NEXT:    movzbl 12(%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
+; SSSE3-NEXT:    movd %eax, %xmm9
+; SSSE3-NEXT:    movzbl 4(%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
+; SSSE3-NEXT:    movd %eax, %xmm3
+; SSSE3-NEXT:    movzbl 14(%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
+; SSSE3-NEXT:    movd %eax, %xmm10
+; SSSE3-NEXT:    movzbl 6(%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
+; SSSE3-NEXT:    movd %eax, %xmm5
+; SSSE3-NEXT:    movzbl 10(%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
+; SSSE3-NEXT:    movd %eax, %xmm11
+; SSSE3-NEXT:    movzbl 2(%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
+; SSSE3-NEXT:    movd %eax, %xmm7
+; SSSE3-NEXT:    movzbl 15(%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
+; SSSE3-NEXT:    movd %eax, %xmm12
+; SSSE3-NEXT:    movzbl 7(%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    movzbl 11(%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
+; SSSE3-NEXT:    movd %eax, %xmm13
+; SSSE3-NEXT:    movzbl 3(%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
+; SSSE3-NEXT:    movd %eax, %xmm6
+; SSSE3-NEXT:    movzbl 13(%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
+; SSSE3-NEXT:    movd %eax, %xmm14
+; SSSE3-NEXT:    movzbl 5(%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
+; SSSE3-NEXT:    movd %eax, %xmm4
+; SSSE3-NEXT:    movzbl 9(%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
+; SSSE3-NEXT:    movd %eax, %xmm15
+; SSSE3-NEXT:    movzbl 1(%rdi), %eax
+; SSSE3-NEXT:    andl $15, %eax
+; SSSE3-NEXT:    movzbl (%rax,%rcx), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm1
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3],xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3],xmm3[4],xmm9[4],xmm3[5],xmm9[5],xmm3[6],xmm9[6],xmm3[7],xmm9[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3],xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm10[0],xmm5[1],xmm10[1],xmm5[2],xmm10[2],xmm5[3],xmm10[3],xmm5[4],xmm10[4],xmm5[5],xmm10[5],xmm5[6],xmm10[6],xmm5[7],xmm10[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm11[0],xmm7[1],xmm11[1],xmm7[2],xmm11[2],xmm7[3],xmm11[3],xmm7[4],xmm11[4],xmm7[5],xmm11[5],xmm7[6],xmm11[6],xmm7[7],xmm11[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3],xmm7[4],xmm5[4],xmm7[5],xmm5[5],xmm7[6],xmm5[6],xmm7[7],xmm5[7]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1],xmm0[2],xmm7[2],xmm0[3],xmm7[3],xmm0[4],xmm7[4],xmm0[5],xmm7[5],xmm0[6],xmm7[6],xmm0[7],xmm7[7]

 ; SSE41-LABEL: mem_shuffle_v16i8_v16i8_xxxxxxxxxxxxxxxx_i8:
 ; SSE41:       # BB#0:
 ; SSE41-NEXT:    pushq %rbp
 ; SSE41-NEXT:    pushq %r15
 ; SSE41-NEXT:    pushq %r14
 ; SSE41-NEXT:    pushq %r13
 ; SSE41-NEXT:    pushq %r12
 ; SSE41-NEXT:    pushq %rbx
-; SSE41-NEXT:    movsbq (%rdi), %rax
+; SSE41-NEXT:    movzbl (%rdi), %r11d
+; SSE41-NEXT:    andl $15, %r11d
 ; SSE41-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE41-NEXT:    movsbq 1(%rdi), %r15
-; SSE41-NEXT:    movsbq 2(%rdi), %r8
-; SSE41-NEXT:    movsbq 3(%rdi), %r9
-; SSE41-NEXT:    movsbq 4(%rdi), %r10
-; SSE41-NEXT:    movsbq 5(%rdi), %r11
-; SSE41-NEXT:    movsbq 6(%rdi), %r14
-; SSE41-NEXT:    movsbq 7(%rdi), %r12
-; SSE41-NEXT:    movsbq 8(%rdi), %r13
-; SSE41-NEXT:    movsbq 9(%rdi), %rdx
-; SSE41-NEXT:    movsbq 10(%rdi), %rcx
-; SSE41-NEXT:    movsbq 11(%rdi), %rsi
-; SSE41-NEXT:    movsbq 12(%rdi), %rbx
+; SSE41-NEXT:    movzbl 1(%rdi), %r9d
+; SSE41-NEXT:    andl $15, %r9d
+; SSE41-NEXT:    movzbl 2(%rdi), %eax
+; SSE41-NEXT:    andl $15, %eax
+; SSE41-NEXT:    movq %rax, -{{[0-9]+}}(%rsp) # 8-byte Spill
+; SSE41-NEXT:    movzbl 3(%rdi), %eax
+; SSE41-NEXT:    andl $15, %eax
+; SSE41-NEXT:    movq %rax, -{{[0-9]+}}(%rsp) # 8-byte Spill
+; SSE41-NEXT:    movzbl 4(%rdi), %r14d
+; SSE41-NEXT:    andl $15, %r14d
+; SSE41-NEXT:    movzbl 5(%rdi), %r15d
+; SSE41-NEXT:    andl $15, %r15d
+; SSE41-NEXT:    movzbl 6(%rdi), %r12d
+; SSE41-NEXT:    andl $15, %r12d
+; SSE41-NEXT:    movzbl 7(%rdi), %r13d
+; SSE41-NEXT:    andl $15, %r13d
+; SSE41-NEXT:    movzbl 8(%rdi), %r8d
+; SSE41-NEXT:    andl $15, %r8d
+; SSE41-NEXT:    movzbl 9(%rdi), %eax
+; SSE41-NEXT:    andl $15, %eax
+; SSE41-NEXT:    movzbl 10(%rdi), %ecx
+; SSE41-NEXT:    andl $15, %ecx
+; SSE41-NEXT:    movzbl 11(%rdi), %edx
+; SSE41-NEXT:    andl $15, %edx
+; SSE41-NEXT:    movzbl 12(%rdi), %esi
+; SSE41-NEXT:    andl $15, %esi
 ; SSE41-NEXT:    leaq -{{[0-9]+}}(%rsp), %rbp
-; SSE41-NEXT:    movzbl (%rax,%rbp), %eax
-; SSE41-NEXT:    movd %eax, %xmm0
-; SSE41-NEXT:    movsbq 13(%rdi), %rax
-; SSE41-NEXT:    pinsrb $1, (%r15,%rbp), %xmm0
-; SSE41-NEXT:    movsbq 14(%rdi), %r15
-; SSE41-NEXT:    movsbq 15(%rdi), %rdi
-; SSE41-NEXT:    movzbl (%rdi,%rbp), %edi
-; SSE41-NEXT:    movzbl (%r15,%rbp), %r15d
-; SSE41-NEXT:    movzbl (%rax,%rbp), %eax
-; SSE41-NEXT:    movzbl (%rbx,%rbp), %ebx
+; SSE41-NEXT:    movzbl (%r11,%rbp), %ebx
+; SSE41-NEXT:    movd %ebx, %xmm0
+; SSE41-NEXT:    movzbl 13(%rdi), %r11d
+; SSE41-NEXT:    andl $15, %r11d
+; SSE41-NEXT:    pinsrb $1, (%r9,%rbp), %xmm0
+; SSE41-NEXT:    movzbl 14(%rdi), %ebx
+; SSE41-NEXT:    andl $15, %ebx
+; SSE41-NEXT:    movzbl 15(%rdi), %edi
+; SSE41-NEXT:    andl $15, %edi
+; SSE41-NEXT:    movzbl (%rdi,%rbp), %r10d
+; SSE41-NEXT:    movzbl (%rbx,%rbp), %r9d
+; SSE41-NEXT:    movzbl (%r11,%rbp), %r11d
 ; SSE41-NEXT:    movzbl (%rsi,%rbp), %esi
-; SSE41-NEXT:    movzbl (%rcx,%rbp), %ecx
 ; SSE41-NEXT:    movzbl (%rdx,%rbp), %edx
+; SSE41-NEXT:    movzbl (%rcx,%rbp), %ecx
+; SSE41-NEXT:    movzbl (%rax,%rbp), %eax
+; SSE41-NEXT:    movzbl (%r8,%rbp), %r8d
 ; SSE41-NEXT:    movzbl (%r13,%rbp), %r13d
 ; SSE41-NEXT:    movzbl (%r12,%rbp), %r12d
+; SSE41-NEXT:    movzbl (%r15,%rbp), %r15d
 ; SSE41-NEXT:    movzbl (%r14,%rbp), %r14d
-; SSE41-NEXT:    movzbl (%r11,%rbp), %r11d
-; SSE41-NEXT:    movzbl (%r10,%rbp), %r10d
-; SSE41-NEXT:    movzbl (%r9,%rbp), %r9d
-; SSE41-NEXT:    movzbl (%r8,%rbp), %ebp
+; SSE41-NEXT:    movq -{{[0-9]+}}(%rsp), %rdi # 8-byte Reload
+; SSE41-NEXT:    movzbl (%rdi,%rbp), %edi
+; SSE41-NEXT:    movq -{{[0-9]+}}(%rsp), %rbx # 8-byte Reload
+; SSE41-NEXT:    movzbl (%rbx,%rbp), %ebp
 ; SSE41-NEXT:    pinsrb $2, %ebp, %xmm0
-; SSE41-NEXT:    pinsrb $3, %r9d, %xmm0
-; SSE41-NEXT:    pinsrb $4, %r10d, %xmm0
-; SSE41-NEXT:    pinsrb $5, %r11d, %xmm0
-; SSE41-NEXT:    pinsrb $6, %r14d, %xmm0
-; SSE41-NEXT:    pinsrb $7, %r12d, %xmm0
-; SSE41-NEXT:    pinsrb $8, %r13d, %xmm0
-; SSE41-NEXT:    pinsrb $9, %edx, %xmm0
+; SSE41-NEXT:    pinsrb $3, %edi, %xmm0
+; SSE41-NEXT:    pinsrb $4, %r14d, %xmm0
+; SSE41-NEXT:    pinsrb $5, %r15d, %xmm0
+; SSE41-NEXT:    pinsrb $6, %r12d, %xmm0
+; SSE41-NEXT:    pinsrb $7, %r13d, %xmm0
+; SSE41-NEXT:    pinsrb $8, %r8d, %xmm0
+; SSE41-NEXT:    pinsrb $9, %eax, %xmm0
 ; SSE41-NEXT:    pinsrb $10, %ecx, %xmm0
-; SSE41-NEXT:    pinsrb $11, %esi, %xmm0
-; SSE41-NEXT:    pinsrb $12, %ebx, %xmm0
-; SSE41-NEXT:    pinsrb $13, %eax, %xmm0
-; SSE41-NEXT:    pinsrb $14, %r15d, %xmm0
-; SSE41-NEXT:    pinsrb $15, %edi, %xmm0
+; SSE41-NEXT:    pinsrb $11, %edx, %xmm0
+; SSE41-NEXT:    pinsrb $12, %esi, %xmm0
+; SSE41-NEXT:    pinsrb $13, %r11d, %xmm0
+; SSE41-NEXT:    pinsrb $14, %r9d, %xmm0
+; SSE41-NEXT:    pinsrb $15, %r10d, %xmm0
 ; SSE41-NEXT:    popq %rbx
 ; SSE41-NEXT:    popq %r12
 ; SSE41-NEXT:    popq %r13
 ; SSE41-NEXT:    popq %r14
 ; SSE41-NEXT:    popq %r15
 ; SSE41-NEXT:    popq %rbp
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: mem_shuffle_v16i8_v16i8_xxxxxxxxxxxxxxxx_i8:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    pushq %rbp
 ; AVX-NEXT:    pushq %r15
 ; AVX-NEXT:    pushq %r14
 ; AVX-NEXT:    pushq %r13
 ; AVX-NEXT:    pushq %r12
 ; AVX-NEXT:    pushq %rbx
-; AVX-NEXT:    movsbq (%rdi), %rsi
+; AVX-NEXT:    movzbl (%rdi), %r11d
+; AVX-NEXT:    andl $15, %r11d
 ; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX-NEXT:    movsbq 1(%rdi), %r15
-; AVX-NEXT:    movsbq 2(%rdi), %r8
-; AVX-NEXT:    movsbq 3(%rdi), %r9
-; AVX-NEXT:    movsbq 4(%rdi), %r10
-; AVX-NEXT:    movsbq 5(%rdi), %r11
-; AVX-NEXT:    movsbq 6(%rdi), %r14
-; AVX-NEXT:    movsbq 7(%rdi), %r12
-; AVX-NEXT:    movsbq 8(%rdi), %r13
-; AVX-NEXT:    movsbq 9(%rdi), %rdx
-; AVX-NEXT:    movsbq 10(%rdi), %rax
-; AVX-NEXT:    movsbq 11(%rdi), %rcx
-; AVX-NEXT:    movsbq 12(%rdi), %rbx
+; AVX-NEXT:    movzbl 1(%rdi), %r9d
+; AVX-NEXT:    andl $15, %r9d
+; AVX-NEXT:    movzbl 2(%rdi), %eax
+; AVX-NEXT:    andl $15, %eax
+; AVX-NEXT:    movq %rax, -{{[0-9]+}}(%rsp) # 8-byte Spill
+; AVX-NEXT:    movzbl 3(%rdi), %eax
+; AVX-NEXT:    andl $15, %eax
+; AVX-NEXT:    movq %rax, -{{[0-9]+}}(%rsp) # 8-byte Spill
+; AVX-NEXT:    movzbl 4(%rdi), %r14d
+; AVX-NEXT:    andl $15, %r14d
+; AVX-NEXT:    movzbl 5(%rdi), %r15d
+; AVX-NEXT:    andl $15, %r15d
+; AVX-NEXT:    movzbl 6(%rdi), %r12d
+; AVX-NEXT:    andl $15, %r12d
+; AVX-NEXT:    movzbl 7(%rdi), %r13d
+; AVX-NEXT:    andl $15, %r13d
+; AVX-NEXT:    movzbl 8(%rdi), %r8d
+; AVX-NEXT:    andl $15, %r8d
+; AVX-NEXT:    movzbl 9(%rdi), %eax
+; AVX-NEXT:    andl $15, %eax
+; AVX-NEXT:    movzbl 10(%rdi), %ecx
+; AVX-NEXT:    andl $15, %ecx
+; AVX-NEXT:    movzbl 11(%rdi), %edx
+; AVX-NEXT:    andl $15, %edx
+; AVX-NEXT:    movzbl 12(%rdi), %esi
+; AVX-NEXT:    andl $15, %esi
 ; AVX-NEXT:    leaq -{{[0-9]+}}(%rsp), %rbp
+; AVX-NEXT:    movzbl (%r11,%rbp), %ebx
+; AVX-NEXT:    vmovd %ebx, %xmm0
+; AVX-NEXT:    movzbl 13(%rdi), %r11d
+; AVX-NEXT:    andl $15, %r11d
+; AVX-NEXT:    vpinsrb $1, (%r9,%rbp), %xmm0, %xmm0
+; AVX-NEXT:    movzbl 14(%rdi), %ebx
+; AVX-NEXT:    andl $15, %ebx
+; AVX-NEXT:    movzbl 15(%rdi), %edi
+; AVX-NEXT:    andl $15, %edi
+; AVX-NEXT:    movzbl (%rdi,%rbp), %r10d
+; AVX-NEXT:    movzbl (%rbx,%rbp), %r9d
+; AVX-NEXT:    movzbl (%r11,%rbp), %r11d
 ; AVX-NEXT:    movzbl (%rsi,%rbp), %esi
-; AVX-NEXT:    vmovd %esi, %xmm0
-; AVX-NEXT:    movsbq 13(%rdi), %rsi
-; AVX-NEXT:    vpinsrb $1, (%r15,%rbp), %xmm0, %xmm0
-; AVX-NEXT:    movsbq 14(%rdi), %r15
-; AVX-NEXT:    movsbq 15(%rdi), %rdi
-; AVX-NEXT:    movzbl (%rdi,%rbp), %edi
-; AVX-NEXT:    movzbl (%r15,%rbp), %r15d
-; AVX-NEXT:    movzbl (%rsi,%rbp), %esi
-; AVX-NEXT:    movzbl (%rbx,%rbp), %ebx
+; AVX-NEXT:    movzbl (%rdx,%rbp), %edx
 ; AVX-NEXT:    movzbl (%rcx,%rbp), %ecx
 ; AVX-NEXT:    movzbl (%rax,%rbp), %eax
-; AVX-NEXT:    movzbl (%rdx,%rbp), %edx
+; AVX-NEXT:    movzbl (%r8,%rbp), %r8d
 ; AVX-NEXT:    movzbl (%r13,%rbp), %r13d
 ; AVX-NEXT:    movzbl (%r12,%rbp), %r12d
+; AVX-NEXT:    movzbl (%r15,%rbp), %r15d
 ; AVX-NEXT:    movzbl (%r14,%rbp), %r14d
-; AVX-NEXT:    movzbl (%r11,%rbp), %r11d
-; AVX-NEXT:    movzbl (%r10,%rbp), %r10d
-; AVX-NEXT:    movzbl (%r9,%rbp), %r9d
-; AVX-NEXT:    movzbl (%r8,%rbp), %ebp
+; AVX-NEXT:    movq -{{[0-9]+}}(%rsp), %rdi # 8-byte Reload
+; AVX-NEXT:    movzbl (%rdi,%rbp), %edi
+; AVX-NEXT:    movq -{{[0-9]+}}(%rsp), %rbx # 8-byte Reload
+; AVX-NEXT:    movzbl (%rbx,%rbp), %ebp
 ; AVX-NEXT:    vpinsrb $2, %ebp, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $3, %r9d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $4, %r10d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $5, %r11d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $6, %r14d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $7, %r12d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $8, %r13d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $9, %edx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $10, %eax, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $11, %ecx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $12, %ebx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $13, %esi, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $14, %r15d, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrb $15, %edi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $3, %edi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $4, %r14d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $5, %r15d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $6, %r12d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $7, %r13d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $8, %r8d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $9, %eax, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $10, %ecx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $11, %edx, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $12, %esi, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $13, %r11d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $14, %r9d, %xmm0, %xmm0
+; AVX-NEXT:    vpinsrb $15, %r10d, %xmm0, %xmm0
 ; AVX-NEXT:    popq %rbx
 ; AVX-NEXT:    popq %r12
 ; AVX-NEXT:    popq %r13
 ; AVX-NEXT:    popq %r14
 ; AVX-NEXT:    popq %r15
 ; AVX-NEXT:    popq %rbp
 ; AVX-NEXT:    retq
   %p0  = getelementptr inbounds i8, i8* %i, i64 0

 
 ;
 ; Binary shuffle indices from registers
 ;
 
 define <4 x float> @var_shuffle_v4f32_v4f32_x0yx_i32(<4 x float> %x, <4 x float> %y, i32 %i0, i32 %i1, i32 %i2, i32 %i3) nounwind {
 ; SSE-LABEL: var_shuffle_v4f32_v4f32_x0yx_i32:
 ; SSE:       # BB#0:
-; SSE-NEXT:    movslq %edi, %rax
+; SSE-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; SSE-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; SSE-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
+; SSE-NEXT:    andl $3, %edi
 ; SSE-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE-NEXT:    movslq %edx, %rdx
+; SSE-NEXT:    andl $3, %edx
 ; SSE-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE-NEXT:    movslq %ecx, %rcx
+; SSE-NEXT:    andl $3, %ecx
 ; SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE-NEXT:    movss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: var_shuffle_v4f32_v4f32_x0yx_i32:
 ; AVX:       # BB#0:
-; AVX-NEXT:    movslq %edi, %rax
+; AVX-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; AVX-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; AVX-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
+; AVX-NEXT:    andl $3, %edi
 ; AVX-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
-; AVX-NEXT:    movslq %edx, %rdx
+; AVX-NEXT:    andl $3, %edx
 ; AVX-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX-NEXT:    movslq %ecx, %rcx
+; AVX-NEXT:    andl $3, %ecx
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; AVX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; AVX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX-NEXT:    retq
   %x0 = extractelement <4 x float> %x, i32 %i0
   %x1 = extractelement <4 x float> %x, i32 %i1

 ; SSE2-LABEL: var_shuffle_v8i16_v8i16_xyxyxy00_i16:
 ; SSE2:       # BB#0:
 ; SSE2-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
 ; SSE2-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
 ; SSE2-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
 ; SSE2-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
 ; SSE2-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
 ; SSE2-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; SSE2-NEXT:    movswq %di, %r10
-; SSE2-NEXT:    movswq %si, %rsi
-; SSE2-NEXT:    movswq %dx, %r11
-; SSE2-NEXT:    movswq %cx, %rcx
+; SSE2-NEXT:    andl $7, %edi
+; SSE2-NEXT:    andl $7, %esi
+; SSE2-NEXT:    andl $7, %edx
+; SSE2-NEXT:    andl $7, %ecx
 ; SSE2-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movswq %r8w, %rdi
+; SSE2-NEXT:    andl $7, %r8d
 ; SSE2-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT:    movswq %r9w, %rax
-; SSE2-NEXT:    movzwl -24(%rsp,%rsi,2), %esi
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    andl $7, %r9d
+; SSE2-NEXT:    movzwl -24(%rsp,%rsi,2), %eax
+; SSE2-NEXT:    xorl %esi, %esi
+; SSE2-NEXT:    movd %esi, %xmm0
 ; SSE2-NEXT:    movzwl -24(%rsp,%rcx,2), %ecx
 ; SSE2-NEXT:    movd %ecx, %xmm1
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT:    movd %esi, %xmm2
-; SSE2-NEXT:    movzwl -24(%rsp,%rax,2), %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    movzwl -24(%rsp,%r9,2), %eax
 ; SSE2-NEXT:    movd %eax, %xmm3
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE2-NEXT:    movzwl -40(%rsp,%r10,2), %eax
-; SSE2-NEXT:    movzwl -40(%rsp,%r11,2), %ecx
+; SSE2-NEXT:    movzwl -40(%rsp,%rdi,2), %eax
+; SSE2-NEXT:    movzwl -40(%rsp,%rdx,2), %ecx
 ; SSE2-NEXT:    movd %ecx, %xmm1
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    movzwl -40(%rsp,%rdi,2), %eax
+; SSE2-NEXT:    movzwl -40(%rsp,%r8,2), %eax
 ; SSE2-NEXT:    movd %eax, %xmm3
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: var_shuffle_v8i16_v8i16_xyxyxy00_i16:
 ; SSSE3:       # BB#0:
 ; SSSE3-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
 ; SSSE3-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
 ; SSSE3-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
 ; SSSE3-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
 ; SSSE3-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
 ; SSSE3-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; SSSE3-NEXT:    movswq %di, %r10
-; SSSE3-NEXT:    movswq %si, %rsi
-; SSSE3-NEXT:    movswq %dx, %r11
-; SSSE3-NEXT:    movswq %cx, %rcx
+; SSSE3-NEXT:    andl $7, %edi
+; SSSE3-NEXT:    andl $7, %esi
+; SSSE3-NEXT:    andl $7, %edx
+; SSSE3-NEXT:    andl $7, %ecx
 ; SSSE3-NEXT:    movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSSE3-NEXT:    movswq %r8w, %rdi
+; SSSE3-NEXT:    andl $7, %r8d
 ; SSSE3-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSSE3-NEXT:    movswq %r9w, %rax
-; SSSE3-NEXT:    movzwl -24(%rsp,%rsi,2), %esi
-; SSSE3-NEXT:    xorl %edx, %edx
-; SSSE3-NEXT:    movd %edx, %xmm0
+; SSSE3-NEXT:    andl $7, %r9d
+; SSSE3-NEXT:    movzwl -24(%rsp,%rsi,2), %eax
+; SSSE3-NEXT:    xorl %esi, %esi
+; SSSE3-NEXT:    movd %esi, %xmm0
 ; SSSE3-NEXT:    movzwl -24(%rsp,%rcx,2), %ecx
 ; SSSE3-NEXT:    movd %ecx, %xmm1
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSSE3-NEXT:    movd %esi, %xmm2
-; SSSE3-NEXT:    movzwl -24(%rsp,%rax,2), %eax
+; SSSE3-NEXT:    movd %eax, %xmm2
+; SSSE3-NEXT:    movzwl -24(%rsp,%r9,2), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm3
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSSE3-NEXT:    movzwl -40(%rsp,%r10,2), %eax
-; SSSE3-NEXT:    movzwl -40(%rsp,%r11,2), %ecx
+; SSSE3-NEXT:    movzwl -40(%rsp,%rdi,2), %eax
+; SSSE3-NEXT:    movzwl -40(%rsp,%rdx,2), %ecx
 ; SSSE3-NEXT:    movd %ecx, %xmm1
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSSE3-NEXT:    movd %eax, %xmm0
-; SSSE3-NEXT:    movzwl -40(%rsp,%rdi,2), %eax
+; SSSE3-NEXT:    movzwl -40(%rsp,%r8,2), %eax
 ; SSSE3-NEXT:    movd %eax, %xmm3
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: var_shuffle_v8i16_v8i16_xyxyxy00_i16:
 ; SSE41:       # BB#0:
 ; SSE41-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
 ; SSE41-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
 ; SSE41-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
 ; SSE41-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
 ; SSE41-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
 ; SSE41-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; SSE41-NEXT:    movswq %di, %rax
-; SSE41-NEXT:    movswq %si, %rsi
-; SSE41-NEXT:    movswq %dx, %rdx
-; SSE41-NEXT:    movswq %cx, %r10
+; SSE41-NEXT:    andl $7, %edi
+; SSE41-NEXT:    andl $7, %esi
+; SSE41-NEXT:    andl $7, %edx
+; SSE41-NEXT:    andl $7, %ecx
 ; SSE41-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE41-NEXT:    movswq %r8w, %rdi
+; SSE41-NEXT:    andl $7, %r8d
 ; SSE41-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE41-NEXT:    movswq %r9w, %rcx
-; SSE41-NEXT:    movzwl -40(%rsp,%rax,2), %eax
+; SSE41-NEXT:    andl $7, %r9d
+; SSE41-NEXT:    movzwl -40(%rsp,%rdi,2), %eax
 ; SSE41-NEXT:    movd %eax, %xmm1
 ; SSE41-NEXT:    pinsrw $1, -24(%rsp,%rsi,2), %xmm1
 ; SSE41-NEXT:    pinsrw $2, -40(%rsp,%rdx,2), %xmm1
-; SSE41-NEXT:    pinsrw $3, -24(%rsp,%r10,2), %xmm1
-; SSE41-NEXT:    pinsrw $4, -40(%rsp,%rdi,2), %xmm1
-; SSE41-NEXT:    pinsrw $5, -24(%rsp,%rcx,2), %xmm1
+; SSE41-NEXT:    pinsrw $3, -24(%rsp,%rcx,2), %xmm1
+; SSE41-NEXT:    pinsrw $4, -40(%rsp,%r8,2), %xmm1
+; SSE41-NEXT:    pinsrw $5, -24(%rsp,%r9,2), %xmm1
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5],xmm0[6,7]
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: var_shuffle_v8i16_v8i16_xyxyxy00_i16:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
 ; AVX1-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
 ; AVX1-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
 ; AVX1-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
 ; AVX1-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
 ; AVX1-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; AVX1-NEXT:    movswq %di, %r10
-; AVX1-NEXT:    movswq %si, %r11
-; AVX1-NEXT:    movswq %dx, %rdx
-; AVX1-NEXT:    movswq %cx, %rcx
+; AVX1-NEXT:    andl $7, %edi
+; AVX1-NEXT:    andl $7, %esi
+; AVX1-NEXT:    andl $7, %edx
+; AVX1-NEXT:    andl $7, %ecx
 ; AVX1-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    movswq %r8w, %rdi
+; AVX1-NEXT:    andl $7, %r8d
 ; AVX1-NEXT:    vmovdqa %xmm1, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    movswq %r9w, %rax
-; AVX1-NEXT:    movzwl -40(%rsp,%r10,2), %esi
-; AVX1-NEXT:    vmovd %esi, %xmm0
-; AVX1-NEXT:    vpinsrw $1, -24(%rsp,%r11,2), %xmm0, %xmm0
+; AVX1-NEXT:    andl $7, %r9d
+; AVX1-NEXT:    movzwl -40(%rsp,%rdi,2), %eax
+; AVX1-NEXT:    vmovd %eax, %xmm0
+; AVX1-NEXT:    vpinsrw $1, -24(%rsp,%rsi,2), %xmm0, %xmm0
 ; AVX1-NEXT:    vpinsrw $2, -40(%rsp,%rdx,2), %xmm0, %xmm0
 ; AVX1-NEXT:    vpinsrw $3, -24(%rsp,%rcx,2), %xmm0, %xmm0
-; AVX1-NEXT:    vpinsrw $4, -40(%rsp,%rdi,2), %xmm0, %xmm0
-; AVX1-NEXT:    vpinsrw $5, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX1-NEXT:    vpinsrw $4, -40(%rsp,%r8,2), %xmm0, %xmm0
+; AVX1-NEXT:    vpinsrw $5, -24(%rsp,%r9,2), %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_shuffle_v8i16_v8i16_xyxyxy00_i16:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
 ; AVX2-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
 ; AVX2-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
 ; AVX2-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
 ; AVX2-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
 ; AVX2-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; AVX2-NEXT:    movswq %di, %r10
-; AVX2-NEXT:    movswq %si, %r11
-; AVX2-NEXT:    movswq %dx, %rdx
-; AVX2-NEXT:    movswq %cx, %rcx
+; AVX2-NEXT:    andl $7, %edi
+; AVX2-NEXT:    andl $7, %esi
+; AVX2-NEXT:    andl $7, %edx
+; AVX2-NEXT:    andl $7, %ecx
 ; AVX2-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movswq %r8w, %rdi
+; AVX2-NEXT:    andl $7, %r8d
 ; AVX2-NEXT:    vmovdqa %xmm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movswq %r9w, %rax
-; AVX2-NEXT:    movzwl -40(%rsp,%r10,2), %esi
-; AVX2-NEXT:    vmovd %esi, %xmm0
-; AVX2-NEXT:    vpinsrw $1, -24(%rsp,%r11,2), %xmm0, %xmm0
+; AVX2-NEXT:    andl $7, %r9d
+; AVX2-NEXT:    movzwl -40(%rsp,%rdi,2), %eax
+; AVX2-NEXT:    vmovd %eax, %xmm0
+; AVX2-NEXT:    vpinsrw $1, -24(%rsp,%rsi,2), %xmm0, %xmm0
 ; AVX2-NEXT:    vpinsrw $2, -40(%rsp,%rdx,2), %xmm0, %xmm0
 ; AVX2-NEXT:    vpinsrw $3, -24(%rsp,%rcx,2), %xmm0, %xmm0
-; AVX2-NEXT:    vpinsrw $4, -40(%rsp,%rdi,2), %xmm0, %xmm0
-; AVX2-NEXT:    vpinsrw $5, -24(%rsp,%rax,2), %xmm0, %xmm0
+; AVX2-NEXT:    vpinsrw $4, -40(%rsp,%r8,2), %xmm0, %xmm0
+; AVX2-NEXT:    vpinsrw $5, -24(%rsp,%r9,2), %xmm0, %xmm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
 ; AVX2-NEXT:    retq
   %x0 = extractelement <8 x i16> %x, i16 %i0
   %y1 = extractelement <8 x i16> %y, i16 %i1
   %x2 = extractelement <8 x i16> %x, i16 %i2
   %y3 = extractelement <8 x i16> %y, i16 %i3
   %x4 = extractelement <8 x i16> %x, i16 %i4

test/CodeGen/X86/vector-shuffle-variable-256.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
 
 ;
 ; Unary shuffle indices from registers
 ;
 
 define <4 x double> @var_shuffle_v4f64_v4f64_xxxx_i64(<4 x double> %x, i64 %i0, i64 %i1, i64 %i2, i64 %i3) nounwind {
 ; ALL-LABEL: var_shuffle_v4f64_v4f64_xxxx_i64:
 ; ALL:       # BB#0:
 ; ALL-NEXT:    pushq %rbp
 ; ALL-NEXT:    movq %rsp, %rbp
 ; ALL-NEXT:    andq $-32, %rsp
 ; ALL-NEXT:    subq $64, %rsp
+; ALL-NEXT:    andl $3, %ecx
+; ALL-NEXT:    andl $3, %edx
+; ALL-NEXT:    andl $3, %esi
+; ALL-NEXT:    andl $3, %edi
 ; ALL-NEXT:    vmovaps %ymm0, (%rsp)
 ; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; ALL-NEXT:    vmovhpd {{.*#+}} xmm0 = xmm0[0],mem[0]
 ; ALL-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
 ; ALL-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
 ; ALL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; ALL-NEXT:    movq %rbp, %rsp
 ; ALL-NEXT:    popq %rbp

 
 define <4 x double> @var_shuffle_v4f64_v4f64_uxx0_i64(<4 x double> %x, i64 %i0, i64 %i1, i64 %i2, i64 %i3) nounwind {
 ; ALL-LABEL: var_shuffle_v4f64_v4f64_uxx0_i64:
 ; ALL:       # BB#0:
 ; ALL-NEXT:    pushq %rbp
 ; ALL-NEXT:    movq %rsp, %rbp
 ; ALL-NEXT:    andq $-32, %rsp
 ; ALL-NEXT:    subq $64, %rsp
+; ALL-NEXT:    andl $3, %edx
+; ALL-NEXT:    andl $3, %esi
 ; ALL-NEXT:    vmovaps %ymm0, (%rsp)
 ; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; ALL-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; ALL-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
 ; ALL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; ALL-NEXT:    movq %rbp, %rsp
 ; ALL-NEXT:    popq %rbp
 ; ALL-NEXT:    retq
   %x0 = extractelement <4 x double> %x, i64 %i0
   %x1 = extractelement <4 x double> %x, i64 %i1
   %x2 = extractelement <4 x double> %x, i64 %i2
   %x3 = extractelement <4 x double> %x, i64 %i3
   %r0 = insertelement <4 x double> undef, double undef, i32 0
   %r1 = insertelement <4 x double>   %r0, double   %x1, i32 1
   %r2 = insertelement <4 x double>   %r1, double   %x2, i32 2
   %r3 = insertelement <4 x double>   %r2, double   0.0, i32 3
   ret <4 x double> %r3
 }
 
 define <4 x double> @var_shuffle_v4f64_v2f64_xxxx_i64(<2 x double> %x, i64 %i0, i64 %i1, i64 %i2, i64 %i3) nounwind {
 ; ALL-LABEL: var_shuffle_v4f64_v2f64_xxxx_i64:
 ; ALL:       # BB#0:
+; ALL-NEXT:    andl $1, %ecx
+; ALL-NEXT:    andl $1, %edx
+; ALL-NEXT:    andl $1, %esi
+; ALL-NEXT:    andl $1, %edi
 ; ALL-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
 ; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; ALL-NEXT:    vmovhpd {{.*#+}} xmm0 = xmm0[0],mem[0]
 ; ALL-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
 ; ALL-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
 ; ALL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; ALL-NEXT:    retq
   %x0 = extractelement <2 x double> %x, i64 %i0

 
 define <4 x i64> @var_shuffle_v4i64_v4i64_xxxx_i64(<4 x i64> %x, i64 %i0, i64 %i1, i64 %i2, i64 %i3) nounwind {
 ; AVX1-LABEL: var_shuffle_v4i64_v4i64_xxxx_i64:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    pushq %rbp
 ; AVX1-NEXT:    movq %rsp, %rbp
 ; AVX1-NEXT:    andq $-32, %rsp
 ; AVX1-NEXT:    subq $64, %rsp
+; AVX1-NEXT:    andl $3, %ecx
+; AVX1-NEXT:    andl $3, %edx
+; AVX1-NEXT:    andl $3, %esi
+; AVX1-NEXT:    andl $3, %edi
 ; AVX1-NEXT:    vmovaps %ymm0, (%rsp)
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    movq %rbp, %rsp
 ; AVX1-NEXT:    popq %rbp
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_shuffle_v4i64_v4i64_xxxx_i64:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    pushq %rbp
 ; AVX2-NEXT:    movq %rsp, %rbp
 ; AVX2-NEXT:    andq $-32, %rsp
 ; AVX2-NEXT:    subq $64, %rsp
+; AVX2-NEXT:    andl $3, %ecx
+; AVX2-NEXT:    andl $3, %edx
+; AVX2-NEXT:    andl $3, %esi
+; AVX2-NEXT:    andl $3, %edi
 ; AVX2-NEXT:    vmovaps %ymm0, (%rsp)
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0

 
 define <4 x i64> @var_shuffle_v4i64_v4i64_xx00_i64(<4 x i64> %x, i64 %i0, i64 %i1, i64 %i2, i64 %i3) nounwind {
 ; AVX1-LABEL: var_shuffle_v4i64_v4i64_xx00_i64:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    pushq %rbp
 ; AVX1-NEXT:    movq %rsp, %rbp
 ; AVX1-NEXT:    andq $-32, %rsp
 ; AVX1-NEXT:    subq $64, %rsp
+; AVX1-NEXT:    andl $3, %esi
+; AVX1-NEXT:    andl $3, %edi
 ; AVX1-NEXT:    vmovaps %ymm0, (%rsp)
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    movq %rbp, %rsp
 ; AVX1-NEXT:    popq %rbp
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_shuffle_v4i64_v4i64_xx00_i64:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    pushq %rbp
 ; AVX2-NEXT:    movq %rsp, %rbp
 ; AVX2-NEXT:    andq $-32, %rsp
 ; AVX2-NEXT:    subq $64, %rsp
+; AVX2-NEXT:    andl $3, %esi
+; AVX2-NEXT:    andl $3, %edi
 ; AVX2-NEXT:    vmovaps %ymm0, (%rsp)
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    movq %rbp, %rsp
 ; AVX2-NEXT:    popq %rbp
 ; AVX2-NEXT:    retq
   %x0 = extractelement <4 x i64> %x, i64 %i0
   %x1 = extractelement <4 x i64> %x, i64 %i1
   %x2 = extractelement <4 x i64> %x, i64 %i2
   %x3 = extractelement <4 x i64> %x, i64 %i3
   %r0 = insertelement <4 x i64> undef, i64 %x0, i32 0
   %r1 = insertelement <4 x i64>   %r0, i64 %x1, i32 1
   %r2 = insertelement <4 x i64>   %r1, i64   0, i32 2
   %r3 = insertelement <4 x i64>   %r2, i64   0, i32 3
   ret <4 x i64> %r3
 }
 
 define <4 x i64> @var_shuffle_v4i64_v2i64_xxxx_i64(<2 x i64> %x, i64 %i0, i64 %i1, i64 %i2, i64 %i3) nounwind {
 ; AVX1-LABEL: var_shuffle_v4i64_v2i64_xxxx_i64:
 ; AVX1:       # BB#0:
+; AVX1-NEXT:    andl $1, %ecx
+; AVX1-NEXT:    andl $1, %edx
+; AVX1-NEXT:    andl $1, %esi
+; AVX1-NEXT:    andl $1, %edi
 ; AVX1-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_shuffle_v4i64_v2i64_xxxx_i64:
 ; AVX2:       # BB#0:
+; AVX2-NEXT:    andl $1, %ecx
+; AVX2-NEXT:    andl $1, %edx
+; AVX2-NEXT:    andl $1, %esi
+; AVX2-NEXT:    andl $1, %edi
 ; AVX2-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0

 
 define <8 x float> @var_shuffle_v8f32_v8f32_xxxxxxxx_i32(<8 x float> %x, i32 %i0, i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5, i32 %i6, i32 %i7) nounwind {
 ; AVX1-LABEL: var_shuffle_v8f32_v8f32_xxxxxxxx_i32:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    pushq %rbp
 ; AVX1-NEXT:    movq %rsp, %rbp
 ; AVX1-NEXT:    andq $-32, %rsp
 ; AVX1-NEXT:    subq $64, %rsp
-; AVX1-NEXT:    movslq %edi, %rax
-; AVX1-NEXT:    movslq %esi, %rsi
-; AVX1-NEXT:    movslq %edx, %rdx
-; AVX1-NEXT:    movslq %ecx, %r11
-; AVX1-NEXT:    movslq %r8d, %r10
+; AVX1-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
+; AVX1-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
+; AVX1-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; AVX1-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; AVX1-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; AVX1-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
+; AVX1-NEXT:    andl $7, %edi
+; AVX1-NEXT:    andl $7, %esi
+; AVX1-NEXT:    andl $7, %edx
+; AVX1-NEXT:    andl $7, %ecx
+; AVX1-NEXT:    andl $7, %r8d
 ; AVX1-NEXT:    vmovaps %ymm0, (%rsp)
-; AVX1-NEXT:    movslq %r9d, %r8
-; AVX1-NEXT:    movslq 16(%rbp), %rdi
-; AVX1-NEXT:    movslq 24(%rbp), %rcx
+; AVX1-NEXT:    andl $7, %r9d
+; AVX1-NEXT:    movl 16(%rbp), %r10d
+; AVX1-NEXT:    andl $7, %r10d
+; AVX1-NEXT:    movl 24(%rbp), %eax
+; AVX1-NEXT:    andl $7, %eax
 ; AVX1-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX1-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
 ; AVX1-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
 ; AVX1-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[2,3]
; AVX2-NEXT:    retq
   %r6 = insertelement <8 x float>   %r5, float %x6, i32 6
   %r7 = insertelement <8 x float>   %r6, float %x7, i32 7
   ret <8 x float> %r7
 }
 
 define <8 x float> @var_shuffle_v8f32_v4f32_xxxxxxxx_i32(<4 x float> %x, i32 %i0, i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5, i32 %i6, i32 %i7) nounwind {
 ; ALL-LABEL: var_shuffle_v8f32_v4f32_xxxxxxxx_i32:
 ; ALL:       # BB#0:
-; ALL-NEXT:    movslq %edi, %rax
-; ALL-NEXT:    movslq %esi, %rsi
-; ALL-NEXT:    movslq %edx, %rdx
-; ALL-NEXT:    movslq %ecx, %r11
-; ALL-NEXT:    movslq %r8d, %r10
+; ALL-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
+; ALL-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
+; ALL-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; ALL-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; ALL-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; ALL-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
+; ALL-NEXT:    andl $3, %edi
+; ALL-NEXT:    andl $3, %esi
+; ALL-NEXT:    andl $3, %edx
+; ALL-NEXT:    andl $3, %ecx
+; ALL-NEXT:    andl $3, %r8d
 ; ALL-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; ALL-NEXT:    movslq %r9d, %r8
-; ALL-NEXT:    movslq {{[0-9]+}}(%rsp), %rdi
-; ALL-NEXT:    movslq {{[0-9]+}}(%rsp), %rcx
+; ALL-NEXT:    andl $3, %r9d
+; ALL-NEXT:    movl {{[0-9]+}}(%rsp), %r10d
+; ALL-NEXT:    andl $3, %r10d
+; ALL-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; ALL-NEXT:    andl $3, %eax
 ; ALL-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ALL-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; ALL-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; ALL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]
 ; ALL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]
 ; ALL-NEXT:    vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]
 ; ALL-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
 ; ALL-NEXT:    vinsertps {{.*#+}} xmm3 = xmm3[0],mem[0],xmm3[2,3]

 
 define <16 x i16> @var_shuffle_v16i16_v16i16_xxxxxxxxxxxxxxxx_i16(<16 x i16> %x, i32 %i0, i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5, i32 %i6, i32 %i7, i32 %i8, i32 %i9, i32 %i10, i32 %i11, i32 %i12, i32 %i13, i32 %i14, i32 %i15) nounwind {
 ; AVX1-LABEL: var_shuffle_v16i16_v16i16_xxxxxxxxxxxxxxxx_i16:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    pushq %rbp
 ; AVX1-NEXT:    movq %rsp, %rbp
 ; AVX1-NEXT:    andq $-32, %rsp
 ; AVX1-NEXT:    subq $64, %rsp
+; AVX1-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
+; AVX1-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
+; AVX1-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; AVX1-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; AVX1-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; AVX1-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
 ; AVX1-NEXT:    vmovaps %ymm0, (%rsp)
-; AVX1-NEXT:    movslq 32(%rbp), %rax
+; AVX1-NEXT:    movl 32(%rbp), %eax
+; AVX1-NEXT:    andl $15, %eax
 ; AVX1-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    movslq 40(%rbp), %rax
+; AVX1-NEXT:    movl 40(%rbp), %eax
+; AVX1-NEXT:    andl $15, %eax
 ; AVX1-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movslq 48(%rbp), %rax
+; AVX1-NEXT:    movl 48(%rbp), %eax
+; AVX1-NEXT:    andl $15, %eax
 ; AVX1-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movslq 56(%rbp), %rax
+; AVX1-NEXT:    movl 56(%rbp), %eax
+; AVX1-NEXT:    andl $15, %eax
 ; AVX1-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $3, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movslq 64(%rbp), %rax
+; AVX1-NEXT:    movl 64(%rbp), %eax
+; AVX1-NEXT:    andl $15, %eax
 ; AVX1-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movslq 72(%rbp), %rax
+; AVX1-NEXT:    movl 72(%rbp), %eax
+; AVX1-NEXT:    andl $15, %eax
 ; AVX1-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movslq 80(%rbp), %rax
+; AVX1-NEXT:    movl 80(%rbp), %eax
+; AVX1-NEXT:    andl $15, %eax
 ; AVX1-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movslq 88(%rbp), %rax
+; AVX1-NEXT:    movl 88(%rbp), %eax
+; AVX1-NEXT:    andl $15, %eax
 ; AVX1-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movslq %edi, %rax
-; AVX1-NEXT:    movzwl (%rsp,%rax,2), %eax
+; AVX1-NEXT:    andl $15, %edi
+; AVX1-NEXT:    movzwl (%rsp,%rdi,2), %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm1
-; AVX1-NEXT:    movslq %esi, %rax
-; AVX1-NEXT:    vpinsrw $1, (%rsp,%rax,2), %xmm1, %xmm1
-; AVX1-NEXT:    movslq %edx, %rax
-; AVX1-NEXT:    vpinsrw $2, (%rsp,%rax,2), %xmm1, %xmm1
-; AVX1-NEXT:    movslq %ecx, %rax
-; AVX1-NEXT:    vpinsrw $3, (%rsp,%rax,2), %xmm1, %xmm1
-; AVX1-NEXT:    movslq %r8d, %rax
-; AVX1-NEXT:    vpinsrw $4, (%rsp,%rax,2), %xmm1, %xmm1
-; AVX1-NEXT:    movslq %r9d, %rax
-; AVX1-NEXT:    vpinsrw $5, (%rsp,%rax,2), %xmm1, %xmm1
-; AVX1-NEXT:    movslq 16(%rbp), %rax
+; AVX1-NEXT:    andl $15, %esi
+; AVX1-NEXT:    vpinsrw $1, (%rsp,%rsi,2), %xmm1, %xmm1
+; AVX1-NEXT:    andl $15, %edx
+; AVX1-NEXT:    vpinsrw $2, (%rsp,%rdx,2), %xmm1, %xmm1
+; AVX1-NEXT:    andl $15, %ecx
+; AVX1-NEXT:    vpinsrw $3, (%rsp,%rcx,2), %xmm1, %xmm1
+; AVX1-NEXT:    andl $15, %r8d
+; AVX1-NEXT:    vpinsrw $4, (%rsp,%r8,2), %xmm1, %xmm1
+; AVX1-NEXT:    andl $15, %r9d
+; AVX1-NEXT:    vpinsrw $5, (%rsp,%r9,2), %xmm1, %xmm1
+; AVX1-NEXT:    movl 16(%rbp), %eax
+; AVX1-NEXT:    andl $15, %eax
 ; AVX1-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $6, %eax, %xmm1, %xmm1
-; AVX1-NEXT:    movslq 24(%rbp), %rax
+; AVX1-NEXT:    movl 24(%rbp), %eax
+; AVX1-NEXT:    andl $15, %eax
 ; AVX1-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    movq %rbp, %rsp
 ; AVX1-NEXT:    popq %rbp
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_shuffle_v16i16_v16i16_xxxxxxxxxxxxxxxx_i16:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    pushq %rbp
 ; AVX2-NEXT:    movq %rsp, %rbp
 ; AVX2-NEXT:    andq $-32, %rsp
 ; AVX2-NEXT:    subq $64, %rsp
+; AVX2-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
+; AVX2-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
+; AVX2-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; AVX2-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; AVX2-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; AVX2-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
 ; AVX2-NEXT:    vmovaps %ymm0, (%rsp)
-; AVX2-NEXT:    movslq 32(%rbp), %rax
+; AVX2-NEXT:    movl 32(%rbp), %eax
+; AVX2-NEXT:    andl $15, %eax
 ; AVX2-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vmovd %eax, %xmm0
-; AVX2-NEXT:    movslq 40(%rbp), %rax
+; AVX2-NEXT:    movl 40(%rbp), %eax
+; AVX2-NEXT:    andl $15, %eax
 ; AVX2-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movslq 48(%rbp), %rax
+; AVX2-NEXT:    movl 48(%rbp), %eax
+; AVX2-NEXT:    andl $15, %eax
 ; AVX2-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movslq 56(%rbp), %rax
+; AVX2-NEXT:    movl 56(%rbp), %eax
+; AVX2-NEXT:    andl $15, %eax
 ; AVX2-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $3, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movslq 64(%rbp), %rax
+; AVX2-NEXT:    movl 64(%rbp), %eax
+; AVX2-NEXT:    andl $15, %eax
 ; AVX2-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movslq 72(%rbp), %rax
+; AVX2-NEXT:    movl 72(%rbp), %eax
+; AVX2-NEXT:    andl $15, %eax
 ; AVX2-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movslq 80(%rbp), %rax
+; AVX2-NEXT:    movl 80(%rbp), %eax
+; AVX2-NEXT:    andl $15, %eax
 ; AVX2-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movslq 88(%rbp), %rax
+; AVX2-NEXT:    movl 88(%rbp), %eax
+; AVX2-NEXT:    andl $15, %eax
 ; AVX2-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movslq %edi, %rax
-; AVX2-NEXT:    movzwl (%rsp,%rax,2), %eax
+; AVX2-NEXT:    andl $15, %edi
+; AVX2-NEXT:    movzwl (%rsp,%rdi,2), %eax
 ; AVX2-NEXT:    vmovd %eax, %xmm1
-; AVX2-NEXT:    movslq %esi, %rax
-; AVX2-NEXT:    vpinsrw $1, (%rsp,%rax,2), %xmm1, %xmm1
-; AVX2-NEXT:    movslq %edx, %rax
-; AVX2-NEXT:    vpinsrw $2, (%rsp,%rax,2), %xmm1, %xmm1
-; AVX2-NEXT:    movslq %ecx, %rax
-; AVX2-NEXT:    vpinsrw $3, (%rsp,%rax,2), %xmm1, %xmm1
-; AVX2-NEXT:    movslq %r8d, %rax
-; AVX2-NEXT:    vpinsrw $4, (%rsp,%rax,2), %xmm1, %xmm1
-; AVX2-NEXT:    movslq %r9d, %rax
-; AVX2-NEXT:    vpinsrw $5, (%rsp,%rax,2), %xmm1, %xmm1
-; AVX2-NEXT:    movslq 16(%rbp), %rax
+; AVX2-NEXT:    andl $15, %esi
+; AVX2-NEXT:    vpinsrw $1, (%rsp,%rsi,2), %xmm1, %xmm1
+; AVX2-NEXT:    andl $15, %edx
+; AVX2-NEXT:    vpinsrw $2, (%rsp,%rdx,2), %xmm1, %xmm1
+; AVX2-NEXT:    andl $15, %ecx
+; AVX2-NEXT:    vpinsrw $3, (%rsp,%rcx,2), %xmm1, %xmm1
+; AVX2-NEXT:    andl $15, %r8d
+; AVX2-NEXT:    vpinsrw $4, (%rsp,%r8,2), %xmm1, %xmm1
+; AVX2-NEXT:    andl $15, %r9d
+; AVX2-NEXT:    vpinsrw $5, (%rsp,%r9,2), %xmm1, %xmm1
+; AVX2-NEXT:    movl 16(%rbp), %eax
+; AVX2-NEXT:    andl $15, %eax
 ; AVX2-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $6, %eax, %xmm1, %xmm1
-; AVX2-NEXT:    movslq 24(%rbp), %rax
+; AVX2-NEXT:    movl 24(%rbp), %eax
+; AVX2-NEXT:    andl $15, %eax
 ; AVX2-NEXT:    movzwl (%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm1
 ; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
 ; AVX2-NEXT:    movq %rbp, %rsp
 ; AVX2-NEXT:    popq %rbp
 ; AVX2-NEXT:    retq
   %x0  = extractelement <16 x i16> %x, i32 %i0
   %x1  = extractelement <16 x i16> %x, i32 %i1
; AVX2-NEXT:    retq
   %r14 = insertelement <16 x i16>  %r13, i16 %x14, i32 14
   %r15 = insertelement <16 x i16>  %r14, i16 %x15, i32 15
   ret <16 x i16> %r15
 }
 
 define <16 x i16> @var_shuffle_v16i16_v8i16_xxxxxxxxxxxxxxxx_i16(<8 x i16> %x, i32 %i0, i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5, i32 %i6, i32 %i7, i32 %i8, i32 %i9, i32 %i10, i32 %i11, i32 %i12, i32 %i13, i32 %i14, i32 %i15) nounwind {
 ; AVX1-LABEL: var_shuffle_v16i16_v8i16_xxxxxxxxxxxxxxxx_i16:
 ; AVX1:       # BB#0:
+; AVX1-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
+; AVX1-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
+; AVX1-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; AVX1-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; AVX1-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; AVX1-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
 ; AVX1-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX1-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX1-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX1-NEXT:    andl $7, %eax
 ; AVX1-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX1-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX1-NEXT:    andl $7, %eax
 ; AVX1-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX1-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX1-NEXT:    andl $7, %eax
 ; AVX1-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX1-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX1-NEXT:    andl $7, %eax
 ; AVX1-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $3, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX1-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX1-NEXT:    andl $7, %eax
 ; AVX1-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX1-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX1-NEXT:    andl $7, %eax
 ; AVX1-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX1-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX1-NEXT:    andl $7, %eax
 ; AVX1-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX1-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX1-NEXT:    andl $7, %eax
 ; AVX1-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0
-; AVX1-NEXT:    movslq %edi, %rax
-; AVX1-NEXT:    movzwl -24(%rsp,%rax,2), %eax
+; AVX1-NEXT:    andl $7, %edi
+; AVX1-NEXT:    movzwl -24(%rsp,%rdi,2), %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm1
-; AVX1-NEXT:    movslq %esi, %rax
-; AVX1-NEXT:    vpinsrw $1, -24(%rsp,%rax,2), %xmm1, %xmm1
-; AVX1-NEXT:    movslq %edx, %rax
-; AVX1-NEXT:    vpinsrw $2, -24(%rsp,%rax,2), %xmm1, %xmm1
-; AVX1-NEXT:    movslq %ecx, %rax
-; AVX1-NEXT:    vpinsrw $3, -24(%rsp,%rax,2), %xmm1, %xmm1
-; AVX1-NEXT:    movslq %r8d, %rax
-; AVX1-NEXT:    vpinsrw $4, -24(%rsp,%rax,2), %xmm1, %xmm1
-; AVX1-NEXT:    movslq %r9d, %rax
-; AVX1-NEXT:    vpinsrw $5, -24(%rsp,%rax,2), %xmm1, %xmm1
-; AVX1-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX1-NEXT:    andl $7, %esi
+; AVX1-NEXT:    vpinsrw $1, -24(%rsp,%rsi,2), %xmm1, %xmm1
+; AVX1-NEXT:    andl $7, %edx
+; AVX1-NEXT:    vpinsrw $2, -24(%rsp,%rdx,2), %xmm1, %xmm1
+; AVX1-NEXT:    andl $7, %ecx
+; AVX1-NEXT:    vpinsrw $3, -24(%rsp,%rcx,2), %xmm1, %xmm1
+; AVX1-NEXT:    andl $7, %r8d
+; AVX1-NEXT:    vpinsrw $4, -24(%rsp,%r8,2), %xmm1, %xmm1
+; AVX1-NEXT:    andl $7, %r9d
+; AVX1-NEXT:    vpinsrw $5, -24(%rsp,%r9,2), %xmm1, %xmm1
+; AVX1-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX1-NEXT:    andl $7, %eax
 ; AVX1-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $6, %eax, %xmm1, %xmm1
-; AVX1-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX1-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX1-NEXT:    andl $7, %eax
 ; AVX1-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX1-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_shuffle_v16i16_v8i16_xxxxxxxxxxxxxxxx_i16:
 ; AVX2:       # BB#0:
+; AVX2-NEXT:    # kill: %R9D<def> %R9D<kill> %R9<def>
+; AVX2-NEXT:    # kill: %R8D<def> %R8D<kill> %R8<def>
+; AVX2-NEXT:    # kill: %ECX<def> %ECX<kill> %RCX<def>
+; AVX2-NEXT:    # kill: %EDX<def> %EDX<kill> %RDX<def>
+; AVX2-NEXT:    # kill: %ESI<def> %ESI<kill> %RSI<def>
+; AVX2-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
 ; AVX2-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT:    andl $7, %eax
 ; AVX2-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vmovd %eax, %xmm0
-; AVX2-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT:    andl $7, %eax
 ; AVX2-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $1, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT:    andl $7, %eax
 ; AVX2-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT:    andl $7, %eax
 ; AVX2-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $3, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT:    andl $7, %eax
 ; AVX2-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT:    andl $7, %eax
 ; AVX2-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $5, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT:    andl $7, %eax
 ; AVX2-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT:    andl $7, %eax
 ; AVX2-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0
-; AVX2-NEXT:    movslq %edi, %rax
-; AVX2-NEXT:    movzwl -24(%rsp,%rax,2), %eax
+; AVX2-NEXT:    andl $7, %edi
+; AVX2-NEXT:    movzwl -24(%rsp,%rdi,2), %eax
 ; AVX2-NEXT:    vmovd %eax, %xmm1
-; AVX2-NEXT:    movslq %esi, %rax
-; AVX2-NEXT:    vpinsrw $1, -24(%rsp,%rax,2), %xmm1, %xmm1
-; AVX2-NEXT:    movslq %edx, %rax
-; AVX2-NEXT:    vpinsrw $2, -24(%rsp,%rax,2), %xmm1, %xmm1
-; AVX2-NEXT:    movslq %ecx, %rax
-; AVX2-NEXT:    vpinsrw $3, -24(%rsp,%rax,2), %xmm1, %xmm1
-; AVX2-NEXT:    movslq %r8d, %rax
-; AVX2-NEXT:    vpinsrw $4, -24(%rsp,%rax,2), %xmm1, %xmm1
-; AVX2-NEXT:    movslq %r9d, %rax
-; AVX2-NEXT:    vpinsrw $5, -24(%rsp,%rax,2), %xmm1, %xmm1
-; AVX2-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT:    andl $7, %esi
+; AVX2-NEXT:    vpinsrw $1, -24(%rsp,%rsi,2), %xmm1, %xmm1
+; AVX2-NEXT:    andl $7, %edx
+; AVX2-NEXT:    vpinsrw $2, -24(%rsp,%rdx,2), %xmm1, %xmm1
+; AVX2-NEXT:    andl $7, %ecx
+; AVX2-NEXT:    vpinsrw $3, -24(%rsp,%rcx,2), %xmm1, %xmm1
+; AVX2-NEXT:    andl $7, %r8d
+; AVX2-NEXT:    vpinsrw $4, -24(%rsp,%r8,2), %xmm1, %xmm1
+; AVX2-NEXT:    andl $7, %r9d
+; AVX2-NEXT:    vpinsrw $5, -24(%rsp,%r9,2), %xmm1, %xmm1
+; AVX2-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT:    andl $7, %eax
 ; AVX2-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $6, %eax, %xmm1, %xmm1
-; AVX2-NEXT:    movslq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT:    andl $7, %eax
 ; AVX2-NEXT:    movzwl -24(%rsp,%rax,2), %eax
 ; AVX2-NEXT:    vpinsrw $7, %eax, %xmm1, %xmm1
 ; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
   %x0  = extractelement <8 x i16> %x, i32 %i0
   %x1  = extractelement <8 x i16> %x, i32 %i1
   %x2  = extractelement <8 x i16> %x, i32 %i2
   %x3  = extractelement <8 x i16> %x, i32 %i3

 ; AVX1-LABEL: mem_shuffle_v4i64_v4i64_xxxx_i64:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    pushq %rbp
 ; AVX1-NEXT:    movq %rsp, %rbp
 ; AVX1-NEXT:    andq $-32, %rsp
 ; AVX1-NEXT:    subq $64, %rsp
 ; AVX1-NEXT:    movq (%rdi), %rax
 ; AVX1-NEXT:    movq 8(%rdi), %rcx
+; AVX1-NEXT:    andl $3, %eax
+; AVX1-NEXT:    andl $3, %ecx
 ; AVX1-NEXT:    movq 16(%rdi), %rdx
+; AVX1-NEXT:    andl $3, %edx
 ; AVX1-NEXT:    movq 24(%rdi), %rsi
+; AVX1-NEXT:    andl $3, %esi
 ; AVX1-NEXT:    vmovaps %ymm0, (%rsp)
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    movq %rbp, %rsp
 ; AVX1-NEXT:    popq %rbp
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: mem_shuffle_v4i64_v4i64_xxxx_i64:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    pushq %rbp
 ; AVX2-NEXT:    movq %rsp, %rbp
 ; AVX2-NEXT:    andq $-32, %rsp
 ; AVX2-NEXT:    subq $64, %rsp
 ; AVX2-NEXT:    movq (%rdi), %rax
 ; AVX2-NEXT:    movq 8(%rdi), %rcx
+; AVX2-NEXT:    andl $3, %eax
+; AVX2-NEXT:    andl $3, %ecx
 ; AVX2-NEXT:    movq 16(%rdi), %rdx
+; AVX2-NEXT:    andl $3, %edx
 ; AVX2-NEXT:    movq 24(%rdi), %rsi
+; AVX2-NEXT:    andl $3, %esi
 ; AVX2-NEXT:    vmovaps %ymm0, (%rsp)
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX2-NEXT:    retq
   ret <4 x i64> %r3
 }
 
 define <4 x i64> @mem_shuffle_v4i64_v2i64_xxxx_i64(<2 x i64> %x, i64* %i) nounwind {
 ; AVX1-LABEL: mem_shuffle_v4i64_v2i64_xxxx_i64:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    movq (%rdi), %rax
 ; AVX1-NEXT:    movq 8(%rdi), %rcx
+; AVX1-NEXT:    andl $1, %eax
+; AVX1-NEXT:    andl $1, %ecx
 ; AVX1-NEXT:    movq 16(%rdi), %rdx
+; AVX1-NEXT:    andl $1, %edx
 ; AVX1-NEXT:    movq 24(%rdi), %rsi
+; AVX1-NEXT:    andl $1, %esi
 ; AVX1-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; AVX1-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: mem_shuffle_v4i64_v2i64_xxxx_i64:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    movq (%rdi), %rax
 ; AVX2-NEXT:    movq 8(%rdi), %rcx
+; AVX2-NEXT:    andl $1, %eax
+; AVX2-NEXT:    andl $1, %ecx
 ; AVX2-NEXT:    movq 16(%rdi), %rdx
+; AVX2-NEXT:    andl $1, %edx
 ; AVX2-NEXT:    movq 24(%rdi), %rsi
+; AVX2-NEXT:    andl $1, %esi
 ; AVX2-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; AVX2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0