[X86][SSE] Allow folding of store/zext with PEXTRW of 0'th element

[RKSimon]

Summary

Under normal circumstances we prefer the higher performance MOVD to extract the 0'th element of a v8i16 vector instead of PEXTRW.

But as detailed on PR27265, this prevents the SSE41 implementation of PEXTRW from folding the store of the 0'th element. Additionally it prevents us from making use of the fact that the (SSE2) reg-reg version of PEXTRW implicitly zero-extends the i16 element to the i32/i64 destination register.

This patch only preferentially lowers to MOVD if we will not be zero-extending the extracted i16, nor prevent a store from being folded (on SSSE41).

I can commit these separately but included them together for review.

Fix for PR27265.

[spatel]

LGTM - one refactoring suggestion inline.

lib/Target/X86/X86ISelLowering.cpp

12531–12545

Can you pull this chunk (if bitsize == 16 {}) out as a helper function so we don't have 2 copies of it?

lib/Target/X86/X86ISelLowering.cpp

 static bool MayFoldLoad(SDValue Op) {
   return Op.hasOneUse() && ISD::isNormalLoad(Op.getNode());
 }
 
 static bool MayFoldIntoStore(SDValue Op) {
   return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin());
 }
 
+static bool MayFoldIntoZeroExtend(SDValue Op) {
+  if (Op.hasOneUse()) {
+    unsigned Opcode = Op.getNode()->use_begin()->getOpcode();
+    return (ISD::ZERO_EXTEND == Opcode);
+  }
+  return false;
+}
+
 static bool isTargetShuffle(unsigned Opcode) {
   switch(Opcode) {
   default: return false;
   case X86ISD::BLENDI:
   case X86ISD::PSHUFB:
   case X86ISD::PSHUFD:
   case X86ISD::PSHUFHW:
   case X86ISD::PSHUFLW:
if (VT.getSizeInBits() == 8) {
     SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32,
                                   Op.getOperand(0), Op.getOperand(1));
     SDValue Assert  = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract,
                                   DAG.getValueType(VT));
     return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
   }
 
   if (VT.getSizeInBits() == 16) {
-    // If Idx is 0, it's cheaper to do a move instead of a pextrw.
-    if (isNullConstant(Op.getOperand(1)))
+    // If Idx is 0, it's cheaper to do a move instead of a pextrw, unless we're
+    // going to fold the store or zero extend the register.
+    if (isNullConstant(Op.getOperand(1)) && !MayFoldIntoStore(Op) &&
+        !MayFoldIntoZeroExtend(Op))
       return DAG.getNode(
           ISD::TRUNCATE, dl, MVT::i16,
           DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
                       DAG.getBitcast(MVT::v4i32, Op.getOperand(0)),
                       Op.getOperand(1)));
     SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32,
                                   Op.getOperand(0), Op.getOperand(1));
     SDValue Assert  = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract,
X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
   assert(VecVT.is128BitVector() && "Unexpected vector length");
 
   if (Subtarget.hasSSE41())
     if (SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG))
       return Res;
 
   MVT VT = Op.getSimpleValueType();
   // TODO: handle v16i8.
   if (VT.getSizeInBits() == 16) {
-    if (IdxVal == 0)
+    // If Idx is 0, it's cheaper to do a move instead of a pextrw, unless we're
+    // going to zero extend the register.
+    if (IdxVal == 0 && !MayFoldIntoZeroExtend(Op))
       return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
                                      DAG.getBitcast(MVT::v4i32, Vec), Idx));
 
     // Transform it so it match pextrw which produces a 32-bit result.
     MVT EltVT = MVT::i32;
     SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, EltVT, Vec, Idx);
     SDValue Assert  = DAG.getNode(ISD::AssertZext, dl, EltVT, Extract,
                                   DAG.getValueType(VT));
     return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
   }
 
   if (VT.getSizeInBits() == 32) {
     if (IdxVal == 0)
       return Op;
 
     // SHUFPS the element to the lowest double word, then movss.
     int Mask[4] = { static_cast<int>(IdxVal), -1, -1, -1 };
     Vec = DAG.getVectorShuffle(VecVT, dl, Vec, DAG.getUNDEF(VecVT), Mask);

test/CodeGen/X86/2011-12-8-bitcastintprom.ll

 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    movw %ax, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: prom_bug:
 ; SSE41:       ## BB#0:
 ; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    movw %ax, (%rdi)
+; SSE41-NEXT:    pextrw $0, %xmm0, (%rdi)
 ; SSE41-NEXT:    retq
   %r = bitcast <4 x i8> %t to <2 x i16>
   %o = extractelement <2 x i16> %r, i32 0
   store i16 %o, i16* %p
   ret void
 }
 

test/CodeGen/X86/avx512-trunc.ll

; ALL-NEXT:    retq
   %x = trunc <2 x i64> %i to <2 x i8>
   ret <2 x i8> %x
 }
 
 define void @trunc_qb_128_mem(<2 x i64> %i, <2 x i8>* %res) #0 {
 ; KNL-LABEL: trunc_qb_128_mem:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; KNL-NEXT:    vmovd %xmm0, %eax
-; KNL-NEXT:    movw %ax, (%rdi)
+; KNL-NEXT:    vpextrw $0, %xmm0, (%rdi)
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: trunc_qb_128_mem:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    vpmovqb %xmm0, (%rdi)
 ; SKX-NEXT:    retq
     %x = trunc <2 x i64> %i to <2 x i8>
     store <2 x i8> %x, <2 x i8>* %res

test/CodeGen/X86/extract-store.ll

 ; AVX-NEXT:    vpextrb $15, %xmm0, (%rdi)
 ; AVX-NEXT:    retq
   %vecext = extractelement <16 x i8> %foo, i32 15
   store i8 %vecext, i8* %dst, align 1
   ret void
 }
 
 define void @extract_i16_0(i16* nocapture %dst, <8 x i16> %foo) {
-; SSE-LABEL: extract_i16_0:
-; SSE:       # BB#0:
-; SSE-NEXT:    movd %xmm0, %eax
-; SSE-NEXT:    movw %ax, (%rdi)
-; SSE-NEXT:    retq
+; SSE2-LABEL: extract_i16_0:
+; SSE2:       # BB#0:
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract_i16_0:
+; SSE41:       # BB#0:
+; SSE41-NEXT:    pextrw $0, %xmm0, (%rdi)
+; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: extract_i16_0:
 ; AVX:       # BB#0:
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    movw %ax, (%rdi)
+; AVX-NEXT:    vpextrw $0, %xmm0, (%rdi)
 ; AVX-NEXT:    retq
   %vecext = extractelement <8 x i16> %foo, i32 0
   store i16 %vecext, i16* %dst, align 1
   ret void
 }
 
 define void @extract_i16_7(i16* nocapture %dst, <8 x i16> %foo) {
 ; SSE2-LABEL: extract_i16_7:

test/CodeGen/X86/lower-vec-shift-2.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse2 < %s | FileCheck %s --check-prefix=SSE2
 ; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+avx < %s | FileCheck %s --check-prefix=AVX
 
 define <8 x i16> @test1(<8 x i16> %A, <8 x i16> %B) {
 ; SSE2-LABEL: test1:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    movzwl %ax, %eax
+; SSE2-NEXT:    pextrw $0, %xmm1, %eax
 ; SSE2-NEXT:    movd %eax, %xmm1
 ; SSE2-NEXT:    psllw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test1:
 ; AVX:       # BB#0: # %entry
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
entry:
   %vecinit2 = shufflevector <2 x i64> %B, <2 x i64> undef, <2 x i32> zeroinitializer
   %shl = shl <2 x i64> %A, %vecinit2
   ret <2 x i64> %shl
 }
 
 define <8 x i16> @test4(<8 x i16> %A, <8 x i16> %B) {
 ; SSE2-LABEL: test4:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    movzwl %ax, %eax
+; SSE2-NEXT:    pextrw $0, %xmm1, %eax
 ; SSE2-NEXT:    movd %eax, %xmm1
 ; SSE2-NEXT:    psrlw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test4:
 ; AVX:       # BB#0: # %entry
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
entry:
   %vecinit2 = shufflevector <2 x i64> %B, <2 x i64> undef, <2 x i32> zeroinitializer
   %shr = lshr <2 x i64> %A, %vecinit2
   ret <2 x i64> %shr
 }
 
 define <8 x i16> @test7(<8 x i16> %A, <8 x i16> %B) {
 ; SSE2-LABEL: test7:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    movzwl %ax, %eax
+; SSE2-NEXT:    pextrw $0, %xmm1, %eax
 ; SSE2-NEXT:    movd %eax, %xmm1
 ; SSE2-NEXT:    psraw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test7:
 ; AVX:       # BB#0: # %entry
 ; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]

test/CodeGen/X86/masked_memop.ll

 
 define void @test_mask_store_8xi16(<8 x i1> %mask, <8 x i16>* %addr, <8 x i16> %val) {
 ; AVX-LABEL: test_mask_store_8xi16:
 ; AVX:       ## BB#0:
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX-NEXT:    testb $1, %al
 ; AVX-NEXT:    je LBB59_2
 ; AVX-NEXT:  ## BB#1: ## %cond.store
-; AVX-NEXT:    vmovd %xmm1, %eax
-; AVX-NEXT:    movw %ax, (%rdi)
+; AVX-NEXT:    vpextrw $0, %xmm1, (%rdi)
 ; AVX-NEXT:  LBB59_2: ## %else
 ; AVX-NEXT:    vpextrb $2, %xmm0, %eax
 ; AVX-NEXT:    testb $1, %al
 ; AVX-NEXT:    je LBB59_4
 ; AVX-NEXT:  ## BB#3: ## %cond.store1
 ; AVX-NEXT:    vpextrw $1, %xmm1, 2(%rdi)
 ; AVX-NEXT:  LBB59_4: ## %else2
 ; AVX-NEXT:    vpextrb $4, %xmm0, %eax

 ; AVX512F-NEXT:    vpsllq $63, %zmm0, %zmm0
 ; AVX512F-NEXT:    vptestmq %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kshiftlw $15, %k0, %k1
 ; AVX512F-NEXT:    kshiftrw $15, %k1, %k1
 ; AVX512F-NEXT:    kmovw %k1, %eax
 ; AVX512F-NEXT:    testb %al, %al
 ; AVX512F-NEXT:    je LBB59_2
 ; AVX512F-NEXT:  ## BB#1: ## %cond.store
-; AVX512F-NEXT:    vmovd %xmm1, %eax
-; AVX512F-NEXT:    movw %ax, (%rdi)
+; AVX512F-NEXT:    vpextrw $0, %xmm1, (%rdi)
 ; AVX512F-NEXT:  LBB59_2: ## %else
 ; AVX512F-NEXT:    kshiftlw $14, %k0, %k1
 ; AVX512F-NEXT:    kshiftrw $15, %k1, %k1
 ; AVX512F-NEXT:    kmovw %k1, %eax
 ; AVX512F-NEXT:    testb %al, %al
 ; AVX512F-NEXT:    je LBB59_4
 ; AVX512F-NEXT:  ## BB#3: ## %cond.store1
 ; AVX512F-NEXT:    vpextrw $1, %xmm1, 2(%rdi)

 
 define void @test_mask_store_16xi16(<16 x i1> %mask, <16 x i16>* %addr, <16 x i16> %val) {
 ; AVX1-LABEL: test_mask_store_16xi16:
 ; AVX1:       ## BB#0:
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je LBB60_2
 ; AVX1-NEXT:  ## BB#1: ## %cond.store
-; AVX1-NEXT:    vmovd %xmm1, %eax
-; AVX1-NEXT:    movw %ax, (%rdi)
+; AVX1-NEXT:    vpextrw $0, %xmm1, (%rdi)
 ; AVX1-NEXT:  LBB60_2: ## %else
 ; AVX1-NEXT:    vpextrb $1, %xmm0, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je LBB60_4
 ; AVX1-NEXT:  ## BB#3: ## %cond.store1
 ; AVX1-NEXT:    vpextrw $1, %xmm1, 2(%rdi)
 ; AVX1-NEXT:  LBB60_4: ## %else2
 ; AVX1-NEXT:    vpextrb $2, %xmm0, %eax

 ; AVX1-NEXT:  ## BB#15: ## %cond.store13
 ; AVX1-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
 ; AVX1-NEXT:  LBB60_16: ## %else14
 ; AVX1-NEXT:    vpextrb $8, %xmm0, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je LBB60_18
 ; AVX1-NEXT:  ## BB#17: ## %cond.store15
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vmovd %xmm2, %eax
-; AVX1-NEXT:    movw %ax, 16(%rdi)
+; AVX1-NEXT:    vpextrw $0, %xmm2, 16(%rdi)
 ; AVX1-NEXT:  LBB60_18: ## %else16
 ; AVX1-NEXT:    vpextrb $9, %xmm0, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je LBB60_20
 ; AVX1-NEXT:  ## BB#19: ## %cond.store17
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpextrw $1, %xmm2, 18(%rdi)
 ; AVX1-NEXT:  LBB60_20: ## %else18

 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: test_mask_store_16xi16:
 ; AVX2:       ## BB#0:
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je LBB60_2
 ; AVX2-NEXT:  ## BB#1: ## %cond.store
-; AVX2-NEXT:    vmovd %xmm1, %eax
-; AVX2-NEXT:    movw %ax, (%rdi)
+; AVX2-NEXT:    vpextrw $0, %xmm1, (%rdi)
 ; AVX2-NEXT:  LBB60_2: ## %else
 ; AVX2-NEXT:    vpextrb $1, %xmm0, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je LBB60_4
 ; AVX2-NEXT:  ## BB#3: ## %cond.store1
 ; AVX2-NEXT:    vpextrw $1, %xmm1, 2(%rdi)
 ; AVX2-NEXT:  LBB60_4: ## %else2
 ; AVX2-NEXT:    vpextrb $2, %xmm0, %eax

 ; AVX2-NEXT:  ## BB#15: ## %cond.store13
 ; AVX2-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
 ; AVX2-NEXT:  LBB60_16: ## %else14
 ; AVX2-NEXT:    vpextrb $8, %xmm0, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je LBB60_18
 ; AVX2-NEXT:  ## BB#17: ## %cond.store15
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT:    vmovd %xmm2, %eax
-; AVX2-NEXT:    movw %ax, 16(%rdi)
+; AVX2-NEXT:    vpextrw $0, %xmm2, 16(%rdi)
 ; AVX2-NEXT:  LBB60_18: ## %else16
 ; AVX2-NEXT:    vpextrb $9, %xmm0, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je LBB60_20
 ; AVX2-NEXT:  ## BB#19: ## %cond.store17
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vpextrw $1, %xmm2, 18(%rdi)
 ; AVX2-NEXT:  LBB60_20: ## %else18

 ; AVX512F-NEXT:    vpslld $31, %zmm0, %zmm0
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; AVX512F-NEXT:    kshiftlw $15, %k0, %k1
 ; AVX512F-NEXT:    kshiftrw $15, %k1, %k1
 ; AVX512F-NEXT:    kmovw %k1, %eax
 ; AVX512F-NEXT:    testb %al, %al
 ; AVX512F-NEXT:    je LBB60_2
 ; AVX512F-NEXT:  ## BB#1: ## %cond.store
-; AVX512F-NEXT:    vmovd %xmm1, %eax
-; AVX512F-NEXT:    movw %ax, (%rdi)
+; AVX512F-NEXT:    vpextrw $0, %xmm1, (%rdi)
 ; AVX512F-NEXT:  LBB60_2: ## %else
 ; AVX512F-NEXT:    kshiftlw $14, %k0, %k1
 ; AVX512F-NEXT:    kshiftrw $15, %k1, %k1
 ; AVX512F-NEXT:    kmovw %k1, %eax
 ; AVX512F-NEXT:    testb %al, %al
 ; AVX512F-NEXT:    je LBB60_4
 ; AVX512F-NEXT:  ## BB#3: ## %cond.store1
 ; AVX512F-NEXT:    vpextrw $1, %xmm1, 2(%rdi)

 ; AVX512F-NEXT:  LBB60_16: ## %else14
 ; AVX512F-NEXT:    kshiftlw $7, %k0, %k1
 ; AVX512F-NEXT:    kshiftrw $15, %k1, %k1
 ; AVX512F-NEXT:    kmovw %k1, %eax
 ; AVX512F-NEXT:    testb %al, %al
 ; AVX512F-NEXT:    je LBB60_18
 ; AVX512F-NEXT:  ## BB#17: ## %cond.store15
 ; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0
-; AVX512F-NEXT:    vmovd %xmm0, %eax
-; AVX512F-NEXT:    movw %ax, 16(%rdi)
+; AVX512F-NEXT:    vpextrw $0, %xmm0, 16(%rdi)
 ; AVX512F-NEXT:  LBB60_18: ## %else16
 ; AVX512F-NEXT:    kshiftlw $6, %k0, %k1
 ; AVX512F-NEXT:    kshiftrw $15, %k1, %k1
 ; AVX512F-NEXT:    kmovw %k1, %eax
 ; AVX512F-NEXT:    testb %al, %al
 ; AVX512F-NEXT:    je LBB60_20
 ; AVX512F-NEXT:  ## BB#19: ## %cond.store17
 ; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm0

 
 define void @test_mask_store_32xi16(<32 x i1> %mask, <32 x i16>* %addr, <32 x i16> %val) {
 ; AVX1-LABEL: test_mask_store_32xi16:
 ; AVX1:       ## BB#0:
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je LBB61_2
 ; AVX1-NEXT:  ## BB#1: ## %cond.store
-; AVX1-NEXT:    vmovd %xmm1, %eax
-; AVX1-NEXT:    movw %ax, (%rdi)
+; AVX1-NEXT:    vpextrw $0, %xmm1, (%rdi)
 ; AVX1-NEXT:  LBB61_2: ## %else
 ; AVX1-NEXT:    vpextrb $1, %xmm0, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je LBB61_4
 ; AVX1-NEXT:  ## BB#3: ## %cond.store1
 ; AVX1-NEXT:    vpextrw $1, %xmm1, 2(%rdi)
 ; AVX1-NEXT:  LBB61_4: ## %else2
 ; AVX1-NEXT:    vpextrb $2, %xmm0, %eax

 ; AVX1-NEXT:  ## BB#15: ## %cond.store13
 ; AVX1-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
 ; AVX1-NEXT:  LBB61_16: ## %else14
 ; AVX1-NEXT:    vpextrb $8, %xmm0, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je LBB61_18
 ; AVX1-NEXT:  ## BB#17: ## %cond.store15
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vmovd %xmm3, %eax
-; AVX1-NEXT:    movw %ax, 16(%rdi)
+; AVX1-NEXT:    vpextrw $0, %xmm3, 16(%rdi)
 ; AVX1-NEXT:  LBB61_18: ## %else16
 ; AVX1-NEXT:    vpextrb $9, %xmm0, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je LBB61_20
 ; AVX1-NEXT:  ## BB#19: ## %cond.store17
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
 ; AVX1-NEXT:    vpextrw $1, %xmm3, 18(%rdi)
 ; AVX1-NEXT:  LBB61_20: ## %else18

 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
 ; AVX1-NEXT:    vpextrw $7, %xmm1, 30(%rdi)
 ; AVX1-NEXT:  LBB61_32: ## %else30
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je LBB61_34
 ; AVX1-NEXT:  ## BB#33: ## %cond.store31
-; AVX1-NEXT:    vmovd %xmm2, %eax
-; AVX1-NEXT:    movw %ax, 32(%rdi)
+; AVX1-NEXT:    vpextrw $0, %xmm2, 32(%rdi)
 ; AVX1-NEXT:  LBB61_34: ## %else32
 ; AVX1-NEXT:    vpextrb $1, %xmm0, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je LBB61_36
 ; AVX1-NEXT:  ## BB#35: ## %cond.store33
 ; AVX1-NEXT:    vpextrw $1, %xmm2, 34(%rdi)
 ; AVX1-NEXT:  LBB61_36: ## %else34
 ; AVX1-NEXT:    vpextrb $2, %xmm0, %eax

 ; AVX1-NEXT:  ## BB#47: ## %cond.store45
 ; AVX1-NEXT:    vpextrw $7, %xmm2, 46(%rdi)
 ; AVX1-NEXT:  LBB61_48: ## %else46
 ; AVX1-NEXT:    vpextrb $8, %xmm0, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je LBB61_50
 ; AVX1-NEXT:  ## BB#49: ## %cond.store47
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
-; AVX1-NEXT:    vmovd %xmm1, %eax
-; AVX1-NEXT:    movw %ax, 48(%rdi)
+; AVX1-NEXT:    vpextrw $0, %xmm1, 48(%rdi)
 ; AVX1-NEXT:  LBB61_50: ## %else48
 ; AVX1-NEXT:    vpextrb $9, %xmm0, %eax
 ; AVX1-NEXT:    testb $1, %al
 ; AVX1-NEXT:    je LBB61_52
 ; AVX1-NEXT:  ## BB#51: ## %cond.store49
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm1
 ; AVX1-NEXT:    vpextrw $1, %xmm1, 50(%rdi)
 ; AVX1-NEXT:  LBB61_52: ## %else50

 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: test_mask_store_32xi16:
 ; AVX2:       ## BB#0:
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je LBB61_2
 ; AVX2-NEXT:  ## BB#1: ## %cond.store
-; AVX2-NEXT:    vmovd %xmm1, %eax
-; AVX2-NEXT:    movw %ax, (%rdi)
+; AVX2-NEXT:    vpextrw $0, %xmm1, (%rdi)
 ; AVX2-NEXT:  LBB61_2: ## %else
 ; AVX2-NEXT:    vpextrb $1, %xmm0, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je LBB61_4
 ; AVX2-NEXT:  ## BB#3: ## %cond.store1
 ; AVX2-NEXT:    vpextrw $1, %xmm1, 2(%rdi)
 ; AVX2-NEXT:  LBB61_4: ## %else2
 ; AVX2-NEXT:    vpextrb $2, %xmm0, %eax

 ; AVX2-NEXT:  ## BB#15: ## %cond.store13
 ; AVX2-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
 ; AVX2-NEXT:  LBB61_16: ## %else14
 ; AVX2-NEXT:    vpextrb $8, %xmm0, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je LBB61_18
 ; AVX2-NEXT:  ## BB#17: ## %cond.store15
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX2-NEXT:    vmovd %xmm3, %eax
-; AVX2-NEXT:    movw %ax, 16(%rdi)
+; AVX2-NEXT:    vpextrw $0, %xmm3, 16(%rdi)
 ; AVX2-NEXT:  LBB61_18: ## %else16
 ; AVX2-NEXT:    vpextrb $9, %xmm0, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je LBB61_20
 ; AVX2-NEXT:  ## BB#19: ## %cond.store17
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
 ; AVX2-NEXT:    vpextrw $1, %xmm3, 18(%rdi)
 ; AVX2-NEXT:  LBB61_20: ## %else18

 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX2-NEXT:    vpextrw $7, %xmm1, 30(%rdi)
 ; AVX2-NEXT:  LBB61_32: ## %else30
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je LBB61_34
 ; AVX2-NEXT:  ## BB#33: ## %cond.store31
-; AVX2-NEXT:    vmovd %xmm2, %eax
-; AVX2-NEXT:    movw %ax, 32(%rdi)
+; AVX2-NEXT:    vpextrw $0, %xmm2, 32(%rdi)
 ; AVX2-NEXT:  LBB61_34: ## %else32
 ; AVX2-NEXT:    vpextrb $1, %xmm0, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je LBB61_36
 ; AVX2-NEXT:  ## BB#35: ## %cond.store33
 ; AVX2-NEXT:    vpextrw $1, %xmm2, 34(%rdi)
 ; AVX2-NEXT:  LBB61_36: ## %else34
 ; AVX2-NEXT:    vpextrb $2, %xmm0, %eax

 ; AVX2-NEXT:  ## BB#47: ## %cond.store45
 ; AVX2-NEXT:    vpextrw $7, %xmm2, 46(%rdi)
 ; AVX2-NEXT:  LBB61_48: ## %else46
 ; AVX2-NEXT:    vpextrb $8, %xmm0, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je LBB61_50
 ; AVX2-NEXT:  ## BB#49: ## %cond.store47
 ; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX2-NEXT:    vmovd %xmm1, %eax
-; AVX2-NEXT:    movw %ax, 48(%rdi)
+; AVX2-NEXT:    vpextrw $0, %xmm1, 48(%rdi)
 ; AVX2-NEXT:  LBB61_50: ## %else48
 ; AVX2-NEXT:    vpextrb $9, %xmm0, %eax
 ; AVX2-NEXT:    testb $1, %al
 ; AVX2-NEXT:    je LBB61_52
 ; AVX2-NEXT:  ## BB#51: ## %cond.store49
 ; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm1
 ; AVX2-NEXT:    vpextrw $1, %xmm1, 50(%rdi)
 ; AVX2-NEXT:  LBB61_52: ## %else50

 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: test_mask_store_32xi16:
 ; AVX512F:       ## BB#0:
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512F-NEXT:    testb $1, %al
 ; AVX512F-NEXT:    je LBB61_2
 ; AVX512F-NEXT:  ## BB#1: ## %cond.store
-; AVX512F-NEXT:    vmovd %xmm1, %eax
-; AVX512F-NEXT:    movw %ax, (%rdi)
+; AVX512F-NEXT:    vpextrw $0, %xmm1, (%rdi)
 ; AVX512F-NEXT:  LBB61_2: ## %else
 ; AVX512F-NEXT:    vpextrb $1, %xmm0, %eax
 ; AVX512F-NEXT:    testb $1, %al
 ; AVX512F-NEXT:    je LBB61_4
 ; AVX512F-NEXT:  ## BB#3: ## %cond.store1
 ; AVX512F-NEXT:    vpextrw $1, %xmm1, 2(%rdi)
 ; AVX512F-NEXT:  LBB61_4: ## %else2
 ; AVX512F-NEXT:    vpextrb $2, %xmm0, %eax

 ; AVX512F-NEXT:  ## BB#15: ## %cond.store13
 ; AVX512F-NEXT:    vpextrw $7, %xmm1, 14(%rdi)
 ; AVX512F-NEXT:  LBB61_16: ## %else14
 ; AVX512F-NEXT:    vpextrb $8, %xmm0, %eax
 ; AVX512F-NEXT:    testb $1, %al
 ; AVX512F-NEXT:    je LBB61_18
 ; AVX512F-NEXT:  ## BB#17: ## %cond.store15
 ; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX512F-NEXT:    vmovd %xmm3, %eax
-; AVX512F-NEXT:    movw %ax, 16(%rdi)
+; AVX512F-NEXT:    vpextrw $0, %xmm3, 16(%rdi)
 ; AVX512F-NEXT:  LBB61_18: ## %else16
 ; AVX512F-NEXT:    vpextrb $9, %xmm0, %eax
 ; AVX512F-NEXT:    testb $1, %al
 ; AVX512F-NEXT:    je LBB61_20
 ; AVX512F-NEXT:  ## BB#19: ## %cond.store17
 ; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm3
 ; AVX512F-NEXT:    vpextrw $1, %xmm3, 18(%rdi)
 ; AVX512F-NEXT:  LBB61_20: ## %else18

 ; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512F-NEXT:    vpextrw $7, %xmm1, 30(%rdi)
 ; AVX512F-NEXT:  LBB61_32: ## %else30
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512F-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512F-NEXT:    testb $1, %al
 ; AVX512F-NEXT:    je LBB61_34
 ; AVX512F-NEXT:  ## BB#33: ## %cond.store31
-; AVX512F-NEXT:    vmovd %xmm2, %eax
-; AVX512F-NEXT:    movw %ax, 32(%rdi)
+; AVX512F-NEXT:    vpextrw $0, %xmm2, 32(%rdi)
 ; AVX512F-NEXT:  LBB61_34: ## %else32
 ; AVX512F-NEXT:    vpextrb $1, %xmm0, %eax
 ; AVX512F-NEXT:    testb $1, %al
 ; AVX512F-NEXT:    je LBB61_36
 ; AVX512F-NEXT:  ## BB#35: ## %cond.store33
 ; AVX512F-NEXT:    vpextrw $1, %xmm2, 34(%rdi)
 ; AVX512F-NEXT:  LBB61_36: ## %else34
 ; AVX512F-NEXT:    vpextrb $2, %xmm0, %eax

 ; AVX512F-NEXT:  ## BB#47: ## %cond.store45
 ; AVX512F-NEXT:    vpextrw $7, %xmm2, 46(%rdi)
 ; AVX512F-NEXT:  LBB61_48: ## %else46
 ; AVX512F-NEXT:    vpextrb $8, %xmm0, %eax
 ; AVX512F-NEXT:    testb $1, %al
 ; AVX512F-NEXT:    je LBB61_50
 ; AVX512F-NEXT:  ## BB#49: ## %cond.store47
 ; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX512F-NEXT:    vmovd %xmm1, %eax
-; AVX512F-NEXT:    movw %ax, 48(%rdi)
+; AVX512F-NEXT:    vpextrw $0, %xmm1, 48(%rdi)
 ; AVX512F-NEXT:  LBB61_50: ## %else48
 ; AVX512F-NEXT:    vpextrb $9, %xmm0, %eax
 ; AVX512F-NEXT:    testb $1, %al
 ; AVX512F-NEXT:    je LBB61_52
 ; AVX512F-NEXT:  ## BB#51: ## %cond.store49
 ; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm1
 ; AVX512F-NEXT:    vpextrw $1, %xmm1, 50(%rdi)
 ; AVX512F-NEXT:  LBB61_52: ## %else50

test/CodeGen/X86/trunc-ext-ld-st.ll

 ; SSE2-NEXT:    movw %ax, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: load_2_i8:
 ; SSE41:       # BB#0:
 ; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
 ; SSE41-NEXT:    paddq {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    movw %ax, (%rdi)
+; SSE41-NEXT:    pextrw $0, %xmm0, (%rdi)
 ; SSE41-NEXT:    retq
    %T = load <2 x i8>, <2 x i8>* %A
    %G = add <2 x i8> %T, <i8 9, i8 7>
    store <2 x i8> %G, <2 x i8>* %A
    ret void
 }
 
 ; Read 32-bits

test/CodeGen/X86/vector-shift-ashr-128.ll

; X32-SSE-NEXT:    retl
   %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
   %shift = ashr <4 x i32> %a, %splat
   ret <4 x i32> %shift
 }
 
 define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v8i16:
 ; SSE2:       # BB#0:
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    movzwl %ax, %eax
+; SSE2-NEXT:    pextrw $0, %xmm1, %eax
 ; SSE2-NEXT:    movd %eax, %xmm1
 ; SSE2-NEXT:    psraw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v8i16:
 ; SSE41:       # BB#0:
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3,4,5,6,7]

 ; AVX512:       ## BB#0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
 ; AVX512-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v8i16:
 ; X32-SSE:       # BB#0:
-; X32-SSE-NEXT:    movd %xmm1, %eax
-; X32-SSE-NEXT:    movzwl %ax, %eax
+; X32-SSE-NEXT:    pextrw $0, %xmm1, %eax
 ; X32-SSE-NEXT:    movd %eax, %xmm1
 ; X32-SSE-NEXT:    psraw %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> zeroinitializer
   %shift = ashr <8 x i16> %a, %splat
   ret <8 x i16> %shift
 }
 

test/CodeGen/X86/vector-shift-ashr-256.ll

; AVX512-NEXT:    retq
   %shift = ashr <8 x i32> %a, %splat
   ret <8 x i32> %shift
 }
 
 define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; AVX1-LABEL: splatvar_shift_v16i16:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vmovd %xmm1, %eax
-; AVX1-NEXT:    movzwl %ax, %eax
+; AVX1-NEXT:    vpextrw $0, %xmm1, %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm1
 ; AVX1-NEXT:    vpsraw %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_shift_v16i16:
 ; AVX2:       # BB#0:
-; AVX2-NEXT:    vmovd %xmm1, %eax
-; AVX2-NEXT:    movzwl %ax, %eax
+; AVX2-NEXT:    vpextrw $0, %xmm1, %eax
 ; AVX2-NEXT:    vmovd %eax, %xmm1
 ; AVX2-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_shift_v16i16:
 ; XOPAVX1:       # BB#0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; XOPAVX1-NEXT:    vmovd %xmm1, %eax
-; XOPAVX1-NEXT:    movzwl %ax, %eax
+; XOPAVX1-NEXT:    vpextrw $0, %xmm1, %eax
 ; XOPAVX1-NEXT:    vmovd %eax, %xmm1
 ; XOPAVX1-NEXT:    vpsraw %xmm1, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_shift_v16i16:
 ; XOPAVX2:       # BB#0:
-; XOPAVX2-NEXT:    vmovd %xmm1, %eax
-; XOPAVX2-NEXT:    movzwl %ax, %eax
+; XOPAVX2-NEXT:    vpextrw $0, %xmm1, %eax
 ; XOPAVX2-NEXT:    vmovd %eax, %xmm1
 ; XOPAVX2-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_shift_v16i16:
 ; AVX512:       ## BB#0:
-; AVX512-NEXT:    vmovd %xmm1, %eax
-; AVX512-NEXT:    movzwl %ax, %eax
+; AVX512-NEXT:    vpextrw $0, %xmm1, %eax
 ; AVX512-NEXT:    vmovd %eax, %xmm1
 ; AVX512-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
   %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer
   %shift = ashr <16 x i16> %a, %splat
   ret <16 x i16> %shift
 }
 

test/CodeGen/X86/vector-shift-ashr-512.ll

; ALL-NEXT:    retq
   %splat = shufflevector <16 x i32> %b, <16 x i32> undef, <16 x i32> zeroinitializer
   %shift = ashr <16 x i32> %a, %splat
   ret <16 x i32> %shift
 }
 
 define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 ; AVX512DQ-LABEL: splatvar_shift_v32i16:
 ; AVX512DQ:       ## BB#0:
-; AVX512DQ-NEXT:    vmovd %xmm2, %eax
-; AVX512DQ-NEXT:    movzwl %ax, %eax
+; AVX512DQ-NEXT:    vpextrw $0, %xmm2, %eax
 ; AVX512DQ-NEXT:    vmovd %eax, %xmm2
 ; AVX512DQ-NEXT:    vpsraw %xmm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpsraw %xmm2, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v32i16:
 ; AVX512BW:       ## BB#0:
-; AVX512BW-NEXT:    vmovd %xmm1, %eax
-; AVX512BW-NEXT:    movzwl %ax, %eax
+; AVX512BW-NEXT:    vpextrw $0, %xmm1, %eax
 ; AVX512BW-NEXT:    vmovd %eax, %xmm1
 ; AVX512BW-NEXT:    vpsraw %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
   %splat = shufflevector <32 x i16> %b, <32 x i16> undef, <32 x i32> zeroinitializer
   %shift = ashr <32 x i16> %a, %splat
   ret <32 x i16> %shift
 }
 

test/CodeGen/X86/vector-shift-lshr-128.ll

; X32-SSE-NEXT:    retl
   %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
   %shift = lshr <4 x i32> %a, %splat
   ret <4 x i32> %shift
 }
 
 define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v8i16:
 ; SSE2:       # BB#0:
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    movzwl %ax, %eax
+; SSE2-NEXT:    pextrw $0, %xmm1, %eax
 ; SSE2-NEXT:    movd %eax, %xmm1
 ; SSE2-NEXT:    psrlw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v8i16:
 ; SSE41:       # BB#0:
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3,4,5,6,7]

 ; AVX512:       ## BB#0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
 ; AVX512-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v8i16:
 ; X32-SSE:       # BB#0:
-; X32-SSE-NEXT:    movd %xmm1, %eax
-; X32-SSE-NEXT:    movzwl %ax, %eax
+; X32-SSE-NEXT:    pextrw $0, %xmm1, %eax
 ; X32-SSE-NEXT:    movd %eax, %xmm1
 ; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> zeroinitializer
   %shift = lshr <8 x i16> %a, %splat
   ret <8 x i16> %shift
 }
 

test/CodeGen/X86/vector-shift-lshr-256.ll

; AVX512-NEXT:    retq
   %shift = lshr <8 x i32> %a, %splat
   ret <8 x i32> %shift
 }
 
 define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; AVX1-LABEL: splatvar_shift_v16i16:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vmovd %xmm1, %eax
-; AVX1-NEXT:    movzwl %ax, %eax
+; AVX1-NEXT:    vpextrw $0, %xmm1, %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm1
 ; AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_shift_v16i16:
 ; AVX2:       # BB#0:
-; AVX2-NEXT:    vmovd %xmm1, %eax
-; AVX2-NEXT:    movzwl %ax, %eax
+; AVX2-NEXT:    vpextrw $0, %xmm1, %eax
 ; AVX2-NEXT:    vmovd %eax, %xmm1
 ; AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_shift_v16i16:
 ; XOPAVX1:       # BB#0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; XOPAVX1-NEXT:    vmovd %xmm1, %eax
-; XOPAVX1-NEXT:    movzwl %ax, %eax
+; XOPAVX1-NEXT:    vpextrw $0, %xmm1, %eax
 ; XOPAVX1-NEXT:    vmovd %eax, %xmm1
 ; XOPAVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_shift_v16i16:
 ; XOPAVX2:       # BB#0:
-; XOPAVX2-NEXT:    vmovd %xmm1, %eax
-; XOPAVX2-NEXT:    movzwl %ax, %eax
+; XOPAVX2-NEXT:    vpextrw $0, %xmm1, %eax
 ; XOPAVX2-NEXT:    vmovd %eax, %xmm1
 ; XOPAVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_shift_v16i16:
 ; AVX512:       ## BB#0:
-; AVX512-NEXT:    vmovd %xmm1, %eax
-; AVX512-NEXT:    movzwl %ax, %eax
+; AVX512-NEXT:    vpextrw $0, %xmm1, %eax
 ; AVX512-NEXT:    vmovd %eax, %xmm1
 ; AVX512-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
   %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer
   %shift = lshr <16 x i16> %a, %splat
   ret <16 x i16> %shift
 }
 

test/CodeGen/X86/vector-shift-lshr-512.ll

; ALL-NEXT:    retq
   %splat = shufflevector <16 x i32> %b, <16 x i32> undef, <16 x i32> zeroinitializer
   %shift = lshr <16 x i32> %a, %splat
   ret <16 x i32> %shift
 }
 
 define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 ; AVX512DQ-LABEL: splatvar_shift_v32i16:
 ; AVX512DQ:       ## BB#0:
-; AVX512DQ-NEXT:    vmovd %xmm2, %eax
-; AVX512DQ-NEXT:    movzwl %ax, %eax
+; AVX512DQ-NEXT:    vpextrw $0, %xmm2, %eax
 ; AVX512DQ-NEXT:    vmovd %eax, %xmm2
 ; AVX512DQ-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v32i16:
 ; AVX512BW:       ## BB#0:
-; AVX512BW-NEXT:    vmovd %xmm1, %eax
-; AVX512BW-NEXT:    movzwl %ax, %eax
+; AVX512BW-NEXT:    vpextrw $0, %xmm1, %eax
 ; AVX512BW-NEXT:    vmovd %eax, %xmm1
 ; AVX512BW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
   %splat = shufflevector <32 x i16> %b, <32 x i16> undef, <32 x i32> zeroinitializer
   %shift = lshr <32 x i16> %a, %splat
   ret <32 x i16> %shift
 }
 

test/CodeGen/X86/vector-shift-shl-128.ll

; X32-SSE-NEXT:    retl
   %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
   %shift = shl <4 x i32> %a, %splat
   ret <4 x i32> %shift
 }
 
 define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; SSE2-LABEL: splatvar_shift_v8i16:
 ; SSE2:       # BB#0:
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    movzwl %ax, %eax
+; SSE2-NEXT:    pextrw $0, %xmm1, %eax
 ; SSE2-NEXT:    movd %eax, %xmm1
 ; SSE2-NEXT:    psllw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_shift_v8i16:
 ; SSE41:       # BB#0:
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3,4,5,6,7]

 ; AVX512:       ## BB#0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
 ; AVX512-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_shift_v8i16:
 ; X32-SSE:       # BB#0:
-; X32-SSE-NEXT:    movd %xmm1, %eax
-; X32-SSE-NEXT:    movzwl %ax, %eax
+; X32-SSE-NEXT:    pextrw $0, %xmm1, %eax
 ; X32-SSE-NEXT:    movd %eax, %xmm1
 ; X32-SSE-NEXT:    psllw %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> zeroinitializer
   %shift = shl <8 x i16> %a, %splat
   ret <8 x i16> %shift
 }
 

test/CodeGen/X86/vector-shift-shl-256.ll

; AVX512-NEXT:    retq
   %shift = shl <8 x i32> %a, %splat
   ret <8 x i32> %shift
 }
 
 define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; AVX1-LABEL: splatvar_shift_v16i16:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vmovd %xmm1, %eax
-; AVX1-NEXT:    movzwl %ax, %eax
+; AVX1-NEXT:    vpextrw $0, %xmm1, %eax
 ; AVX1-NEXT:    vmovd %eax, %xmm1
 ; AVX1-NEXT:    vpsllw %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_shift_v16i16:
 ; AVX2:       # BB#0:
-; AVX2-NEXT:    vmovd %xmm1, %eax
-; AVX2-NEXT:    movzwl %ax, %eax
+; AVX2-NEXT:    vpextrw $0, %xmm1, %eax
 ; AVX2-NEXT:    vmovd %eax, %xmm1
 ; AVX2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_shift_v16i16:
 ; XOPAVX1:       # BB#0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; XOPAVX1-NEXT:    vmovd %xmm1, %eax
-; XOPAVX1-NEXT:    movzwl %ax, %eax
+; XOPAVX1-NEXT:    vpextrw $0, %xmm1, %eax
 ; XOPAVX1-NEXT:    vmovd %eax, %xmm1
 ; XOPAVX1-NEXT:    vpsllw %xmm1, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_shift_v16i16:
 ; XOPAVX2:       # BB#0:
-; XOPAVX2-NEXT:    vmovd %xmm1, %eax
-; XOPAVX2-NEXT:    movzwl %ax, %eax
+; XOPAVX2-NEXT:    vpextrw $0, %xmm1, %eax
 ; XOPAVX2-NEXT:    vmovd %eax, %xmm1
 ; XOPAVX2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_shift_v16i16:
 ; AVX512:       ## BB#0:
-; AVX512-NEXT:    vmovd %xmm1, %eax
-; AVX512-NEXT:    movzwl %ax, %eax
+; AVX512-NEXT:    vpextrw $0, %xmm1, %eax
 ; AVX512-NEXT:    vmovd %eax, %xmm1
 ; AVX512-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
   %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer
   %shift = shl <16 x i16> %a, %splat
   ret <16 x i16> %shift
 }
 

test/CodeGen/X86/vector-shift-shl-512.ll

; ALL-NEXT:    retq
   %splat = shufflevector <16 x i32> %b, <16 x i32> undef, <16 x i32> zeroinitializer
   %shift = shl <16 x i32> %a, %splat
   ret <16 x i32> %shift
 }
 
 define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 ; AVX512DQ-LABEL: splatvar_shift_v32i16:
 ; AVX512DQ:       ## BB#0:
-; AVX512DQ-NEXT:    vmovd %xmm2, %eax
-; AVX512DQ-NEXT:    movzwl %ax, %eax
+; AVX512DQ-NEXT:    vpextrw $0, %xmm2, %eax
 ; AVX512DQ-NEXT:    vmovd %eax, %xmm2
 ; AVX512DQ-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpsllw %xmm2, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_shift_v32i16:
 ; AVX512BW:       ## BB#0:
-; AVX512BW-NEXT:    vmovd %xmm1, %eax
-; AVX512BW-NEXT:    movzwl %ax, %eax
+; AVX512BW-NEXT:    vpextrw $0, %xmm1, %eax
 ; AVX512BW-NEXT:    vmovd %eax, %xmm1
 ; AVX512BW-NEXT:    vpsllw %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
   %splat = shufflevector <32 x i16> %b, <32 x i16> undef, <32 x i32> zeroinitializer
   %shift = shl <32 x i16> %a, %splat
   ret <32 x i16> %shift
 }
 

test/CodeGen/X86/widen_conv-1.ll

 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movdqa (%ecx), %xmm0
 ; X86-NEXT:    paddd {{\.LCPI.*}}, %xmm0
 ; X86-NEXT:    pextrb $8, %xmm0, 2(%eax)
 ; X86-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; X86-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; X86-NEXT:    movd %xmm0, %ecx
-; X86-NEXT:    movw %cx, (%eax)
+; X86-NEXT:    pextrw $0, %xmm0, (%eax)
 ; X86-NEXT:    popl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: convert_v3i32_to_v3i8:
 ; X64:       # BB#0: # %entry
 ; X64-NEXT:    movdqa (%rsi), %xmm0
 ; X64-NEXT:    paddd {{.*}}(%rip), %xmm0
 ; X64-NEXT:    pextrb $8, %xmm0, 2(%rdi)
 ; X64-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; X64-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; X64-NEXT:    movd %xmm0, %eax
-; X64-NEXT:    movw %ax, (%rdi)
+; X64-NEXT:    pextrw $0, %xmm0, (%rdi)
 ; X64-NEXT:    retq
 entry:
   %load = load <3 x i32>, <3 x i32>* %src.addr
   %val = trunc <3 x i32> %load to <3 x i8>
   %add = add <3 x i8> %val, < i8 1, i8 1, i8 1 >
   store <3 x i8> %add, <3 x i8>* %dst.addr
   ret void
 }

test/CodeGen/X86/widen_load-2.ll

 ; CHECK-LABEL: add3i8:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
 ; CHECK-NEXT:    pmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
 ; CHECK-NEXT:    paddd %xmm0, %xmm1
 ; CHECK-NEXT:    pextrb $8, %xmm1, 2(%rdi)
 ; CHECK-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; CHECK-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, (%rdi)
+; CHECK-NEXT:    pextrw $0, %xmm0, (%rdi)
 ; CHECK-NEXT:    movq %rdi, %rax
 ; CHECK-NEXT:    retq
   %a = load %i8vec3, %i8vec3* %ap, align 16
   %b = load %i8vec3, %i8vec3* %bp, align 16
   %x = add %i8vec3 %a, %b
   store %i8vec3 %x, %i8vec3* %ret, align 16
   ret void
 }

 %i8vec3pack = type { <3 x i8>, i8 }
 define void @rot(%i8vec3pack* nocapture sret %result, %i8vec3pack* %X, %i8vec3pack* %rot) nounwind {
 ; CHECK-LABEL: rot:
 ; CHECK:       # BB#0: # %entry
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm0 = <0,4,8,128,u,u,u,u,u,u,u,u,u,u,u,u>
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = <158,158,158,u>
 ; CHECK-NEXT:    pshufb %xmm0, %xmm1
 ; CHECK-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; CHECK-NEXT:    movd %xmm1, %eax
-; CHECK-NEXT:    movw %ax, (%rsi)
+; CHECK-NEXT:    pextrw $0, %xmm1, (%rsi)
 ; CHECK-NEXT:    movb $-98, 2(%rsi)
 ; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = <1,1,1,u>
 ; CHECK-NEXT:    pshufb %xmm0, %xmm1
 ; CHECK-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, (%rdx)
+; CHECK-NEXT:    pextrw $0, %xmm0, (%rdx)
 ; CHECK-NEXT:    movb $1, 2(%rdx)
 ; CHECK-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
 ; CHECK-NEXT:    movdqa %xmm0, %xmm1
 ; CHECK-NEXT:    psrld $1, %xmm1
 ; CHECK-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5],xmm0[6,7]
 ; CHECK-NEXT:    pextrb $8, %xmm1, 2(%rdi)
 ; CHECK-NEXT:    pshufb {{.*#+}} xmm1 = xmm1[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
 ; CHECK-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    movw %ax, (%rdi)
+; CHECK-NEXT:    pextrw $0, %xmm0, (%rdi)
 ; CHECK-NEXT:    movq %rdi, %rax
 ; CHECK-NEXT:    retq
 entry:
   %storetmp = bitcast %i8vec3pack* %X to <3 x i8>*
   store <3 x i8> <i8 -98, i8 -98, i8 -98>, <3 x i8>* %storetmp
   %storetmp1 = bitcast %i8vec3pack* %rot to <3 x i8>*
   store <3 x i8> <i8 1, i8 1, i8 1>, <3 x i8>* %storetmp1
   %tmp = load %i8vec3pack, %i8vec3pack* %X