[INSTCOMBINE][X86][SSE] Replace sign/zero extension intrinsics with native IR

[RKSimon]

Summary

Now that we are generating sane codegen for vector sext/zext nodes on SSE targets, this patch uses instcombine to replace the SSE41/AVX2 pmovsx and pmovzx intrinsics with the equivalent native IR code.

I have investigated removing these intrinsics completely, but we are still up against the issue of debug/-O0 codegen. At present this would be doable for some of the AVX2 intrinsics only (the ones that extend the entire xmm to a ymm) - if people think this is worthwhile I can provide patches that update the avx2 headers to use __builtin_convertvector where possible and remove those specific intrinsics entirely but I thought it best to keep them all for now and include them in this instcombine patch.

[spatel]

LGTM; see inline comments for nits.

But as Paul suggested recently, we should decide how to handle these in general since we have a mishmash of implementations...
Then start the drudge work of fixing the hundreds of intrinsics that we decide are done wrong. :(

Reference:
http://reviews.llvm.org/D10555
https://llvm.org/bugs/show_bug.cgi?id=24125 and related bugs

lib/Transforms/InstCombine/InstCombineCalls.cpp

208

Remove 'sign' from comment since this is a general extend now?

215–217

Can fold these 2 lines together; no need for EX temp.

[RKSimon]

Updated based on Sanjay's feedback

[spatel]

Forgot to set the 'Accept' action here on Phab.

I don't see much downside to checking this in, regardless of the outcome of any future decision about how to best handle intrinsics in general.

lib/Transforms/InstCombine/InstCombineCalls.cpp

if (Len <= 8 && isPowerOf2_32((uint32_t)Len)) {
     // Set the size of the copy to 0, it will be deleted on the next iteration.
     MI->setLength(Constant::getNullValue(LenC->getType()));
     return MI;
   }
 
   return nullptr;
 }
 
+static Value *SimplifyX86extend(const IntrinsicInst &II,
+                                InstCombiner::BuilderTy &Builder,
+                                bool SignExtend) {
+  VectorType *SrcTy = cast<VectorType>(II.getArgOperand(0)->getType());
+  VectorType *DstTy = cast<VectorType>(II.getType());
+  unsigned NumDstElts = DstTy->getNumElements();
+
+  // Extract a subvector of the first NumElts lanes and sign extend.
+  SmallVector<int, 8> ShuffleMask;
+  for (int i = 0; i != NumDstElts; ++i)
+    ShuffleMask.push_back(i);
+
+  Value *SV = Builder.CreateShuffleVector(II.getArgOperand(0),
+                                          UndefValue::get(SrcTy), ShuffleMask);
+  Value *EX = SignExtend ? Builder.CreateSExt(SV, DstTy)
+                         : Builder.CreateZExt(SV, DstTy);
+  return EX;
+}
+
 static Value *SimplifyX86insertps(const IntrinsicInst &II,
                                   InstCombiner::BuilderTy &Builder) {
   if (auto *CInt = dyn_cast<ConstantInt>(II.getArgOperand(2))) {
     VectorType *VecTy = cast<VectorType>(II.getType());
     assert(VecTy->getNumElements() == 4 && "insertps with wrong vector type");
-    
     // The immediate permute control byte looks like this:
     //    [3:0] - zero mask for each 32-bit lane
     //    [5:4] - select one 32-bit destination lane
     //    [7:6] - select one 32-bit source lane
 
     uint8_t Imm = CInt->getZExtValue();
     uint8_t ZMask = Imm & 0xf;
     uint8_t DestLane = (Imm >> 4) & 0x3;
if (ZMask) {
       } else {
         // TODO: Model this case as 2 shuffles or a 'logical and' plus shuffle?
         return nullptr;
       }
     } else {
       // Replace the selected destination lane with the selected source lane.
       ShuffleMask[DestLane] = SourceLane + 4;
     }
-  
     return Builder.CreateShuffleVector(II.getArgOperand(0), V1, ShuffleMask);
   }
   return nullptr;
 }
 
 /// The shuffle mask for a perm2*128 selects any two halves of two 256-bit
 /// source vectors, unless a zero bit is set. If a zero bit is set,
 /// then ignore that half of the mask and clear that half of the vector.
if (auto *CInt = dyn_cast<ConstantInt>(II.getArgOperand(2))) {
     // If 0 or 1 zero mask bits are set, this is a simple shuffle.
     unsigned NumElts = VecTy->getNumElements();
     unsigned HalfSize = NumElts / 2;
     SmallVector<int, 8> ShuffleMask(NumElts);
 
     // The high bit of the selection field chooses the 1st or 2nd operand.
     bool LowInputSelect = Imm & 0x02;
     bool HighInputSelect = Imm & 0x20;
-    
     // The low bit of the selection field chooses the low or high half
     // of the selected operand.
     bool LowHalfSelect = Imm & 0x01;
     bool HighHalfSelect = Imm & 0x10;
 
     // Determine which operand(s) are actually in use for this instruction.
     Value *V0 = LowInputSelect ? II.getArgOperand(1) : II.getArgOperand(0);
     Value *V1 = HighInputSelect ? II.getArgOperand(1) : II.getArgOperand(0);
-    
     // If needed, replace operands based on zero mask.
     V0 = LowHalfZero ? ZeroVector : V0;
     V1 = HighHalfZero ? ZeroVector : V1;
-    
     // Permute low half of result.
     unsigned StartIndex = LowHalfSelect ? HalfSize : 0;
     for (unsigned i = 0; i < HalfSize; ++i)
       ShuffleMask[i] = StartIndex + i;
 
     // Permute high half of result.
     StartIndex = HighHalfSelect ? HalfSize : 0;
     StartIndex += NumElts;
case Intrinsic::x86_avx2_psrli_w: {
     if (isPackedShiftLeft)
       return BinaryOperator::CreateShl(Vec,
           Builder->CreateVectorSplat(VWidth, VTCI));
 
     return BinaryOperator::CreateLShr(Vec,
         Builder->CreateVectorSplat(VWidth, VTCI));
   }
 
+  case Intrinsic::x86_sse41_pmovsxbd:
+  case Intrinsic::x86_sse41_pmovsxbq:
   case Intrinsic::x86_sse41_pmovsxbw:
-  case Intrinsic::x86_sse41_pmovsxwd:
   case Intrinsic::x86_sse41_pmovsxdq:
+  case Intrinsic::x86_sse41_pmovsxwd:
+  case Intrinsic::x86_sse41_pmovsxwq:
+  case Intrinsic::x86_avx2_pmovsxbd:
+  case Intrinsic::x86_avx2_pmovsxbq:
+  case Intrinsic::x86_avx2_pmovsxbw:
+  case Intrinsic::x86_avx2_pmovsxdq:
+  case Intrinsic::x86_avx2_pmovsxwd:
+  case Intrinsic::x86_avx2_pmovsxwq:
+    if (Value *V = SimplifyX86extend(*II, *Builder, true))
+      return ReplaceInstUsesWith(*II, V);
+    break;
+
+  case Intrinsic::x86_sse41_pmovzxbd:
+  case Intrinsic::x86_sse41_pmovzxbq:
   case Intrinsic::x86_sse41_pmovzxbw:
+  case Intrinsic::x86_sse41_pmovzxdq:
   case Intrinsic::x86_sse41_pmovzxwd:
-  case Intrinsic::x86_sse41_pmovzxdq: {
-    // pmov{s|z}x ignores the upper half of their input vectors.
-    unsigned VWidth =
-      cast<VectorType>(II->getArgOperand(0)->getType())->getNumElements();
-    unsigned LowHalfElts = VWidth / 2;
-    APInt InputDemandedElts(APInt::getBitsSet(VWidth, 0, LowHalfElts));
-    APInt UndefElts(VWidth, 0);
-    if (Value *TmpV = SimplifyDemandedVectorElts(
-            II->getArgOperand(0), InputDemandedElts, UndefElts)) {
-      II->setArgOperand(0, TmpV);
-      return II;
-    }
+  case Intrinsic::x86_sse41_pmovzxwq:
+  case Intrinsic::x86_avx2_pmovzxbd:
+  case Intrinsic::x86_avx2_pmovzxbq:
+  case Intrinsic::x86_avx2_pmovzxbw:
+  case Intrinsic::x86_avx2_pmovzxdq:
+  case Intrinsic::x86_avx2_pmovzxwd:
+  case Intrinsic::x86_avx2_pmovzxwq:
+    if (Value *V = SimplifyX86extend(*II, *Builder, false))
+      return ReplaceInstUsesWith(*II, V);
     break;
-  }
+
   case Intrinsic::x86_sse41_insertps:
     if (Value *V = SimplifyX86insertps(*II, *Builder))
       return ReplaceInstUsesWith(*II, V);
     break;
-    
   case Intrinsic::x86_sse4a_insertqi: {
     // insertqi x, y, 64, 0 can just copy y's lower bits and leave the top
     // ones undef
     // TODO: eventually we should lower this intrinsic to IR
     if (auto CIWidth = dyn_cast<ConstantInt>(II->getArgOperand(2))) {
       if (auto CIStart = dyn_cast<ConstantInt>(II->getArgOperand(3))) {
         unsigned Index = CIStart->getZExtValue();
         // From AMD documentation: "a value of zero in the field length is

test/Transforms/InstCombine/vec_demanded_elts.ll

 declare i64 @llvm.x86.sse.cvtss2si64(<4 x float>)
 declare i32 @llvm.x86.sse.cvttss2si(<4 x float>)
 declare i64 @llvm.x86.sse.cvttss2si64(<4 x float>)
 declare i32 @llvm.x86.sse2.cvtsd2si(<2 x double>)
 declare i64 @llvm.x86.sse2.cvtsd2si64(<2 x double>)
 declare i32 @llvm.x86.sse2.cvttsd2si(<2 x double>)
 declare i64 @llvm.x86.sse2.cvttsd2si64(<2 x double>)
 
-; <rdar://problem/6945110>
-define <4 x i32> @kernel3_vertical(<4 x i16> * %src, <8 x i16> * %foo) nounwind {
-entry:
-  %tmp = load <4 x i16>, <4 x i16>* %src
-  %tmp1 = load <8 x i16>, <8 x i16>* %foo
-; CHECK: %tmp2 = shufflevector
-  %tmp2 = shufflevector <4 x i16> %tmp, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-; pmovzxwd ignores the upper 64-bits of its input; -instcombine should remove this shuffle:
-; CHECK-NOT: shufflevector
-  %tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: pmovzxwd
-  %0 = call <4 x i32> @llvm.x86.sse41.pmovzxwd(<8 x i16> %tmp3)
-  ret <4 x i32> %0
-}
-declare <4 x i32> @llvm.x86.sse41.pmovzxwd(<8 x i16>) nounwind readnone
-
 define <4 x float> @dead_shuffle_elt(<4 x float> %x, <2 x float> %y) nounwind {
 entry:
 ; CHECK-LABEL: define <4 x float> @dead_shuffle_elt(
 ; CHECK: shufflevector <2 x float> %y, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
   %shuffle.i = shufflevector <2 x float> %y, <2 x float> %y, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   %shuffle9.i = shufflevector <4 x float> %x, <4 x float> %shuffle.i, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
   ret <4 x float> %shuffle9.i
 }

test/Transforms/InstCombine/x86-pmovsx.ll

+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare <4 x i32>  @llvm.x86.sse41.pmovsxbd(<16 x i8>) nounwind readnone
+declare <2 x i64>  @llvm.x86.sse41.pmovsxbq(<16 x i8>) nounwind readnone
+declare <8 x i16>  @llvm.x86.sse41.pmovsxbw(<16 x i8>) nounwind readnone
+declare <2 x i64>  @llvm.x86.sse41.pmovsxdq(<4 x i32>) nounwind readnone
+declare <4 x i32>  @llvm.x86.sse41.pmovsxwd(<8 x i16>) nounwind readnone
+declare <2 x i64>  @llvm.x86.sse41.pmovsxwq(<8 x i16>) nounwind readnone
+
+declare <8 x i32>  @llvm.x86.avx2.pmovsxbd(<16 x i8>) nounwind readnone
+declare <4 x i64>  @llvm.x86.avx2.pmovsxbq(<16 x i8>) nounwind readnone
+declare <16 x i16> @llvm.x86.avx2.pmovsxbw(<16 x i8>) nounwind readnone
+declare <4 x i64>  @llvm.x86.avx2.pmovsxdq(<4 x i32>) nounwind readnone
+declare <8 x i32>  @llvm.x86.avx2.pmovsxwd(<8 x i16>) nounwind readnone
+declare <4 x i64>  @llvm.x86.avx2.pmovsxwq(<8 x i16>) nounwind readnone
+
+;
+; Basic sign extension tests
+;
+
+define <4 x i32> @sse41_pmovsxbd(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovsxbd
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  sext <4 x i8> %1 to <4 x i32>
+; CHECK-NEXT:  ret <4 x i32> %2
+
+  %res = call <4 x i32> @llvm.x86.sse41.pmovsxbd(<16 x i8> %v)
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @sse41_pmovsxbq(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovsxbq
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:  sext <2 x i8> %1 to <2 x i64>
+; CHECK-NEXT:  ret <2 x i64> %2
+
+  %res = call <2 x i64> @llvm.x86.sse41.pmovsxbq(<16 x i8> %v)
+  ret <2 x i64> %res
+}
+
+define <8 x i16> @sse41_pmovsxbw(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovsxbw
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  sext <8 x i8> %1 to <8 x i16>
+; CHECK-NEXT:  ret <8 x i16> %2
+
+  %res = call <8 x i16> @llvm.x86.sse41.pmovsxbw(<16 x i8> %v)
+  ret <8 x i16> %res
+}
+
+define <2 x i64> @sse41_pmovsxdq(<4 x i32> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovsxdq
+; CHECK-NEXT:  shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:  sext <2 x i32> %1 to <2 x i64>
+; CHECK-NEXT:  ret <2 x i64> %2
+
+  %res = call <2 x i64> @llvm.x86.sse41.pmovsxdq(<4 x i32> %v)
+  ret <2 x i64> %res
+}
+
+define <4 x i32> @sse41_pmovsxwd(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovsxwd
+; CHECK-NEXT:  shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  sext <4 x i16> %1 to <4 x i32>
+; CHECK-NEXT:  ret <4 x i32> %2
+
+  %res = call <4 x i32> @llvm.x86.sse41.pmovsxwd(<8 x i16> %v)
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @sse41_pmovsxwq(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovsxwq
+; CHECK-NEXT:  shufflevector <8 x i16> %v, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:  sext <2 x i16> %1 to <2 x i64>
+; CHECK-NEXT:  ret <2 x i64> %2
+
+  %res = call <2 x i64> @llvm.x86.sse41.pmovsxwq(<8 x i16> %v)
+  ret <2 x i64> %res
+}
+
+define <8 x i32> @avx2_pmovsxbd(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovsxbd
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  sext <8 x i8> %1 to <8 x i32>
+; CHECK-NEXT:  ret <8 x i32> %2
+
+  %res = call <8 x i32> @llvm.x86.avx2.pmovsxbd(<16 x i8> %v)
+  ret <8 x i32> %res
+}
+
+define <4 x i64> @avx2_pmovsxbq(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovsxbq
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  sext <4 x i8> %1 to <4 x i64>
+; CHECK-NEXT:  ret <4 x i64> %2
+
+  %res = call <4 x i64> @llvm.x86.avx2.pmovsxbq(<16 x i8> %v)
+  ret <4 x i64> %res
+}
+
+define <16 x i16> @avx2_pmovsxbw(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovsxbw
+; CHECK-NEXT:  sext <16 x i8> %v to <16 x i16>
+; CHECK-NEXT:  ret <16 x i16> %1
+
+  %res = call <16 x i16> @llvm.x86.avx2.pmovsxbw(<16 x i8> %v)
+  ret <16 x i16> %res
+}
+
+define <4 x i64> @avx2_pmovsxdq(<4 x i32> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovsxdq
+; CHECK-NEXT:  sext <4 x i32> %v to <4 x i64>
+; CHECK-NEXT:  ret <4 x i64> %1
+
+  %res = call <4 x i64> @llvm.x86.avx2.pmovsxdq(<4 x i32> %v)
+  ret <4 x i64> %res
+}
+
+define <8 x i32> @avx2_pmovsxwd(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovsxwd
+; CHECK-NEXT:  sext <8 x i16> %v to <8 x i32>
+; CHECK-NEXT:  ret <8 x i32> %1
+
+  %res = call <8 x i32> @llvm.x86.avx2.pmovsxwd(<8 x i16> %v)
+  ret <8 x i32> %res
+}
+
+define <4 x i64> @avx2_pmovsxwq(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovsxwq
+; CHECK-NEXT:  shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  sext <4 x i16> %1 to <4 x i64>
+; CHECK-NEXT:  ret <4 x i64> %2
+
+  %res = call <4 x i64> @llvm.x86.avx2.pmovsxwq(<8 x i16> %v)
+  ret <4 x i64> %res
+}

test/Transforms/InstCombine/x86-pmovzx.ll

+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare <4 x i32>  @llvm.x86.sse41.pmovzxbd(<16 x i8>) nounwind readnone
+declare <2 x i64>  @llvm.x86.sse41.pmovzxbq(<16 x i8>) nounwind readnone
+declare <8 x i16>  @llvm.x86.sse41.pmovzxbw(<16 x i8>) nounwind readnone
+declare <2 x i64>  @llvm.x86.sse41.pmovzxdq(<4 x i32>) nounwind readnone
+declare <4 x i32>  @llvm.x86.sse41.pmovzxwd(<8 x i16>) nounwind readnone
+declare <2 x i64>  @llvm.x86.sse41.pmovzxwq(<8 x i16>) nounwind readnone
+
+declare <8 x i32>  @llvm.x86.avx2.pmovzxbd(<16 x i8>) nounwind readnone
+declare <4 x i64>  @llvm.x86.avx2.pmovzxbq(<16 x i8>) nounwind readnone
+declare <16 x i16> @llvm.x86.avx2.pmovzxbw(<16 x i8>) nounwind readnone
+declare <4 x i64>  @llvm.x86.avx2.pmovzxdq(<4 x i32>) nounwind readnone
+declare <8 x i32>  @llvm.x86.avx2.pmovzxwd(<8 x i16>) nounwind readnone
+declare <4 x i64>  @llvm.x86.avx2.pmovzxwq(<8 x i16>) nounwind readnone
+
+;
+; Basic zero extension tests
+;
+
+define <4 x i32> @sse41_pmovzxbd(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovzxbd
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  zext <4 x i8> %1 to <4 x i32>
+; CHECK-NEXT:  ret <4 x i32> %2
+
+  %res = call <4 x i32> @llvm.x86.sse41.pmovzxbd(<16 x i8> %v)
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @sse41_pmovzxbq(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovzxbq
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:  zext <2 x i8> %1 to <2 x i64>
+; CHECK-NEXT:  ret <2 x i64> %2
+
+  %res = call <2 x i64> @llvm.x86.sse41.pmovzxbq(<16 x i8> %v)
+  ret <2 x i64> %res
+}
+
+define <8 x i16> @sse41_pmovzxbw(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovzxbw
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  zext <8 x i8> %1 to <8 x i16>
+; CHECK-NEXT:  ret <8 x i16> %2
+
+  %res = call <8 x i16> @llvm.x86.sse41.pmovzxbw(<16 x i8> %v)
+  ret <8 x i16> %res
+}
+
+define <2 x i64> @sse41_pmovzxdq(<4 x i32> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovzxdq
+; CHECK-NEXT:  shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:  zext <2 x i32> %1 to <2 x i64>
+; CHECK-NEXT:  ret <2 x i64> %2
+
+  %res = call <2 x i64> @llvm.x86.sse41.pmovzxdq(<4 x i32> %v)
+  ret <2 x i64> %res
+}
+
+define <4 x i32> @sse41_pmovzxwd(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovzxwd
+; CHECK-NEXT:  shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  zext <4 x i16> %1 to <4 x i32>
+; CHECK-NEXT:  ret <4 x i32> %2
+
+  %res = call <4 x i32> @llvm.x86.sse41.pmovzxwd(<8 x i16> %v)
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @sse41_pmovzxwq(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @sse41_pmovzxwq
+; CHECK-NEXT:  shufflevector <8 x i16> %v, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:  zext <2 x i16> %1 to <2 x i64>
+; CHECK-NEXT:  ret <2 x i64> %2
+
+  %res = call <2 x i64> @llvm.x86.sse41.pmovzxwq(<8 x i16> %v)
+  ret <2 x i64> %res
+}
+
+define <8 x i32> @avx2_pmovzxbd(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovzxbd
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  zext <8 x i8> %1 to <8 x i32>
+; CHECK-NEXT:  ret <8 x i32> %2
+
+  %res = call <8 x i32> @llvm.x86.avx2.pmovzxbd(<16 x i8> %v)
+  ret <8 x i32> %res
+}
+
+define <4 x i64> @avx2_pmovzxbq(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovzxbq
+; CHECK-NEXT:  shufflevector <16 x i8> %v, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  zext <4 x i8> %1 to <4 x i64>
+; CHECK-NEXT:  ret <4 x i64> %2
+
+  %res = call <4 x i64> @llvm.x86.avx2.pmovzxbq(<16 x i8> %v)
+  ret <4 x i64> %res
+}
+
+define <16 x i16> @avx2_pmovzxbw(<16 x i8> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovzxbw
+; CHECK-NEXT:  zext <16 x i8> %v to <16 x i16>
+; CHECK-NEXT:  ret <16 x i16> %1
+
+  %res = call <16 x i16> @llvm.x86.avx2.pmovzxbw(<16 x i8> %v)
+  ret <16 x i16> %res
+}
+
+define <4 x i64> @avx2_pmovzxdq(<4 x i32> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovzxdq
+; CHECK-NEXT:  zext <4 x i32> %v to <4 x i64>
+; CHECK-NEXT:  ret <4 x i64> %1
+
+  %res = call <4 x i64> @llvm.x86.avx2.pmovzxdq(<4 x i32> %v)
+  ret <4 x i64> %res
+}
+
+define <8 x i32> @avx2_pmovzxwd(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovzxwd
+; CHECK-NEXT:  zext <8 x i16> %v to <8 x i32>
+; CHECK-NEXT:  ret <8 x i32> %1
+
+  %res = call <8 x i32> @llvm.x86.avx2.pmovzxwd(<8 x i16> %v)
+  ret <8 x i32> %res
+}
+
+define <4 x i64> @avx2_pmovzxwq(<8 x i16> %v) nounwind readnone {
+; CHECK-LABEL: @avx2_pmovzxwq
+; CHECK-NEXT:  shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  zext <4 x i16> %1 to <4 x i64>
+; CHECK-NEXT:  ret <4 x i64> %2
+
+  %res = call <4 x i64> @llvm.x86.avx2.pmovzxwq(<8 x i16> %v)
+  ret <4 x i64> %res
+}