[X86][SSE] Vectorize a bit (AND/XOR/OR) op if a BUILD_VECTOR has the same op for all their scalar elements.

[RKSimon]

Summary

If all a BUILD_VECTOR's source elements are the same bit (AND/XOR/OR) operation type and each has one constant operand, lower to a pair of BUILD_VECTOR and just apply the bit operation to the vectors.

The constant operands will form a constant vector meaning that we still only have a single BUILD_VECTOR to lower and we will have replaced all the scalarized operations with a single SSE equivalent.

Its probably not in our interest to start make a general purpose vectorizer from this, but I'm seeing enough of these scalar bit operations from the later legalization/scalarization stages to support them at least.

[chandlerc]

Thanks for explaining why this is a good idea. I really don't want us doing more at this layer than is necessary to clean up after other legalization transforms, but your explanation clarifies that this is specifically the intent. Also, the test case updates certainly make it very clear that this situation comes up all the time.

Couple of high level points:

1. Why just AND, OR, and XOR? Maybe also support shifts? (fine to leave a TODO for now)
2. We need to leave documentation of the rationale in comments as well. I marked where it seems to fit below.

lib/Target/X86/X86ISelLowering.cpp

6628–6630

This seems like a good place to add comments documenting why its important to do this at this level.

6660–6665

I think using the lambda helper here is a bit overkill. It forces us to do a lot of work that we don't really need to do IMO.

I would get the first element opcode, check that it is one of AND, OR, or XOR, check that it is legal on the target.

Then do the lowering with that opcode.

Shouldn't need a lambda at all, and can use either a switch or 3 tests against ==. I actually think even the switch is likely overkill for just 3 opcodes.

[RKSimon]

Updated based on Chandler's feedback.

Regarding adding support for shifts - I have looked at this (and for add/sub too), but the problem I was seeing was that typicallly one of the elements (#0 most often) didn't share the opcode (i.e. zero shift / offset). I'd be willing to do this for a single 'mis-match' but am worried that it will set a precedent for further relaxations to try and force a vectorization.

[chandlerc]

Looks good with a minor adjustment below.

lib/Target/X86/X86ISelLowering.cpp

6654–6670

I would 'break' from the 3 cases you accept, and add a default that bails out. That will allow you to early-exit and reduce indentation for the entire block here.

lib/Target/X86/X86ISelLowering.cpp

if ((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 ||
     bool isUndefHI = NumUndefsHI == Half;
     return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, true,
                                  isUndefLO, isUndefHI);
   }
 
   return SDValue();
 }
 
+/// If a BUILD_VECTOR's source elements all apply the same bit operation and
+/// one of their operands is constant, lower to a pair of BUILD_VECTOR and
+/// just apply the bit to the vectors.
+/// NOTE: Its not in our interest to start make a general purpose vectorizer
+/// from this, but enough scalar bit operations are created from the later
+/// legalization + scalarization stages to need basic support.
+static SDValue lowerBuildVectorToBitOp(SDValue Op, SelectionDAG &DAG) {
+  SDLoc DL(Op);
+  MVT VT = Op.getSimpleValueType();
+  unsigned NumElems = VT.getVectorNumElements();
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
+  // Check that all elements have the same opcode.
+  // TODO: Should we allow UNDEFS and if so how many?
+  unsigned Opcode = Op.getOperand(0).getOpcode();
+  for (unsigned i = 1; i < NumElems; ++i)
+    if (Opcode != Op.getOperand(i).getOpcode())
+      return SDValue();
+
+  // TODO: We may be able to add support for other Ops (ADD/SUB + shifts).
+  switch (Opcode) {
+  case ISD::AND:
+  case ISD::XOR:
+  case ISD::OR: {
+    if (!TLI.isOperationLegalOrPromote(Opcode, VT))
+      return SDValue();
+
+    SmallVector<SDValue, 4> LHSElts, RHSElts;
+    for (SDValue Elt : Op->ops()) {
+      SDValue LHS = Elt.getOperand(0);
+      SDValue RHS = Elt.getOperand(1);
+
+      // We expect the canonicalized RHS operand to be the constant.
+      if (!isa<ConstantSDNode>(RHS))
+        return SDValue();
+      LHSElts.push_back(LHS);
+      RHSElts.push_back(RHS);
+    }
+
+    SDValue LHS = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, LHSElts);
+    SDValue RHS = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, RHSElts);
+    return DAG.getNode(Opcode, DL, VT, LHS, RHS);
+  }
+  }
+
+  return SDValue();
+}
+
 /// Create a vector constant without a load. SSE/AVX provide the bare minimum
 /// functionality to do this, so it's all zeros, all ones, or some derivation
 /// that is cheap to calculate.
 static SDValue materializeVectorConstant(SDValue Op, SelectionDAG &DAG,
                                          const X86Subtarget &Subtarget) {
   SDLoc DL(Op);
   MVT VT = Op.getSimpleValueType();
 
X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
 
   BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Op.getNode());
   if (SDValue AddSub = LowerToAddSub(BV, Subtarget, DAG))
     return AddSub;
   if (SDValue HorizontalOp = LowerToHorizontalOp(BV, Subtarget, DAG))
     return HorizontalOp;
   if (SDValue Broadcast = LowerVectorBroadcast(Op, Subtarget, DAG))
     return Broadcast;
+  if (SDValue BitOp = lowerBuildVectorToBitOp(Op, DAG))
+    return BitOp;
 
   unsigned EVTBits = ExtVT.getSizeInBits();
 
   unsigned NumZero  = 0;
   unsigned NumNonZero = 0;
   uint64_t NonZeros = 0;
   bool IsAllConstants = true;
   SmallSet<SDValue, 8> Values;

test/CodeGen/X86/pr15267.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-pc-linux -mcpu=corei7-avx | FileCheck %s
 
 define <4 x i3> @test1(<4 x i3>* %in) nounwind {
 ; CHECK-LABEL: test1:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    movzwl (%rdi), %eax
 ; CHECK-NEXT:    movl %eax, %ecx
 ; CHECK-NEXT:    shrl $3, %ecx
-; CHECK-NEXT:    andl $7, %ecx
-; CHECK-NEXT:    movl %eax, %edx
-; CHECK-NEXT:    andl $7, %edx
-; CHECK-NEXT:    vmovd %edx, %xmm0
+; CHECK-NEXT:    vmovd %eax, %xmm0
 ; CHECK-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
 ; CHECK-NEXT:    movl %eax, %ecx
 ; CHECK-NEXT:    shrl $6, %ecx
-; CHECK-NEXT:    andl $7, %ecx
 ; CHECK-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
 ; CHECK-NEXT:    shrl $9, %eax
-; CHECK-NEXT:    andl $7, %eax
 ; CHECK-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; CHECK-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %ret = load <4 x i3>, <4 x i3>* %in, align 1
   ret <4 x i3> %ret
 }
 
 define <4 x i1> @test2(<4 x i1>* %in) nounwind {
 ; CHECK-LABEL: test2:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    movzbl (%rdi), %eax
 ; CHECK-NEXT:    movl %eax, %ecx
 ; CHECK-NEXT:    shrl %ecx
-; CHECK-NEXT:    andl $1, %ecx
-; CHECK-NEXT:    movl %eax, %edx
-; CHECK-NEXT:    andl $1, %edx
-; CHECK-NEXT:    vmovd %edx, %xmm0
+; CHECK-NEXT:    vmovd %eax, %xmm0
 ; CHECK-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
 ; CHECK-NEXT:    movl %eax, %ecx
 ; CHECK-NEXT:    shrl $2, %ecx
-; CHECK-NEXT:    andl $1, %ecx
 ; CHECK-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
 ; CHECK-NEXT:    shrl $3, %eax
-; CHECK-NEXT:    andl $1, %eax
 ; CHECK-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; CHECK-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %ret = load <4 x i1>, <4 x i1>* %in, align 1
   ret <4 x i1> %ret
 }
 
 define <4 x i64> @test3(<4 x i1>* %in) nounwind {
 ; CHECK-LABEL: test3:
 ; CHECK:       # BB#0:

test/CodeGen/X86/vector-lzcnt-256.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512cd -mattr=+avx512vl | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512VLCD
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512cd | FileCheck %s --check-prefix=ALL --check-prefix=AVX512 --check-prefix=AVX512CD
 
 define <4 x i64> @testv4i64(<4 x i64> %in) nounwind {
 ; AVX1-LABEL: testv4i64:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
 ; AVX1-NEXT:    bsrq %rax, %rax
 ; AVX1-NEXT:    movl $127, %ecx
 ; AVX1-NEXT:    cmoveq %rcx, %rax
-; AVX1-NEXT:    xorq $63, %rax
 ; AVX1-NEXT:    vmovq %rax, %xmm2
 ; AVX1-NEXT:    vmovq %xmm1, %rax
 ; AVX1-NEXT:    bsrq %rax, %rax
 ; AVX1-NEXT:    cmoveq %rcx, %rax
-; AVX1-NEXT:    xorq $63, %rax
 ; AVX1-NEXT:    vmovq %rax, %xmm1
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
+; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX1-NEXT:    bsrq %rax, %rax
 ; AVX1-NEXT:    cmoveq %rcx, %rax
-; AVX1-NEXT:    xorq $63, %rax
-; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %rax, %xmm3
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    bsrq %rax, %rax
 ; AVX1-NEXT:    cmoveq %rcx, %rax
-; AVX1-NEXT:    xorq $63, %rax
 ; AVX1-NEXT:    vmovq %rax, %xmm0
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: testv4i64:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
 ; AVX2-NEXT:    bsrq %rax, %rax
 ; AVX2-NEXT:    movl $127, %ecx
 ; AVX2-NEXT:    cmoveq %rcx, %rax
-; AVX2-NEXT:    xorq $63, %rax
 ; AVX2-NEXT:    vmovq %rax, %xmm2
 ; AVX2-NEXT:    vmovq %xmm1, %rax
 ; AVX2-NEXT:    bsrq %rax, %rax
 ; AVX2-NEXT:    cmoveq %rcx, %rax
-; AVX2-NEXT:    xorq $63, %rax
 ; AVX2-NEXT:    vmovq %rax, %xmm1
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
+; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX2-NEXT:    bsrq %rax, %rax
 ; AVX2-NEXT:    cmoveq %rcx, %rax
-; AVX2-NEXT:    xorq $63, %rax
-; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %rax, %xmm3
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    bsrq %rax, %rax
 ; AVX2-NEXT:    cmoveq %rcx, %rax
-; AVX2-NEXT:    xorq $63, %rax
 ; AVX2-NEXT:    vmovq %rax, %xmm0
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512VLCD-LABEL: testv4i64:
 ; AVX512VLCD:       ## BB#0:
 ; AVX512VLCD-NEXT:    vplzcntq %ymm0, %ymm0
 ; AVX512VLCD-NEXT:    retq
 ;
 ; AVX512CD-LABEL: testv4i64:
 ; AVX512CD:       ## BB#0:
 ; AVX512CD-NEXT:    vplzcntq %zmm0, %zmm0
 ; AVX512CD-NEXT:    retq
 
   %out = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %in, i1 0)
   ret <4 x i64> %out
 }
 
 define <4 x i64> @testv4i64u(<4 x i64> %in) nounwind {
 ; AVX1-LABEL: testv4i64u:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
 ; AVX1-NEXT:    bsrq %rax, %rax
-; AVX1-NEXT:    xorq $63, %rax
 ; AVX1-NEXT:    vmovq %rax, %xmm2
 ; AVX1-NEXT:    vmovq %xmm1, %rax
 ; AVX1-NEXT:    bsrq %rax, %rax
-; AVX1-NEXT:    xorq $63, %rax
 ; AVX1-NEXT:    vmovq %rax, %xmm1
 ; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
+; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX1-NEXT:    bsrq %rax, %rax
-; AVX1-NEXT:    xorq $63, %rax
-; AVX1-NEXT:    vmovq %rax, %xmm2
+; AVX1-NEXT:    vmovq %rax, %xmm3
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    bsrq %rax, %rax
-; AVX1-NEXT:    xorq $63, %rax
 ; AVX1-NEXT:    vmovq %rax, %xmm0
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: testv4i64u:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
 ; AVX2-NEXT:    bsrq %rax, %rax
-; AVX2-NEXT:    xorq $63, %rax
 ; AVX2-NEXT:    vmovq %rax, %xmm2
 ; AVX2-NEXT:    vmovq %xmm1, %rax
 ; AVX2-NEXT:    bsrq %rax, %rax
-; AVX2-NEXT:    xorq $63, %rax
 ; AVX2-NEXT:    vmovq %rax, %xmm1
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
+; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rax
 ; AVX2-NEXT:    bsrq %rax, %rax
-; AVX2-NEXT:    xorq $63, %rax
-; AVX2-NEXT:    vmovq %rax, %xmm2
+; AVX2-NEXT:    vmovq %rax, %xmm3
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    bsrq %rax, %rax
-; AVX2-NEXT:    xorq $63, %rax
 ; AVX2-NEXT:    vmovq %rax, %xmm0
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512VLCD-LABEL: testv4i64u:
 ; AVX512VLCD:       ## BB#0:
 ; AVX512VLCD-NEXT:    vplzcntq %ymm0, %ymm0
 ; AVX512VLCD-NEXT:    retq
 ;

 define <8 x i32> @testv8i32(<8 x i32> %in) nounwind {
 ; AVX1-LABEL: testv8i32:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpextrd $1, %xmm1, %eax
 ; AVX1-NEXT:    bsrl %eax, %ecx
 ; AVX1-NEXT:    movl $63, %eax
 ; AVX1-NEXT:    cmovel %eax, %ecx
-; AVX1-NEXT:    xorl $31, %ecx
 ; AVX1-NEXT:    vmovd %xmm1, %edx
 ; AVX1-NEXT:    bsrl %edx, %edx
 ; AVX1-NEXT:    cmovel %eax, %edx
-; AVX1-NEXT:    xorl $31, %edx
 ; AVX1-NEXT:    vmovd %edx, %xmm2
 ; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
 ; AVX1-NEXT:    bsrl %ecx, %ecx
 ; AVX1-NEXT:    cmovel %eax, %ecx
-; AVX1-NEXT:    xorl $31, %ecx
 ; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrd $3, %xmm1, %ecx
 ; AVX1-NEXT:    bsrl %ecx, %ecx
 ; AVX1-NEXT:    cmovel %eax, %ecx
-; AVX1-NEXT:    xorl $31, %ecx
 ; AVX1-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31]
+; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpextrd $1, %xmm0, %ecx
 ; AVX1-NEXT:    bsrl %ecx, %ecx
 ; AVX1-NEXT:    cmovel %eax, %ecx
-; AVX1-NEXT:    xorl $31, %ecx
 ; AVX1-NEXT:    vmovd %xmm0, %edx
 ; AVX1-NEXT:    bsrl %edx, %edx
 ; AVX1-NEXT:    cmovel %eax, %edx
-; AVX1-NEXT:    xorl $31, %edx
-; AVX1-NEXT:    vmovd %edx, %xmm2
-; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vmovd %edx, %xmm3
+; AVX1-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrd $2, %xmm0, %ecx
 ; AVX1-NEXT:    bsrl %ecx, %ecx
 ; AVX1-NEXT:    cmovel %eax, %ecx
-; AVX1-NEXT:    xorl $31, %ecx
-; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpinsrd $2, %ecx, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrd $3, %xmm0, %ecx
 ; AVX1-NEXT:    bsrl %ecx, %ecx
 ; AVX1-NEXT:    cmovel %eax, %ecx
-; AVX1-NEXT:    xorl $31, %ecx
-; AVX1-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm0
+; AVX1-NEXT:    vpinsrd $3, %ecx, %xmm3, %xmm0
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: testv8i32:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpextrd $1, %xmm1, %eax
 ; AVX2-NEXT:    bsrl %eax, %ecx
 ; AVX2-NEXT:    movl $63, %eax
 ; AVX2-NEXT:    cmovel %eax, %ecx
-; AVX2-NEXT:    xorl $31, %ecx
 ; AVX2-NEXT:    vmovd %xmm1, %edx
 ; AVX2-NEXT:    bsrl %edx, %edx
 ; AVX2-NEXT:    cmovel %eax, %edx
-; AVX2-NEXT:    xorl $31, %edx
 ; AVX2-NEXT:    vmovd %edx, %xmm2
 ; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
 ; AVX2-NEXT:    bsrl %ecx, %ecx
 ; AVX2-NEXT:    cmovel %eax, %ecx
-; AVX2-NEXT:    xorl $31, %ecx
 ; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrd $3, %xmm1, %ecx
 ; AVX2-NEXT:    bsrl %ecx, %ecx
 ; AVX2-NEXT:    cmovel %eax, %ecx
-; AVX2-NEXT:    xorl $31, %ecx
 ; AVX2-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm1
+; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpextrd $1, %xmm0, %ecx
 ; AVX2-NEXT:    bsrl %ecx, %ecx
 ; AVX2-NEXT:    cmovel %eax, %ecx
-; AVX2-NEXT:    xorl $31, %ecx
 ; AVX2-NEXT:    vmovd %xmm0, %edx
 ; AVX2-NEXT:    bsrl %edx, %edx
 ; AVX2-NEXT:    cmovel %eax, %edx
-; AVX2-NEXT:    xorl $31, %edx
-; AVX2-NEXT:    vmovd %edx, %xmm2
-; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vmovd %edx, %xmm3
+; AVX2-NEXT:    vpinsrd $1, %ecx, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrd $2, %xmm0, %ecx
 ; AVX2-NEXT:    bsrl %ecx, %ecx
 ; AVX2-NEXT:    cmovel %eax, %ecx
-; AVX2-NEXT:    xorl $31, %ecx
-; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpinsrd $2, %ecx, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrd $3, %xmm0, %ecx
 ; AVX2-NEXT:    bsrl %ecx, %ecx
 ; AVX2-NEXT:    cmovel %eax, %ecx
-; AVX2-NEXT:    xorl $31, %ecx
-; AVX2-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm0
+; AVX2-NEXT:    vpinsrd $3, %ecx, %xmm3, %xmm0
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512VLCD-LABEL: testv8i32:
 ; AVX512VLCD:       ## BB#0:
 ; AVX512VLCD-NEXT:    vplzcntd %ymm0, %ymm0
 ; AVX512VLCD-NEXT:    retq
 ;
 ; AVX512CD-LABEL: testv8i32:
 ; AVX512CD:       ## BB#0:
 ; AVX512CD-NEXT:    vplzcntd %zmm0, %zmm0
 ; AVX512CD-NEXT:    retq
 
   %out = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %in, i1 0)
   ret <8 x i32> %out
 }
 
 define <8 x i32> @testv8i32u(<8 x i32> %in) nounwind {
 ; AVX1-LABEL: testv8i32u:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpextrd $1, %xmm1, %eax
 ; AVX1-NEXT:    bsrl %eax, %eax
-; AVX1-NEXT:    xorl $31, %eax
 ; AVX1-NEXT:    vmovd %xmm1, %ecx
 ; AVX1-NEXT:    bsrl %ecx, %ecx
-; AVX1-NEXT:    xorl $31, %ecx
 ; AVX1-NEXT:    vmovd %ecx, %xmm2
 ; AVX1-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrd $2, %xmm1, %eax
 ; AVX1-NEXT:    bsrl %eax, %eax
-; AVX1-NEXT:    xorl $31, %eax
 ; AVX1-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
 ; AVX1-NEXT:    bsrl %eax, %eax
-; AVX1-NEXT:    xorl $31, %eax
 ; AVX1-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31]
+; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpextrd $1, %xmm0, %eax
 ; AVX1-NEXT:    bsrl %eax, %eax
-; AVX1-NEXT:    xorl $31, %eax
 ; AVX1-NEXT:    vmovd %xmm0, %ecx
 ; AVX1-NEXT:    bsrl %ecx, %ecx
-; AVX1-NEXT:    xorl $31, %ecx
-; AVX1-NEXT:    vmovd %ecx, %xmm2
-; AVX1-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vmovd %ecx, %xmm3
+; AVX1-NEXT:    vpinsrd $1, %eax, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrd $2, %xmm0, %eax
 ; AVX1-NEXT:    bsrl %eax, %eax
-; AVX1-NEXT:    xorl $31, %eax
-; AVX1-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpinsrd $2, %eax, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrd $3, %xmm0, %eax
 ; AVX1-NEXT:    bsrl %eax, %eax
-; AVX1-NEXT:    xorl $31, %eax
-; AVX1-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX1-NEXT:    vpinsrd $3, %eax, %xmm3, %xmm0
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: testv8i32u:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpextrd $1, %xmm1, %eax
 ; AVX2-NEXT:    bsrl %eax, %eax
-; AVX2-NEXT:    xorl $31, %eax
 ; AVX2-NEXT:    vmovd %xmm1, %ecx
 ; AVX2-NEXT:    bsrl %ecx, %ecx
-; AVX2-NEXT:    xorl $31, %ecx
 ; AVX2-NEXT:    vmovd %ecx, %xmm2
 ; AVX2-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrd $2, %xmm1, %eax
 ; AVX2-NEXT:    bsrl %eax, %eax
-; AVX2-NEXT:    xorl $31, %eax
 ; AVX2-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
 ; AVX2-NEXT:    bsrl %eax, %eax
-; AVX2-NEXT:    xorl $31, %eax
 ; AVX2-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm1
+; AVX2-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpextrd $1, %xmm0, %eax
 ; AVX2-NEXT:    bsrl %eax, %eax
-; AVX2-NEXT:    xorl $31, %eax
 ; AVX2-NEXT:    vmovd %xmm0, %ecx
 ; AVX2-NEXT:    bsrl %ecx, %ecx
-; AVX2-NEXT:    xorl $31, %ecx
-; AVX2-NEXT:    vmovd %ecx, %xmm2
-; AVX2-NEXT:    vpinsrd $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vmovd %ecx, %xmm3
+; AVX2-NEXT:    vpinsrd $1, %eax, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrd $2, %xmm0, %eax
 ; AVX2-NEXT:    bsrl %eax, %eax
-; AVX2-NEXT:    xorl $31, %eax
-; AVX2-NEXT:    vpinsrd $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpinsrd $2, %eax, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrd $3, %xmm0, %eax
 ; AVX2-NEXT:    bsrl %eax, %eax
-; AVX2-NEXT:    xorl $31, %eax
-; AVX2-NEXT:    vpinsrd $3, %eax, %xmm2, %xmm0
+; AVX2-NEXT:    vpinsrd $3, %eax, %xmm3, %xmm0
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512VLCD-LABEL: testv8i32u:
 ; AVX512VLCD:       ## BB#0:
 ; AVX512VLCD-NEXT:    vplzcntd %ymm0, %ymm0
 ; AVX512VLCD-NEXT:    retq
 ;

 define <16 x i16> @testv16i16(<16 x i16> %in) nounwind {
 ; AVX1-LABEL: testv16i16:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpextrw $1, %xmm1, %eax
 ; AVX1-NEXT:    bsrw %ax, %cx
 ; AVX1-NEXT:    movw $31, %ax
 ; AVX1-NEXT:    cmovew %ax, %cx
-; AVX1-NEXT:    xorl $15, %ecx
 ; AVX1-NEXT:    vmovd %xmm1, %edx
 ; AVX1-NEXT:    bsrw %dx, %dx
 ; AVX1-NEXT:    cmovew %ax, %dx
-; AVX1-NEXT:    xorl $15, %edx
 ; AVX1-NEXT:    vmovd %edx, %xmm2
 ; AVX1-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrw $2, %xmm1, %ecx
 ; AVX1-NEXT:    bsrw %cx, %cx
 ; AVX1-NEXT:    cmovew %ax, %cx
-; AVX1-NEXT:    xorl $15, %ecx
 ; AVX1-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrw $3, %xmm1, %ecx
 ; AVX1-NEXT:    bsrw %cx, %cx
 ; AVX1-NEXT:    cmovew %ax, %cx
-; AVX1-NEXT:    xorl $15, %ecx
 ; AVX1-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrw $4, %xmm1, %ecx
 ; AVX1-NEXT:    bsrw %cx, %cx
 ; AVX1-NEXT:    cmovew %ax, %cx
-; AVX1-NEXT:    xorl $15, %ecx
 ; AVX1-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrw $5, %xmm1, %ecx
 ; AVX1-NEXT:    bsrw %cx, %cx
 ; AVX1-NEXT:    cmovew %ax, %cx
-; AVX1-NEXT:    xorl $15, %ecx
 ; AVX1-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrw $6, %xmm1, %ecx
 ; AVX1-NEXT:    bsrw %cx, %cx
 ; AVX1-NEXT:    cmovew %ax, %cx
-; AVX1-NEXT:    xorl $15, %ecx
 ; AVX1-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrw $7, %xmm1, %ecx
 ; AVX1-NEXT:    bsrw %cx, %cx
 ; AVX1-NEXT:    cmovew %ax, %cx
-; AVX1-NEXT:    xorl $15, %ecx
 ; AVX1-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
+; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpextrw $1, %xmm0, %ecx
 ; AVX1-NEXT:    bsrw %cx, %cx
 ; AVX1-NEXT:    cmovew %ax, %cx
-; AVX1-NEXT:    xorl $15, %ecx
 ; AVX1-NEXT:    vmovd %xmm0, %edx
 ; AVX1-NEXT:    bsrw %dx, %dx
 ; AVX1-NEXT:    cmovew %ax, %dx
-; AVX1-NEXT:    xorl $15, %edx
-; AVX1-NEXT:    vmovd %edx, %xmm2
-; AVX1-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vmovd %edx, %xmm3
+; AVX1-NEXT:    vpinsrw $1, %ecx, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrw $2, %xmm0, %ecx
 ; AVX1-NEXT:    bsrw %cx, %cx
 ; AVX1-NEXT:    cmovew %ax, %cx
-; AVX1-NEXT:    xorl $15, %ecx
-; AVX1-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpinsrw $2, %ecx, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrw $3, %xmm0, %ecx
 ; AVX1-NEXT:    bsrw %cx, %cx
 ; AVX1-NEXT:    cmovew %ax, %cx
-; AVX1-NEXT:    xorl $15, %ecx
-; AVX1-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpinsrw $3, %ecx, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrw $4, %xmm0, %ecx
 ; AVX1-NEXT:    bsrw %cx, %cx
 ; AVX1-NEXT:    cmovew %ax, %cx
-; AVX1-NEXT:    xorl $15, %ecx
-; AVX1-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpinsrw $4, %ecx, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrw $5, %xmm0, %ecx
 ; AVX1-NEXT:    bsrw %cx, %cx
 ; AVX1-NEXT:    cmovew %ax, %cx
-; AVX1-NEXT:    xorl $15, %ecx
-; AVX1-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpinsrw $5, %ecx, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrw $6, %xmm0, %ecx
 ; AVX1-NEXT:    bsrw %cx, %cx
 ; AVX1-NEXT:    cmovew %ax, %cx
-; AVX1-NEXT:    xorl $15, %ecx
-; AVX1-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX1-NEXT:    vpinsrw $6, %ecx, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrw $7, %xmm0, %ecx
 ; AVX1-NEXT:    bsrw %cx, %cx
 ; AVX1-NEXT:    cmovew %ax, %cx
-; AVX1-NEXT:    xorl $15, %ecx
-; AVX1-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm0
+; AVX1-NEXT:    vpinsrw $7, %ecx, %xmm3, %xmm0
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: testv16i16:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpextrw $1, %xmm1, %eax
 ; AVX2-NEXT:    bsrw %ax, %cx
 ; AVX2-NEXT:    movw $31, %ax
 ; AVX2-NEXT:    cmovew %ax, %cx
-; AVX2-NEXT:    xorl $15, %ecx
 ; AVX2-NEXT:    vmovd %xmm1, %edx
 ; AVX2-NEXT:    bsrw %dx, %dx
 ; AVX2-NEXT:    cmovew %ax, %dx
-; AVX2-NEXT:    xorl $15, %edx
 ; AVX2-NEXT:    vmovd %edx, %xmm2
 ; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrw $2, %xmm1, %ecx
 ; AVX2-NEXT:    bsrw %cx, %cx
 ; AVX2-NEXT:    cmovew %ax, %cx
-; AVX2-NEXT:    xorl $15, %ecx
 ; AVX2-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrw $3, %xmm1, %ecx
 ; AVX2-NEXT:    bsrw %cx, %cx
 ; AVX2-NEXT:    cmovew %ax, %cx
-; AVX2-NEXT:    xorl $15, %ecx
 ; AVX2-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrw $4, %xmm1, %ecx
 ; AVX2-NEXT:    bsrw %cx, %cx
 ; AVX2-NEXT:    cmovew %ax, %cx
-; AVX2-NEXT:    xorl $15, %ecx
 ; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrw $5, %xmm1, %ecx
 ; AVX2-NEXT:    bsrw %cx, %cx
 ; AVX2-NEXT:    cmovew %ax, %cx
-; AVX2-NEXT:    xorl $15, %ecx
 ; AVX2-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrw $6, %xmm1, %ecx
 ; AVX2-NEXT:    bsrw %cx, %cx
 ; AVX2-NEXT:    cmovew %ax, %cx
-; AVX2-NEXT:    xorl $15, %ecx
 ; AVX2-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrw $7, %xmm1, %ecx
 ; AVX2-NEXT:    bsrw %cx, %cx
 ; AVX2-NEXT:    cmovew %ax, %cx
-; AVX2-NEXT:    xorl $15, %ecx
 ; AVX2-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpextrw $1, %xmm0, %ecx
 ; AVX2-NEXT:    bsrw %cx, %cx
 ; AVX2-NEXT:    cmovew %ax, %cx
-; AVX2-NEXT:    xorl $15, %ecx
 ; AVX2-NEXT:    vmovd %xmm0, %edx
 ; AVX2-NEXT:    bsrw %dx, %dx
 ; AVX2-NEXT:    cmovew %ax, %dx
-; AVX2-NEXT:    xorl $15, %edx
-; AVX2-NEXT:    vmovd %edx, %xmm2
-; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vmovd %edx, %xmm3
+; AVX2-NEXT:    vpinsrw $1, %ecx, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrw $2, %xmm0, %ecx
 ; AVX2-NEXT:    bsrw %cx, %cx
 ; AVX2-NEXT:    cmovew %ax, %cx
-; AVX2-NEXT:    xorl $15, %ecx
-; AVX2-NEXT:    vpinsrw $2, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpinsrw $2, %ecx, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrw $3, %xmm0, %ecx
 ; AVX2-NEXT:    bsrw %cx, %cx
 ; AVX2-NEXT:    cmovew %ax, %cx
-; AVX2-NEXT:    xorl $15, %ecx
-; AVX2-NEXT:    vpinsrw $3, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpinsrw $3, %ecx, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrw $4, %xmm0, %ecx
 ; AVX2-NEXT:    bsrw %cx, %cx
 ; AVX2-NEXT:    cmovew %ax, %cx
-; AVX2-NEXT:    xorl $15, %ecx
-; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpinsrw $4, %ecx, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrw $5, %xmm0, %ecx
 ; AVX2-NEXT:    bsrw %cx, %cx
 ; AVX2-NEXT:    cmovew %ax, %cx
-; AVX2-NEXT:    xorl $15, %ecx
-; AVX2-NEXT:    vpinsrw $5, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpinsrw $5, %ecx, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrw $6, %xmm0, %ecx
 ; AVX2-NEXT:    bsrw %cx, %cx
 ; AVX2-NEXT:    cmovew %ax, %cx
-; AVX2-NEXT:    xorl $15, %ecx
-; AVX2-NEXT:    vpinsrw $6, %ecx, %xmm2, %xmm2
+; AVX2-NEXT:    vpinsrw $6, %ecx, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrw $7, %xmm0, %ecx
 ; AVX2-NEXT:    bsrw %cx, %cx
 ; AVX2-NEXT:    cmovew %ax, %cx
-; AVX2-NEXT:    xorl $15, %ecx
-; AVX2-NEXT:    vpinsrw $7, %ecx, %xmm2, %xmm0
+; AVX2-NEXT:    vpinsrw $7, %ecx, %xmm3, %xmm0
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: testv16i16:
 ; AVX512:       ## BB#0:
 ; AVX512-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512-NEXT:    vplzcntd %zmm0, %zmm0
 ; AVX512-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512-NEXT:    vpsubw {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512-NEXT:    retq
   %out = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %in, i1 0)
   ret <16 x i16> %out
 }
 
 define <16 x i16> @testv16i16u(<16 x i16> %in) nounwind {
 ; AVX1-LABEL: testv16i16u:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpextrw $1, %xmm1, %eax
 ; AVX1-NEXT:    bsrw %ax, %ax
-; AVX1-NEXT:    xorl $15, %eax
 ; AVX1-NEXT:    vmovd %xmm1, %ecx
 ; AVX1-NEXT:    bsrw %cx, %cx
-; AVX1-NEXT:    xorl $15, %ecx
 ; AVX1-NEXT:    vmovd %ecx, %xmm2
 ; AVX1-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrw $2, %xmm1, %eax
 ; AVX1-NEXT:    bsrw %ax, %ax
-; AVX1-NEXT:    xorl $15, %eax
 ; AVX1-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrw $3, %xmm1, %eax
 ; AVX1-NEXT:    bsrw %ax, %ax
-; AVX1-NEXT:    xorl $15, %eax
 ; AVX1-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrw $4, %xmm1, %eax
 ; AVX1-NEXT:    bsrw %ax, %ax
-; AVX1-NEXT:    xorl $15, %eax
 ; AVX1-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrw $5, %xmm1, %eax
 ; AVX1-NEXT:    bsrw %ax, %ax
-; AVX1-NEXT:    xorl $15, %eax
 ; AVX1-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrw $6, %xmm1, %eax
 ; AVX1-NEXT:    bsrw %ax, %ax
-; AVX1-NEXT:    xorl $15, %eax
 ; AVX1-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
 ; AVX1-NEXT:    vpextrw $7, %xmm1, %eax
 ; AVX1-NEXT:    bsrw %ax, %ax
-; AVX1-NEXT:    xorl $15, %eax
 ; AVX1-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
+; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX1-NEXT:    bsrw %ax, %ax
-; AVX1-NEXT:    xorl $15, %eax
 ; AVX1-NEXT:    vmovd %xmm0, %ecx
 ; AVX1-NEXT:    bsrw %cx, %cx
-; AVX1-NEXT:    xorl $15, %ecx
-; AVX1-NEXT:    vmovd %ecx, %xmm2
-; AVX1-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vmovd %ecx, %xmm3
+; AVX1-NEXT:    vpinsrw $1, %eax, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrw $2, %xmm0, %eax
 ; AVX1-NEXT:    bsrw %ax, %ax
-; AVX1-NEXT:    xorl $15, %eax
-; AVX1-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpinsrw $2, %eax, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrw $3, %xmm0, %eax
 ; AVX1-NEXT:    bsrw %ax, %ax
-; AVX1-NEXT:    xorl $15, %eax
-; AVX1-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrw $4, %xmm0, %eax
 ; AVX1-NEXT:    bsrw %ax, %ax
-; AVX1-NEXT:    xorl $15, %eax
-; AVX1-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrw $5, %xmm0, %eax
 ; AVX1-NEXT:    bsrw %ax, %ax
-; AVX1-NEXT:    xorl $15, %eax
-; AVX1-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrw $6, %xmm0, %eax
 ; AVX1-NEXT:    bsrw %ax, %ax
-; AVX1-NEXT:    xorl $15, %eax
-; AVX1-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm3
 ; AVX1-NEXT:    vpextrw $7, %xmm0, %eax
 ; AVX1-NEXT:    bsrw %ax, %ax
-; AVX1-NEXT:    xorl $15, %eax
-; AVX1-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX1-NEXT:    vpinsrw $7, %eax, %xmm3, %xmm0
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: testv16i16u:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpextrw $1, %xmm1, %eax
 ; AVX2-NEXT:    bsrw %ax, %ax
-; AVX2-NEXT:    xorl $15, %eax
 ; AVX2-NEXT:    vmovd %xmm1, %ecx
 ; AVX2-NEXT:    bsrw %cx, %cx
-; AVX2-NEXT:    xorl $15, %ecx
 ; AVX2-NEXT:    vmovd %ecx, %xmm2
 ; AVX2-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrw $2, %xmm1, %eax
 ; AVX2-NEXT:    bsrw %ax, %ax
-; AVX2-NEXT:    xorl $15, %eax
 ; AVX2-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrw $3, %xmm1, %eax
 ; AVX2-NEXT:    bsrw %ax, %ax
-; AVX2-NEXT:    xorl $15, %eax
 ; AVX2-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrw $4, %xmm1, %eax
 ; AVX2-NEXT:    bsrw %ax, %ax
-; AVX2-NEXT:    xorl $15, %eax
 ; AVX2-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrw $5, %xmm1, %eax
 ; AVX2-NEXT:    bsrw %ax, %ax
-; AVX2-NEXT:    xorl $15, %eax
 ; AVX2-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrw $6, %xmm1, %eax
 ; AVX2-NEXT:    bsrw %ax, %ax
-; AVX2-NEXT:    xorl $15, %eax
 ; AVX2-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
 ; AVX2-NEXT:    vpextrw $7, %xmm1, %eax
 ; AVX2-NEXT:    bsrw %ax, %ax
-; AVX2-NEXT:    xorl $15, %eax
 ; AVX2-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
+; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX2-NEXT:    bsrw %ax, %ax
-; AVX2-NEXT:    xorl $15, %eax
 ; AVX2-NEXT:    vmovd %xmm0, %ecx
 ; AVX2-NEXT:    bsrw %cx, %cx
-; AVX2-NEXT:    xorl $15, %ecx
-; AVX2-NEXT:    vmovd %ecx, %xmm2
-; AVX2-NEXT:    vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vmovd %ecx, %xmm3
+; AVX2-NEXT:    vpinsrw $1, %eax, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrw $2, %xmm0, %eax
 ; AVX2-NEXT:    bsrw %ax, %ax
-; AVX2-NEXT:    xorl $15, %eax
-; AVX2-NEXT:    vpinsrw $2, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpinsrw $2, %eax, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrw $3, %xmm0, %eax
 ; AVX2-NEXT:    bsrw %ax, %ax
-; AVX2-NEXT:    xorl $15, %eax
-; AVX2-NEXT:    vpinsrw $3, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpinsrw $3, %eax, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrw $4, %xmm0, %eax
 ; AVX2-NEXT:    bsrw %ax, %ax
-; AVX2-NEXT:    xorl $15, %eax
-; AVX2-NEXT:    vpinsrw $4, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpinsrw $4, %eax, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrw $5, %xmm0, %eax
 ; AVX2-NEXT:    bsrw %ax, %ax
-; AVX2-NEXT:    xorl $15, %eax
-; AVX2-NEXT:    vpinsrw $5, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpinsrw $5, %eax, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrw $6, %xmm0, %eax
 ; AVX2-NEXT:    bsrw %ax, %ax
-; AVX2-NEXT:    xorl $15, %eax
-; AVX2-NEXT:    vpinsrw $6, %eax, %xmm2, %xmm2
+; AVX2-NEXT:    vpinsrw $6, %eax, %xmm3, %xmm3
 ; AVX2-NEXT:    vpextrw $7, %xmm0, %eax
 ; AVX2-NEXT:    bsrw %ax, %ax
-; AVX2-NEXT:    xorl $15, %eax
-; AVX2-NEXT:    vpinsrw $7, %eax, %xmm2, %xmm0
+; AVX2-NEXT:    vpinsrw $7, %eax, %xmm3, %xmm0
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: testv16i16u:
 ; AVX512:       ## BB#0:
 ; AVX512-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512-NEXT:    vplzcntd %zmm0, %zmm0
 ; AVX512-NEXT:    vpmovdw %zmm0, %ymm0

test/CodeGen/X86/vector-pcmp.ll

; AVX-NEXT:    retq
   %not = xor <2 x i64> %sign, <i64 -1, i64 -1>
   ret <2 x i64> %not
 }
 
 define <1 x i128> @test_strange_type(<1 x i128> %x) {
 ; SSE2-LABEL: test_strange_type:
 ; SSE2:       # BB#0:
 ; SSE2-NEXT:    sarq $63, %rsi
-; SSE2-NEXT:    notq %rsi
 ; SSE2-NEXT:    movd %rsi, %xmm0
+; SSE2-NEXT:    notq %rsi
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE2-NEXT:    movd %xmm0, %rax
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %rax
 ; SSE2-NEXT:    movq %rsi, %rdx
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: test_strange_type:
 ; SSE42:       # BB#0:
 ; SSE42-NEXT:    sarq $63, %rsi
-; SSE42-NEXT:    notq %rsi
 ; SSE42-NEXT:    movd %rsi, %xmm0
 ; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; SSE42-NEXT:    movd %xmm0, %rax
-; SSE42-NEXT:    pextrq $1, %xmm0, %rdx
+; SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE42-NEXT:    pxor %xmm0, %xmm1
+; SSE42-NEXT:    movd %xmm1, %rax
+; SSE42-NEXT:    pextrq $1, %xmm1, %rdx
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: test_strange_type:
 ; AVX1:       # BB#0:
 ; AVX1-NEXT:    sarq $63, %rsi
-; AVX1-NEXT:    notq %rsi
 ; AVX1-NEXT:    vmovq %rsi, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vpextrq $1, %xmm0, %rdx
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: test_strange_type:
 ; AVX2:       # BB#0:
 ; AVX2-NEXT:    sarq $63, %rsi
-; AVX2-NEXT:    notq %rsi
 ; AVX2-NEXT:    vmovq %rsi, %xmm0
 ; AVX2-NEXT:    vpbroadcastq %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rdx
 ; AVX2-NEXT:    retq
   %sign = ashr <1 x i128> %x, <i128 127>
   %not = xor <1 x i128> %sign, <i128 -1>
   ret <1 x i128> %not
 }
 

test/CodeGen/X86/vector-sext.ll

  %X = load <4 x i8>, <4 x i8>* %ptr
  %Y = sext <4 x i8> %X to <4 x i32>
  ret <4 x i32> %Y
 }
 
 define <4 x i64> @load_sext_4i1_to_4i64(<4 x i1> *%ptr) {
 ; SSE2-LABEL: load_sext_4i1_to_4i64:
 ; SSE2:       # BB#0: # %entry
-; SSE2-NEXT:    movzbl (%rdi), %eax
+; SSE2-NEXT:    movl (%rdi), %eax
 ; SSE2-NEXT:    movl %eax, %ecx
 ; SSE2-NEXT:    shrl $3, %ecx
-; SSE2-NEXT:    andl $1, %ecx
 ; SSE2-NEXT:    movd %ecx, %xmm0
 ; SSE2-NEXT:    movl %eax, %ecx
 ; SSE2-NEXT:    shrl %ecx
-; SSE2-NEXT:    andl $1, %ecx
 ; SSE2-NEXT:    movd %ecx, %xmm1
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    movl %eax, %ecx
-; SSE2-NEXT:    andl $1, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    movd %eax, %xmm2
 ; SSE2-NEXT:    shrl $2, %eax
-; SSE2-NEXT:    andl $1, %eax
 ; SSE2-NEXT:    movd %eax, %xmm0
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3]
 ; SSE2-NEXT:    psllq $63, %xmm0
 ; SSE2-NEXT:    psrad $31, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]
 ; SSE2-NEXT:    psllq $63, %xmm1
 ; SSE2-NEXT:    psrad $31, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: load_sext_4i1_to_4i64:
 ; SSSE3:       # BB#0: # %entry
-; SSSE3-NEXT:    movzbl (%rdi), %eax
+; SSSE3-NEXT:    movl (%rdi), %eax
 ; SSSE3-NEXT:    movl %eax, %ecx
 ; SSSE3-NEXT:    shrl $3, %ecx
-; SSSE3-NEXT:    andl $1, %ecx
 ; SSSE3-NEXT:    movd %ecx, %xmm0
 ; SSSE3-NEXT:    movl %eax, %ecx
 ; SSSE3-NEXT:    shrl %ecx
-; SSSE3-NEXT:    andl $1, %ecx
 ; SSSE3-NEXT:    movd %ecx, %xmm1
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT:    movl %eax, %ecx
-; SSSE3-NEXT:    andl $1, %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    movd %eax, %xmm2
 ; SSSE3-NEXT:    shrl $2, %eax
-; SSSE3-NEXT:    andl $1, %eax
 ; SSSE3-NEXT:    movd %eax, %xmm0
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,1,1,3]
 ; SSSE3-NEXT:    psllq $63, %xmm0
 ; SSSE3-NEXT:    psrad $31, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,1,3,3]
 ; SSSE3-NEXT:    psllq $63, %xmm1
 ; SSSE3-NEXT:    psrad $31, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: load_sext_4i1_to_4i64:
 ; SSE41:       # BB#0: # %entry
-; SSE41-NEXT:    movzbl (%rdi), %eax
+; SSE41-NEXT:    movl (%rdi), %eax
 ; SSE41-NEXT:    movl %eax, %ecx
 ; SSE41-NEXT:    shrl %ecx
-; SSE41-NEXT:    andl $1, %ecx
-; SSE41-NEXT:    movl %eax, %edx
-; SSE41-NEXT:    andl $1, %edx
-; SSE41-NEXT:    movd %edx, %xmm1
+; SSE41-NEXT:    movd %eax, %xmm1
 ; SSE41-NEXT:    pinsrd $1, %ecx, %xmm1
 ; SSE41-NEXT:    movl %eax, %ecx
 ; SSE41-NEXT:    shrl $2, %ecx
-; SSE41-NEXT:    andl $1, %ecx
 ; SSE41-NEXT:    pinsrd $2, %ecx, %xmm1
 ; SSE41-NEXT:    shrl $3, %eax
-; SSE41-NEXT:    andl $1, %eax
 ; SSE41-NEXT:    pinsrd $3, %eax, %xmm1
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
 ; SSE41-NEXT:    psllq $63, %xmm0
 ; SSE41-NEXT:    psrad $31, %xmm0
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
 ; SSE41-NEXT:    psllq $63, %xmm1
 ; SSE41-NEXT:    psrad $31, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]

 ; AVX2-NEXT:    retq
 ;
 ; X32-SSE41-LABEL: load_sext_4i1_to_4i64:
 ; X32-SSE41:       # BB#0: # %entry
 ; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-SSE41-NEXT:    movzbl (%eax), %eax
 ; X32-SSE41-NEXT:    movl %eax, %ecx
 ; X32-SSE41-NEXT:    shrl %ecx
-; X32-SSE41-NEXT:    andl $1, %ecx
-; X32-SSE41-NEXT:    movl %eax, %edx
-; X32-SSE41-NEXT:    andl $1, %edx
-; X32-SSE41-NEXT:    movd %edx, %xmm1
+; X32-SSE41-NEXT:    movd %eax, %xmm1
 ; X32-SSE41-NEXT:    pinsrd $1, %ecx, %xmm1
 ; X32-SSE41-NEXT:    movl %eax, %ecx
 ; X32-SSE41-NEXT:    shrl $2, %ecx
-; X32-SSE41-NEXT:    andl $1, %ecx
 ; X32-SSE41-NEXT:    pinsrd $2, %ecx, %xmm1
 ; X32-SSE41-NEXT:    shrl $3, %eax
-; X32-SSE41-NEXT:    andl $1, %eax
 ; X32-SSE41-NEXT:    pinsrd $3, %eax, %xmm1
+; X32-SSE41-NEXT:    pand .LCPI16_0, %xmm1
 ; X32-SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
 ; X32-SSE41-NEXT:    psllq $63, %xmm0
 ; X32-SSE41-NEXT:    psrad $31, %xmm0
 ; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
 ; X32-SSE41-NEXT:    psllq $63, %xmm1
 ; X32-SSE41-NEXT:    psrad $31, %xmm1
 ; X32-SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]