[X86] Allow pre-SSE41 targets to extract multiple v16i8 elements coming from the same DWORD/WORD super-element

[RKSimon]

Summary

Pre-SSE41 targets tended to have weak (serial) GPR<->VEC moves, meaning we only allowed a single extraction before spilling the vector to stack and loading the element instead. But this didn't make use of the DWORD/WORD extraction we had to use could extract multiple i8 elements at the same time.

This patch attempts to determine if all uses of a vector are element extractions, and works out whether all the extractions share the same WORD or (lowest) DWORD, in which case we can perform a single extraction and just shift/truncate the individual elements.

[goldstein.w.n]

llvm/lib/Target/X86/X86ISelLowering.cpp

20575

ZEXT_MOVL too no?

20581

Shouldn't the number of bits set be a function of the extraction width? I.e id expect pextrw to set 2x as many bits as pextrb or will it never be legal to have pextr* that doesn't match ele width?

[RKSimon]

llvm/lib/Target/X86/X86ISelLowering.cpp

20575

we only need to handle extractions from the vector to gpr - ZEXT_MOVL is the other way around.

20581

That should be handled by recursion via the BITCAST case below - PEXTRW / PEXTRB only work with their own v8i16 / v16i8 types (although tbh we never needed seperate node types - creating a single X86ISD::PEXTR nodetype would have been enough),

[RKSimon]

any other comments?

[RKSimon]

ping?

[pengfei]

llvm/lib/Target/X86/X86ISelLowering.cpp

20695

Why we use getSizeInBits rather than check for i8?

20713

It's not clear to me here, the old code should have more chance to generate SRL than the new code due to the restriction. Which one it better? I didn't find a case to reflect the difference.

[RKSimon]

llvm/lib/Target/X86/X86ISelLowering.cpp

20695

I'm not sure - my guess if whoever did it was following the pattern for 32-bit scalars below, but that was to handle floats. I'll change it in a pre-commit.

20713

I'm not sure I understand? The original code was always limited to a single extract, and for odd-indices (greater than 3) PEXTRW+SRL would be used. The change just means that we allow a pair of extracts (odd + even) that share a PEXTRW - so we still allow the SRL case.

[RKSimon]

rebase

[pengfei]

LGTM.

llvm/lib/Target/X86/X86ISelLowering.cpp

20713

I see the point, thanks!

llvm/lib/Target/X86/X86ISelLowering.cpp

static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
   // Use kshiftr instruction to move to the lower element.
   Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideVecVT, Vec,
                     DAG.getTargetConstant(IdxVal, dl, MVT::i8));
 
   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
                      DAG.getIntPtrConstant(0, dl));
 }
 
+// Helper to find all the extracted elements from a vector.
+static APInt getExtractedDemandedElts(SDNode *N) {
+  MVT VT = N->getSimpleValueType(0);
+  unsigned NumElts = VT.getVectorNumElements();
+  APInt DemandedElts = APInt::getZero(NumElts);
+  for (SDNode *User : N->uses()) {
+    switch (User->getOpcode()) {
+    case X86ISD::PEXTRB:
+    case X86ISD::PEXTRW:
+    case ISD::EXTRACT_VECTOR_ELT:
+      if (!isa<ConstantSDNode>(User->getOperand(1))) {
+        DemandedElts.setAllBits();
+        return DemandedElts;
+      }
+      DemandedElts.setBit(User->getConstantOperandVal(1));
+      break;
+    case ISD::BITCAST: {
+      if (!User->getValueType(0).isSimple() ||
+          !User->getValueType(0).isVector()) {
+        DemandedElts.setAllBits();
+        return DemandedElts;
+      }
+      APInt DemandedSrcElts = getExtractedDemandedElts(User);
+      DemandedElts |= APIntOps::ScaleBitMask(DemandedSrcElts, NumElts);
+      break;
+    }
+    default:
+      DemandedElts.setAllBits();
+      return DemandedElts;
+    }
+  }
+  return DemandedElts;
+}
+
 SDValue
 X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
                                            SelectionDAG &DAG) const {
   SDLoc dl(Op);
   SDValue Vec = Op.getOperand(0);
   MVT VecVT = Vec.getSimpleValueType();
   SDValue Idx = Op.getOperand(1);
   auto* IdxC = dyn_cast<ConstantSDNode>(Idx);
SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, Vec,
                                   DAG.getTargetConstant(IdxVal, dl, MVT::i8));
     return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract);
   }
 
   if (Subtarget.hasSSE41())
     if (SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG))
       return Res;
 
-  // TODO: We only extract a single element from v16i8, we can probably afford
-  // to be more aggressive here before using the default approach of spilling to
-  // stack.
-  if (VT == MVT::i8 && Op->isOnlyUserOf(Vec.getNode())) {
+  // Only extract a single element from a v16i8 source - determine the common
+  // DWORD/WORD that all extractions share, and extract the sub-byte.
+  // TODO: Add QWORD MOVQ extraction?
+  if (VT == MVT::i8) {
+    APInt DemandedElts = getExtractedDemandedElts(Vec.getNode());
+    assert(DemandedElts.getBitWidth() == 16 && "Vector width mismatch");
+
     // Extract either the lowest i32 or any i16, and extract the sub-byte.
     int DWordIdx = IdxVal / 4;
-    if (DWordIdx == 0) {
+    if (DWordIdx == 0 && DemandedElts == (DemandedElts & 15)) {
       SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
                                 DAG.getBitcast(MVT::v4i32, Vec),
                                 DAG.getIntPtrConstant(DWordIdx, dl));
       int ShiftVal = (IdxVal % 4) * 8;
       if (ShiftVal != 0)
         Res = DAG.getNode(ISD::SRL, dl, MVT::i32, Res,
                           DAG.getConstant(ShiftVal, dl, MVT::i8));
       return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
     }
 
     int WordIdx = IdxVal / 2;
+    if (DemandedElts == (DemandedElts & (3 << (WordIdx * 2)))) {
       SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16,
                                 DAG.getBitcast(MVT::v8i16, Vec),
                                 DAG.getIntPtrConstant(WordIdx, dl));
       int ShiftVal = (IdxVal % 2) * 8;
       if (ShiftVal != 0)
         Res = DAG.getNode(ISD::SRL, dl, MVT::i16, Res,
                           DAG.getConstant(ShiftVal, dl, MVT::i8));
       return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
     }
+  }
 
   if (VT == MVT::f16 || VT.getSizeInBits() == 32) {
     if (IdxVal == 0)
       return Op;
 
     // Shuffle the element to the lowest element, then movss or movsh.
     SmallVector<int, 8> Mask(VecVT.getVectorNumElements(), -1);
     Mask[0] = static_cast<int>(IdxVal);
if ((VT == MVT::v4i32 || VT == MVT::v8i32) &&
       assert(!Subtarget.hasVLX() && "Unexpected features!");
       MVT WideVT = SrcVT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32;
       MVT ResVT = SrcVT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32;
       // Need to concat with zero vector for strict fp to avoid spurious
       // exceptions.
       // TODO: Should we just do this for non-strict as well?
       SDValue Tmp =
           IsStrict ? DAG.getConstantFP(0.0, dl, WideVT) : DAG.getUNDEF(WideVT);
       Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Tmp, Src,
                         DAG.getIntPtrConstant(0, dl));
 
       if (IsStrict) {
         Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, dl, {ResVT, MVT::Other},
                           {Chain, Src});
         Chain = Res.getValue(1);
       } else {
         Res = DAG.getNode(ISD::FP_TO_UINT, dl, ResVT, Src);
       }
 
       Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
                         DAG.getIntPtrConstant(0, dl));
 
       if (IsStrict)
if (!IsStrict && ((VT == MVT::i32 && !Subtarget.is64Bit()) ||
       // (i64) 0 to 2^63-1 ("Small") and from 2^63 to 2^64-1 ("Big").
       SDValue Small =
           DAG.getNode(X86ISD::CVTTS2SI, dl, VT,
                       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, SrcVecVT, Src));
       SDValue Big = DAG.getNode(
           X86ISD::CVTTS2SI, dl, VT,
           DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, SrcVecVT,
                       DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FloatOffset)));
 
       // The "CVTTS2SI" instruction conveniently sets the sign bit if
       // and only if the value was out of range. So we can use that
       // as our indicator that we rather use "Big" instead of "Small".
       //
       // Use "Small" if "IsOverflown" has all bits cleared
       // and "0x80000000 | Big" if all bits in "IsOverflown" are set.
       SDValue IsOverflown = DAG.getNode(
           ISD::SRA, dl, VT, Small, DAG.getConstant(DstBits - 1, dl, MVT::i8));
       return DAG.getNode(ISD::OR, dl, VT, Small,
                          DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown));
     }
 
     // Use default expansion for i64.
     if (VT == MVT::i64)
       return SDValue();
 
     assert(VT == MVT::i32 && "Unexpected VT!");
 
     // Promote i32 to i64 and use a signed operation on 64-bit targets.
     // FIXME: This does not generate an invalid exception if the input does not
     // fit in i32. PR44019
     if (Subtarget.is64Bit()) {
       if (IsStrict) {
         Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {MVT::i64, MVT::Other},
                           {Chain, Src});
         Chain = Res.getValue(1);

llvm/test/CodeGen/X86/bitcast-vector-bool.ll

 ; AVX512-NEXT:    retq
   %1 = trunc <8 x i16> %a0 to <8 x i1>
   %2 = bitcast <8 x i1> %1 to i8
   %3 = icmp ne i8 %2, 0
   ret i1 %3
 }
 
 define i8 @bitcast_v16i8_to_v2i8(<16 x i8> %a0) nounwind {
-; SSE2-SSSE3-LABEL: bitcast_v16i8_to_v2i8:
-; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-SSSE3-NEXT:    movd %eax, %xmm0
-; SSE2-SSSE3-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-SSSE3-NEXT:    addb -{{[0-9]+}}(%rsp), %al
-; SSE2-SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: bitcast_v16i8_to_v2i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovmskb %xmm0, %ecx
-; SSE41-NEXT:    movl %ecx, %eax
-; SSE41-NEXT:    shrl $8, %eax
-; SSE41-NEXT:    addb %cl, %al
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE-LABEL: bitcast_v16i8_to_v2i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmovmskb %xmm0, %ecx
+; SSE-NEXT:    movl %ecx, %eax
+; SSE-NEXT:    shrl $8, %eax
+; SSE-NEXT:    addb %cl, %al
+; SSE-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-NEXT:    retq
 ;
 ; AVX12-LABEL: bitcast_v16i8_to_v2i8:
 ; AVX12:       # %bb.0:
 ; AVX12-NEXT:    vpmovmskb %xmm0, %ecx
 ; AVX12-NEXT:    movl %ecx, %eax
 ; AVX12-NEXT:    shrl $8, %eax
 ; AVX12-NEXT:    addb %cl, %al
 ; AVX12-NEXT:    # kill: def $al killed $al killed $eax

 ; AVX512-NEXT:    retq
   %1 = trunc <8 x i32> %a0 to <8 x i1>
   %2 = bitcast <8 x i1> %1 to i8
   %3 = icmp ne i8 %2, -1
   ret i1 %3
 }
 
 define i8 @bitcast_v16i16_to_v2i8(<16 x i16> %a0) nounwind {
-; SSE2-SSSE3-LABEL: bitcast_v16i16_to_v2i8:
-; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    packsswb %xmm1, %xmm0
-; SSE2-SSSE3-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-SSSE3-NEXT:    movd %eax, %xmm0
-; SSE2-SSSE3-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-SSSE3-NEXT:    addb -{{[0-9]+}}(%rsp), %al
-; SSE2-SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: bitcast_v16i16_to_v2i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    packsswb %xmm1, %xmm0
-; SSE41-NEXT:    pmovmskb %xmm0, %ecx
-; SSE41-NEXT:    movl %ecx, %eax
-; SSE41-NEXT:    shrl $8, %eax
-; SSE41-NEXT:    addb %cl, %al
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE-LABEL: bitcast_v16i16_to_v2i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    packsswb %xmm1, %xmm0
+; SSE-NEXT:    pmovmskb %xmm0, %ecx
+; SSE-NEXT:    movl %ecx, %eax
+; SSE-NEXT:    shrl $8, %eax
+; SSE-NEXT:    addb %cl, %al
+; SSE-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: bitcast_v16i16_to_v2i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %ecx
 ; AVX1-NEXT:    movl %ecx, %eax
 ; AVX1-NEXT:    shrl $8, %eax

 ; AVX512-NEXT:    retq
   %1 = trunc <8 x i64> %a0 to <8 x i1>
   %2 = bitcast <8 x i1> %1 to i8
   %3 = icmp eq i8 %2, -1
   ret i1 %3
 }
 
 define i8 @bitcast_v16i32_to_v2i8(<16 x i32> %a0) nounwind {
-; SSE2-SSSE3-LABEL: bitcast_v16i32_to_v2i8:
-; SSE2-SSSE3:       # %bb.0:
-; SSE2-SSSE3-NEXT:    packssdw %xmm3, %xmm2
-; SSE2-SSSE3-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-SSSE3-NEXT:    packsswb %xmm2, %xmm0
-; SSE2-SSSE3-NEXT:    pmovmskb %xmm0, %eax
-; SSE2-SSSE3-NEXT:    movd %eax, %xmm0
-; SSE2-SSSE3-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-SSSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-SSSE3-NEXT:    addb -{{[0-9]+}}(%rsp), %al
-; SSE2-SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: bitcast_v16i32_to_v2i8:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    packssdw %xmm3, %xmm2
-; SSE41-NEXT:    packssdw %xmm1, %xmm0
-; SSE41-NEXT:    packsswb %xmm2, %xmm0
-; SSE41-NEXT:    pmovmskb %xmm0, %ecx
-; SSE41-NEXT:    movl %ecx, %eax
-; SSE41-NEXT:    shrl $8, %eax
-; SSE41-NEXT:    addb %cl, %al
-; SSE41-NEXT:    # kill: def $al killed $al killed $eax
-; SSE41-NEXT:    retq
+; SSE-LABEL: bitcast_v16i32_to_v2i8:
+; SSE:       # %bb.0:
+; SSE-NEXT:    packssdw %xmm3, %xmm2
+; SSE-NEXT:    packssdw %xmm1, %xmm0
+; SSE-NEXT:    packsswb %xmm2, %xmm0
+; SSE-NEXT:    pmovmskb %xmm0, %ecx
+; SSE-NEXT:    movl %ecx, %eax
+; SSE-NEXT:    shrl $8, %eax
+; SSE-NEXT:    addb %cl, %al
+; SSE-NEXT:    # kill: def $al killed $al killed $eax
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: bitcast_v16i32_to_v2i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0

llvm/test/CodeGen/X86/pr63108.ll

 ; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE-NEXT:    pxor %xmm0, %xmm1
 ; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
 ; SSE-NEXT:    pxor %xmm1, %xmm2
 ; SSE-NEXT:    movdqa %xmm2, %xmm0
 ; SSE-NEXT:    psrld $16, %xmm0
 ; SSE-NEXT:    pxor %xmm2, %xmm0
 ; SSE-NEXT:  .LBB0_5: # %for.cond.cleanup
-; SSE-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE-NEXT:    movsbl -{{[0-9]+}}(%rsp), %ecx
-; SSE-NEXT:    movsbl -{{[0-9]+}}(%rsp), %eax
+; SSE-NEXT:    movd %xmm0, %eax
+; SSE-NEXT:    movsbl %al, %ecx
+; SSE-NEXT:    shrl $8, %eax
+; SSE-NEXT:    movsbl %al, %eax
 ; SSE-NEXT:    addl %ecx, %eax
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: PR63108:
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    xorl %eax, %eax
 ; AVX1-NEXT:    testb %al, %al
 ; AVX1-NEXT:    je .LBB0_2

llvm/test/CodeGen/X86/subvectorwise-store-of-vector-splat.ll

 ; SCALAR-NEXT:    movb %al, 58(%rdx)
 ; SCALAR-NEXT:    movw %cx, 56(%rdx)
 ; SCALAR-NEXT:    movb %al, 62(%rdx)
 ; SCALAR-NEXT:    movw %cx, 60(%rdx)
 ; SCALAR-NEXT:    retq
 ;
 ; SSE2-ONLY-LABEL: vec384_v3i8:
 ; SSE2-ONLY:       # %bb.0:
-; SSE2-ONLY-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-ONLY-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-ONLY-NEXT:    pxor %xmm0, %xmm1
-; SSE2-ONLY-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp)
-; SSE2-ONLY-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-ONLY-NEXT:    movb %al, 2(%rsi)
-; SSE2-ONLY-NEXT:    movd %xmm1, %ecx
-; SSE2-ONLY-NEXT:    movw %cx, (%rsi)
-; SSE2-ONLY-NEXT:    movb %al, 2(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, (%rdx)
-; SSE2-ONLY-NEXT:    movb %al, 6(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, 4(%rdx)
-; SSE2-ONLY-NEXT:    movb %al, 10(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, 8(%rdx)
-; SSE2-ONLY-NEXT:    movb %al, 14(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, 12(%rdx)
-; SSE2-ONLY-NEXT:    movb %al, 18(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, 16(%rdx)
-; SSE2-ONLY-NEXT:    movb %al, 22(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, 20(%rdx)
-; SSE2-ONLY-NEXT:    movb %al, 26(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, 24(%rdx)
-; SSE2-ONLY-NEXT:    movb %al, 30(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, 28(%rdx)
-; SSE2-ONLY-NEXT:    movb %al, 34(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, 32(%rdx)
-; SSE2-ONLY-NEXT:    movb %al, 38(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, 36(%rdx)
-; SSE2-ONLY-NEXT:    movb %al, 42(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, 40(%rdx)
-; SSE2-ONLY-NEXT:    movb %al, 46(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, 44(%rdx)
-; SSE2-ONLY-NEXT:    movb %al, 50(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, 48(%rdx)
-; SSE2-ONLY-NEXT:    movb %al, 54(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, 52(%rdx)
-; SSE2-ONLY-NEXT:    movb %al, 58(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, 56(%rdx)
-; SSE2-ONLY-NEXT:    movb %al, 62(%rdx)
-; SSE2-ONLY-NEXT:    movw %cx, 60(%rdx)
+; SSE2-ONLY-NEXT:    movl (%rdi), %eax
+; SSE2-ONLY-NEXT:    notl %eax
+; SSE2-ONLY-NEXT:    movw %ax, (%rsi)
+; SSE2-ONLY-NEXT:    movl %eax, %ecx
+; SSE2-ONLY-NEXT:    shrl $16, %ecx
+; SSE2-ONLY-NEXT:    movb %cl, 2(%rsi)
+; SSE2-ONLY-NEXT:    movb %cl, 2(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, (%rdx)
+; SSE2-ONLY-NEXT:    movb %cl, 6(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, 4(%rdx)
+; SSE2-ONLY-NEXT:    movb %cl, 10(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, 8(%rdx)
+; SSE2-ONLY-NEXT:    movb %cl, 14(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, 12(%rdx)
+; SSE2-ONLY-NEXT:    movb %cl, 18(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, 16(%rdx)
+; SSE2-ONLY-NEXT:    movb %cl, 22(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, 20(%rdx)
+; SSE2-ONLY-NEXT:    movb %cl, 26(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, 24(%rdx)
+; SSE2-ONLY-NEXT:    movb %cl, 30(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, 28(%rdx)
+; SSE2-ONLY-NEXT:    movb %cl, 34(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, 32(%rdx)
+; SSE2-ONLY-NEXT:    movb %cl, 38(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, 36(%rdx)
+; SSE2-ONLY-NEXT:    movb %cl, 42(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, 40(%rdx)
+; SSE2-ONLY-NEXT:    movb %cl, 46(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, 44(%rdx)
+; SSE2-ONLY-NEXT:    movb %cl, 50(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, 48(%rdx)
+; SSE2-ONLY-NEXT:    movb %cl, 54(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, 52(%rdx)
+; SSE2-ONLY-NEXT:    movb %cl, 58(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, 56(%rdx)
+; SSE2-ONLY-NEXT:    movb %cl, 62(%rdx)
+; SSE2-ONLY-NEXT:    movw %ax, 60(%rdx)
 ; SSE2-ONLY-NEXT:    retq
 ;
 ; SSE3-LABEL: vec384_v3i8:
 ; SSE3:       # %bb.0:
-; SSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE3-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE3-NEXT:    pxor %xmm0, %xmm1
-; SSE3-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp)
-; SSE3-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE3-NEXT:    movb %al, 2(%rsi)
-; SSE3-NEXT:    movd %xmm1, %ecx
-; SSE3-NEXT:    movw %cx, (%rsi)
-; SSE3-NEXT:    movb %al, 2(%rdx)
-; SSE3-NEXT:    movw %cx, (%rdx)
-; SSE3-NEXT:    movb %al, 6(%rdx)
-; SSE3-NEXT:    movw %cx, 4(%rdx)
-; SSE3-NEXT:    movb %al, 10(%rdx)
-; SSE3-NEXT:    movw %cx, 8(%rdx)
-; SSE3-NEXT:    movb %al, 14(%rdx)
-; SSE3-NEXT:    movw %cx, 12(%rdx)
-; SSE3-NEXT:    movb %al, 18(%rdx)
-; SSE3-NEXT:    movw %cx, 16(%rdx)
-; SSE3-NEXT:    movb %al, 22(%rdx)
-; SSE3-NEXT:    movw %cx, 20(%rdx)
-; SSE3-NEXT:    movb %al, 26(%rdx)
-; SSE3-NEXT:    movw %cx, 24(%rdx)
-; SSE3-NEXT:    movb %al, 30(%rdx)
-; SSE3-NEXT:    movw %cx, 28(%rdx)
-; SSE3-NEXT:    movb %al, 34(%rdx)
-; SSE3-NEXT:    movw %cx, 32(%rdx)
-; SSE3-NEXT:    movb %al, 38(%rdx)
-; SSE3-NEXT:    movw %cx, 36(%rdx)
-; SSE3-NEXT:    movb %al, 42(%rdx)
-; SSE3-NEXT:    movw %cx, 40(%rdx)
-; SSE3-NEXT:    movb %al, 46(%rdx)
-; SSE3-NEXT:    movw %cx, 44(%rdx)
-; SSE3-NEXT:    movb %al, 50(%rdx)
-; SSE3-NEXT:    movw %cx, 48(%rdx)
-; SSE3-NEXT:    movb %al, 54(%rdx)
-; SSE3-NEXT:    movw %cx, 52(%rdx)
-; SSE3-NEXT:    movb %al, 58(%rdx)
-; SSE3-NEXT:    movw %cx, 56(%rdx)
-; SSE3-NEXT:    movb %al, 62(%rdx)
-; SSE3-NEXT:    movw %cx, 60(%rdx)
+; SSE3-NEXT:    movl (%rdi), %eax
+; SSE3-NEXT:    notl %eax
+; SSE3-NEXT:    movw %ax, (%rsi)
+; SSE3-NEXT:    movl %eax, %ecx
+; SSE3-NEXT:    shrl $16, %ecx
+; SSE3-NEXT:    movb %cl, 2(%rsi)
+; SSE3-NEXT:    movb %cl, 2(%rdx)
+; SSE3-NEXT:    movw %ax, (%rdx)
+; SSE3-NEXT:    movb %cl, 6(%rdx)
+; SSE3-NEXT:    movw %ax, 4(%rdx)
+; SSE3-NEXT:    movb %cl, 10(%rdx)
+; SSE3-NEXT:    movw %ax, 8(%rdx)
+; SSE3-NEXT:    movb %cl, 14(%rdx)
+; SSE3-NEXT:    movw %ax, 12(%rdx)
+; SSE3-NEXT:    movb %cl, 18(%rdx)
+; SSE3-NEXT:    movw %ax, 16(%rdx)
+; SSE3-NEXT:    movb %cl, 22(%rdx)
+; SSE3-NEXT:    movw %ax, 20(%rdx)
+; SSE3-NEXT:    movb %cl, 26(%rdx)
+; SSE3-NEXT:    movw %ax, 24(%rdx)
+; SSE3-NEXT:    movb %cl, 30(%rdx)
+; SSE3-NEXT:    movw %ax, 28(%rdx)
+; SSE3-NEXT:    movb %cl, 34(%rdx)
+; SSE3-NEXT:    movw %ax, 32(%rdx)
+; SSE3-NEXT:    movb %cl, 38(%rdx)
+; SSE3-NEXT:    movw %ax, 36(%rdx)
+; SSE3-NEXT:    movb %cl, 42(%rdx)
+; SSE3-NEXT:    movw %ax, 40(%rdx)
+; SSE3-NEXT:    movb %cl, 46(%rdx)
+; SSE3-NEXT:    movw %ax, 44(%rdx)
+; SSE3-NEXT:    movb %cl, 50(%rdx)
+; SSE3-NEXT:    movw %ax, 48(%rdx)
+; SSE3-NEXT:    movb %cl, 54(%rdx)
+; SSE3-NEXT:    movw %ax, 52(%rdx)
+; SSE3-NEXT:    movb %cl, 58(%rdx)
+; SSE3-NEXT:    movw %ax, 56(%rdx)
+; SSE3-NEXT:    movb %cl, 62(%rdx)
+; SSE3-NEXT:    movw %ax, 60(%rdx)
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-ONLY-LABEL: vec384_v3i8:
 ; SSSE3-ONLY:       # %bb.0:
-; SSSE3-ONLY-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSSE3-ONLY-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSSE3-ONLY-NEXT:    pxor %xmm0, %xmm1
-; SSSE3-ONLY-NEXT:    movdqa %xmm1, -{{[0-9]+}}(%rsp)
-; SSSE3-ONLY-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSSE3-ONLY-NEXT:    movb %al, 2(%rsi)
-; SSSE3-ONLY-NEXT:    movd %xmm1, %ecx
-; SSSE3-ONLY-NEXT:    movw %cx, (%rsi)
-; SSSE3-ONLY-NEXT:    movb %al, 2(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, (%rdx)
-; SSSE3-ONLY-NEXT:    movb %al, 6(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, 4(%rdx)
-; SSSE3-ONLY-NEXT:    movb %al, 10(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, 8(%rdx)
-; SSSE3-ONLY-NEXT:    movb %al, 14(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, 12(%rdx)
-; SSSE3-ONLY-NEXT:    movb %al, 18(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, 16(%rdx)
-; SSSE3-ONLY-NEXT:    movb %al, 22(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, 20(%rdx)
-; SSSE3-ONLY-NEXT:    movb %al, 26(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, 24(%rdx)
-; SSSE3-ONLY-NEXT:    movb %al, 30(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, 28(%rdx)
-; SSSE3-ONLY-NEXT:    movb %al, 34(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, 32(%rdx)
-; SSSE3-ONLY-NEXT:    movb %al, 38(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, 36(%rdx)
-; SSSE3-ONLY-NEXT:    movb %al, 42(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, 40(%rdx)
-; SSSE3-ONLY-NEXT:    movb %al, 46(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, 44(%rdx)
-; SSSE3-ONLY-NEXT:    movb %al, 50(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, 48(%rdx)
-; SSSE3-ONLY-NEXT:    movb %al, 54(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, 52(%rdx)
-; SSSE3-ONLY-NEXT:    movb %al, 58(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, 56(%rdx)
-; SSSE3-ONLY-NEXT:    movb %al, 62(%rdx)
-; SSSE3-ONLY-NEXT:    movw %cx, 60(%rdx)
+; SSSE3-ONLY-NEXT:    movl (%rdi), %eax
+; SSSE3-ONLY-NEXT:    notl %eax
+; SSSE3-ONLY-NEXT:    movw %ax, (%rsi)
+; SSSE3-ONLY-NEXT:    movl %eax, %ecx
+; SSSE3-ONLY-NEXT:    shrl $16, %ecx
+; SSSE3-ONLY-NEXT:    movb %cl, 2(%rsi)
+; SSSE3-ONLY-NEXT:    movb %cl, 2(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, (%rdx)
+; SSSE3-ONLY-NEXT:    movb %cl, 6(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, 4(%rdx)
+; SSSE3-ONLY-NEXT:    movb %cl, 10(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, 8(%rdx)
+; SSSE3-ONLY-NEXT:    movb %cl, 14(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, 12(%rdx)
+; SSSE3-ONLY-NEXT:    movb %cl, 18(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, 16(%rdx)
+; SSSE3-ONLY-NEXT:    movb %cl, 22(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, 20(%rdx)
+; SSSE3-ONLY-NEXT:    movb %cl, 26(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, 24(%rdx)
+; SSSE3-ONLY-NEXT:    movb %cl, 30(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, 28(%rdx)
+; SSSE3-ONLY-NEXT:    movb %cl, 34(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, 32(%rdx)
+; SSSE3-ONLY-NEXT:    movb %cl, 38(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, 36(%rdx)
+; SSSE3-ONLY-NEXT:    movb %cl, 42(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, 40(%rdx)
+; SSSE3-ONLY-NEXT:    movb %cl, 46(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, 44(%rdx)
+; SSSE3-ONLY-NEXT:    movb %cl, 50(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, 48(%rdx)
+; SSSE3-ONLY-NEXT:    movb %cl, 54(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, 52(%rdx)
+; SSSE3-ONLY-NEXT:    movb %cl, 58(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, 56(%rdx)
+; SSSE3-ONLY-NEXT:    movb %cl, 62(%rdx)
+; SSSE3-ONLY-NEXT:    movw %ax, 60(%rdx)
 ; SSSE3-ONLY-NEXT:    retq
 ;
 ; SSE41-LABEL: vec384_v3i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE41-NEXT:    pcmpeqd %xmm0, %xmm0
 ; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $2, %xmm0, 2(%rsi)