[X86] Add a DAG combine pre type legalization to widen division by constant splat on narrow vectors to avoid scalarization

[craig.topper]

Summary

This is another patch for -x86-experimental-vector-widening. This pre widens narrow division by constants so that we can get pass the legal type check in the generic DAG combiner. Otherwise we end up scalarizing.

I've restricted this to splats for now because it was easy to just call DAG.getConstant. Not sure what we should do for non-splat? Increase the element size?Widen the constant vector by padding with 1?

[RKSimon]

LGTM

lib/Target/X86/X86ISelLowering.cpp

41100 ↗

(On Diff #175343)

match indentation of the other combines

llvm/trunk/lib/Target/X86/X86ISelLowering.cpp

   setTargetDAGCombine(ISD::ZERO_EXTEND_VECTOR_INREG);
   setTargetDAGCombine(ISD::SINT_TO_FP);
   setTargetDAGCombine(ISD::UINT_TO_FP);
   setTargetDAGCombine(ISD::SETCC);
   setTargetDAGCombine(ISD::MUL);
   setTargetDAGCombine(ISD::XOR);
   setTargetDAGCombine(ISD::MSCATTER);
   setTargetDAGCombine(ISD::MGATHER);
+  setTargetDAGCombine(ISD::SDIV);
+  setTargetDAGCombine(ISD::UDIV);
+  setTargetDAGCombine(ISD::SREM);
+  setTargetDAGCombine(ISD::UREM);
 
   computeRegisterProperties(Subtarget.getRegisterInfo());
 
   MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
   MaxStoresPerMemsetOptSize = 8;
   MaxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores
   MaxStoresPerMemcpyOptSize = 4;
   MaxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores
static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
   // PMULDQ/PMULUDQ only uses lower 32 bits from each vector element.
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnesValue(64), DCI))
     return SDValue(N, 0);
 
   return SDValue();
 }
 
+// Try to widen division/remainder by splat constant to avoid scalarization.
+// TODO: Can we do something for non-splat?
+static SDValue combineDivRem(SDNode *N, SelectionDAG &DAG,
+                             TargetLowering::DAGCombinerInfo &DCI) {
+  if (!DCI.isBeforeLegalize())
+    return SDValue();
+
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  LLVMContext &Context = *DAG.getContext();
+  EVT VT = N->getValueType(0);
+  if (!VT.isVector() || 128 % VT.getSizeInBits() != 0 ||
+      TLI.getTypeAction(Context, VT) != TargetLowering::TypeWidenVector)
+    return SDValue();
+
+  APInt SplatVal;
+  if (!ISD::isConstantSplatVector(N->getOperand(1).getNode(), SplatVal))
+    return SDValue();
+
+  unsigned NumConcats = 128 / VT.getSizeInBits();
+  SmallVector<SDValue, 8> Ops0(NumConcats, DAG.getUNDEF(VT));
+  Ops0[0] = N->getOperand(0);
+
+  SDLoc dl(N);
+  EVT ResVT = TLI.getTypeToTransformTo(Context, VT);
+  SDValue N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Ops0);
+  SDValue N1 = DAG.getConstant(SplatVal, dl, ResVT);
+  SDValue Res = DAG.getNode(N->getOpcode(), dl, ResVT, N0, N1);
+  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
+                     DAG.getIntPtrConstant(0, dl));
+}
+
 SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   switch (N->getOpcode()) {
   default: break;
   case ISD::SCALAR_TO_VECTOR:
     return combineScalarToVector(N, DAG);
   case ISD::EXTRACT_VECTOR_ELT:
SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
   case X86ISD::MGATHER:
   case X86ISD::MSCATTER:
   case ISD::MGATHER:
   case ISD::MSCATTER:       return combineGatherScatter(N, DAG, DCI, Subtarget);
   case X86ISD::PCMPEQ:
   case X86ISD::PCMPGT:      return combineVectorCompare(N, DAG, Subtarget);
   case X86ISD::PMULDQ:
   case X86ISD::PMULUDQ:     return combinePMULDQ(N, DAG, DCI);
+  case ISD::UDIV:
+  case ISD::SDIV:
+  case ISD::UREM:
+  case ISD::SREM:           return combineDivRem(N, DAG, DCI);
   }
 
   return SDValue();
 }
 
 bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const {
   if (!isTypeLegal(VT))
     return false;

llvm/trunk/test/CodeGen/X86/vector-idiv-v2i32.ll

 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    retl
 ;
 ; X64_WIDEN-LABEL: test_udiv7_v2i32:
 ; X64_WIDEN:       # %bb.0:
 ; X64_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    imulq $613566757, %rax, %rcx # imm = 0x24924925
-; X64_WIDEN-NEXT:    shrq $32, %rcx
-; X64_WIDEN-NEXT:    subl %ecx, %eax
-; X64_WIDEN-NEXT:    shrl %eax
-; X64_WIDEN-NEXT:    addl %ecx, %eax
-; X64_WIDEN-NEXT:    shrl $2, %eax
-; X64_WIDEN-NEXT:    movd %eax, %xmm1
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    imulq $613566757, %rax, %rcx # imm = 0x24924925
-; X64_WIDEN-NEXT:    shrq $32, %rcx
-; X64_WIDEN-NEXT:    subl %ecx, %eax
-; X64_WIDEN-NEXT:    shrl %eax
-; X64_WIDEN-NEXT:    addl %ecx, %eax
-; X64_WIDEN-NEXT:    shrl $2, %eax
-; X64_WIDEN-NEXT:    movd %eax, %xmm0
-; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X64_WIDEN-NEXT:    movq %xmm1, (%rsi)
+; X64_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
+; X64_WIDEN-NEXT:    movdqa %xmm0, %xmm2
+; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
+; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
+; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X64_WIDEN-NEXT:    psubd %xmm2, %xmm0
+; X64_WIDEN-NEXT:    psrld $1, %xmm0
+; X64_WIDEN-NEXT:    paddd %xmm2, %xmm0
+; X64_WIDEN-NEXT:    psrld $2, %xmm0
+; X64_WIDEN-NEXT:    movq %xmm0, (%rsi)
 ; X64_WIDEN-NEXT:    retq
 ;
 ; X86_WIDEN-LABEL: test_udiv7_v2i32:
 ; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    pushl %edi
-; X86_WIDEN-NEXT:    pushl %esi
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    movd %xmm0, %ecx
-; X86_WIDEN-NEXT:    movl $613566757, %edi # imm = 0x24924925
-; X86_WIDEN-NEXT:    movl %ecx, %eax
-; X86_WIDEN-NEXT:    mull %edi
-; X86_WIDEN-NEXT:    subl %edx, %ecx
-; X86_WIDEN-NEXT:    shrl %ecx
-; X86_WIDEN-NEXT:    addl %edx, %ecx
-; X86_WIDEN-NEXT:    shrl $2, %ecx
-; X86_WIDEN-NEXT:    movd %ecx, %xmm1
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X86_WIDEN-NEXT:    movd %xmm0, %ecx
-; X86_WIDEN-NEXT:    movl %ecx, %eax
-; X86_WIDEN-NEXT:    mull %edi
-; X86_WIDEN-NEXT:    subl %edx, %ecx
-; X86_WIDEN-NEXT:    shrl %ecx
-; X86_WIDEN-NEXT:    addl %edx, %ecx
-; X86_WIDEN-NEXT:    shrl $2, %ecx
-; X86_WIDEN-NEXT:    movd %ecx, %xmm0
-; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86_WIDEN-NEXT:    movq %xmm1, (%esi)
-; X86_WIDEN-NEXT:    popl %esi
-; X86_WIDEN-NEXT:    popl %edi
+; X86_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
+; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm2
+; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
+; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm3
+; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
+; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
+; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86_WIDEN-NEXT:    psubd %xmm2, %xmm0
+; X86_WIDEN-NEXT:    psrld $1, %xmm0
+; X86_WIDEN-NEXT:    paddd %xmm2, %xmm0
+; X86_WIDEN-NEXT:    psrld $2, %xmm0
+; X86_WIDEN-NEXT:    movq %xmm0, (%eax)
 ; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = udiv <2 x i32> %a, <i32 7, i32 7>
   store <2 x i32> %b, <2 x i32>* %y
   ret void
 }
 
 define void @test_urem7_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {

 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
 ;
 ; X64_WIDEN-LABEL: test_urem7_v2i32:
 ; X64_WIDEN:       # %bb.0:
 ; X64_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    imulq $613566757, %rax, %rcx # imm = 0x24924925
-; X64_WIDEN-NEXT:    shrq $32, %rcx
-; X64_WIDEN-NEXT:    movl %eax, %edx
-; X64_WIDEN-NEXT:    subl %ecx, %edx
-; X64_WIDEN-NEXT:    shrl %edx
-; X64_WIDEN-NEXT:    addl %ecx, %edx
-; X64_WIDEN-NEXT:    shrl $2, %edx
-; X64_WIDEN-NEXT:    leal (,%rdx,8), %ecx
-; X64_WIDEN-NEXT:    subl %ecx, %edx
-; X64_WIDEN-NEXT:    addl %eax, %edx
-; X64_WIDEN-NEXT:    movd %edx, %xmm1
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    imulq $613566757, %rax, %rcx # imm = 0x24924925
-; X64_WIDEN-NEXT:    shrq $32, %rcx
-; X64_WIDEN-NEXT:    movl %eax, %edx
-; X64_WIDEN-NEXT:    subl %ecx, %edx
-; X64_WIDEN-NEXT:    shrl %edx
-; X64_WIDEN-NEXT:    addl %ecx, %edx
-; X64_WIDEN-NEXT:    shrl $2, %edx
-; X64_WIDEN-NEXT:    leal (,%rdx,8), %ecx
-; X64_WIDEN-NEXT:    subl %ecx, %edx
-; X64_WIDEN-NEXT:    addl %eax, %edx
-; X64_WIDEN-NEXT:    movd %edx, %xmm0
-; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X64_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
+; X64_WIDEN-NEXT:    movdqa %xmm0, %xmm2
+; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
+; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
+; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X64_WIDEN-NEXT:    movdqa %xmm0, %xmm1
+; X64_WIDEN-NEXT:    psubd %xmm2, %xmm1
+; X64_WIDEN-NEXT:    psrld $1, %xmm1
+; X64_WIDEN-NEXT:    paddd %xmm2, %xmm1
+; X64_WIDEN-NEXT:    psrld $2, %xmm1
+; X64_WIDEN-NEXT:    movdqa %xmm1, %xmm2
+; X64_WIDEN-NEXT:    pslld $3, %xmm2
+; X64_WIDEN-NEXT:    psubd %xmm2, %xmm1
+; X64_WIDEN-NEXT:    paddd %xmm0, %xmm1
 ; X64_WIDEN-NEXT:    movq %xmm1, (%rsi)
 ; X64_WIDEN-NEXT:    retq
 ;
 ; X86_WIDEN-LABEL: test_urem7_v2i32:
 ; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    pushl %edi
-; X86_WIDEN-NEXT:    pushl %esi
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    movd %xmm0, %ecx
-; X86_WIDEN-NEXT:    movl $613566757, %edi # imm = 0x24924925
-; X86_WIDEN-NEXT:    movl %ecx, %eax
-; X86_WIDEN-NEXT:    mull %edi
-; X86_WIDEN-NEXT:    movl %ecx, %eax
-; X86_WIDEN-NEXT:    subl %edx, %eax
-; X86_WIDEN-NEXT:    shrl %eax
-; X86_WIDEN-NEXT:    addl %edx, %eax
-; X86_WIDEN-NEXT:    shrl $2, %eax
-; X86_WIDEN-NEXT:    leal (,%eax,8), %edx
-; X86_WIDEN-NEXT:    subl %edx, %eax
-; X86_WIDEN-NEXT:    addl %ecx, %eax
-; X86_WIDEN-NEXT:    movd %eax, %xmm1
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X86_WIDEN-NEXT:    movd %xmm0, %ecx
-; X86_WIDEN-NEXT:    movl %ecx, %eax
-; X86_WIDEN-NEXT:    mull %edi
-; X86_WIDEN-NEXT:    movl %ecx, %eax
-; X86_WIDEN-NEXT:    subl %edx, %eax
-; X86_WIDEN-NEXT:    shrl %eax
-; X86_WIDEN-NEXT:    addl %edx, %eax
-; X86_WIDEN-NEXT:    shrl $2, %eax
-; X86_WIDEN-NEXT:    leal (,%eax,8), %edx
-; X86_WIDEN-NEXT:    subl %edx, %eax
-; X86_WIDEN-NEXT:    addl %ecx, %eax
-; X86_WIDEN-NEXT:    movd %eax, %xmm0
-; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86_WIDEN-NEXT:    movq %xmm1, (%esi)
-; X86_WIDEN-NEXT:    popl %esi
-; X86_WIDEN-NEXT:    popl %edi
+; X86_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [613566757,613566757,613566757,613566757]
+; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm2
+; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
+; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm3
+; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
+; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
+; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm1
+; X86_WIDEN-NEXT:    psubd %xmm2, %xmm1
+; X86_WIDEN-NEXT:    psrld $1, %xmm1
+; X86_WIDEN-NEXT:    paddd %xmm2, %xmm1
+; X86_WIDEN-NEXT:    psrld $2, %xmm1
+; X86_WIDEN-NEXT:    movdqa %xmm1, %xmm2
+; X86_WIDEN-NEXT:    pslld $3, %xmm2
+; X86_WIDEN-NEXT:    psubd %xmm2, %xmm1
+; X86_WIDEN-NEXT:    paddd %xmm0, %xmm1
+; X86_WIDEN-NEXT:    movq %xmm1, (%eax)
 ; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = urem <2 x i32> %a, <i32 7, i32 7>
   store <2 x i32> %b, <2 x i32>* %y
   ret void
 }
 
 define void @test_sdiv7_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {

 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
 ;
 ; X64_WIDEN-LABEL: test_sdiv7_v2i32:
 ; X64_WIDEN:       # %bb.0:
 ; X64_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    cltq
-; X64_WIDEN-NEXT:    imulq $-1840700269, %rax, %rcx # imm = 0x92492493
-; X64_WIDEN-NEXT:    shrq $32, %rcx
-; X64_WIDEN-NEXT:    addl %ecx, %eax
-; X64_WIDEN-NEXT:    movl %eax, %ecx
-; X64_WIDEN-NEXT:    shrl $31, %ecx
-; X64_WIDEN-NEXT:    sarl $2, %eax
-; X64_WIDEN-NEXT:    addl %ecx, %eax
-; X64_WIDEN-NEXT:    movd %eax, %xmm1
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    cltq
-; X64_WIDEN-NEXT:    imulq $-1840700269, %rax, %rcx # imm = 0x92492493
-; X64_WIDEN-NEXT:    shrq $32, %rcx
-; X64_WIDEN-NEXT:    addl %ecx, %eax
-; X64_WIDEN-NEXT:    movl %eax, %ecx
-; X64_WIDEN-NEXT:    shrl $31, %ecx
-; X64_WIDEN-NEXT:    sarl $2, %eax
-; X64_WIDEN-NEXT:    addl %ecx, %eax
-; X64_WIDEN-NEXT:    movd %eax, %xmm0
-; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X64_WIDEN-NEXT:    movq %xmm1, (%rsi)
+; X64_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,2454267027,2454267027,2454267027]
+; X64_WIDEN-NEXT:    movdqa %xmm0, %xmm2
+; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
+; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
+; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; X64_WIDEN-NEXT:    pxor %xmm3, %xmm3
+; X64_WIDEN-NEXT:    pcmpgtd %xmm0, %xmm3
+; X64_WIDEN-NEXT:    pand %xmm1, %xmm3
+; X64_WIDEN-NEXT:    paddd %xmm0, %xmm3
+; X64_WIDEN-NEXT:    psubd %xmm3, %xmm2
+; X64_WIDEN-NEXT:    paddd %xmm0, %xmm2
+; X64_WIDEN-NEXT:    movdqa %xmm2, %xmm0
+; X64_WIDEN-NEXT:    psrld $31, %xmm0
+; X64_WIDEN-NEXT:    psrad $2, %xmm2
+; X64_WIDEN-NEXT:    paddd %xmm0, %xmm2
+; X64_WIDEN-NEXT:    movq %xmm2, (%rsi)
 ; X64_WIDEN-NEXT:    retq
 ;
 ; X86_WIDEN-LABEL: test_sdiv7_v2i32:
 ; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    pushl %ebx
-; X86_WIDEN-NEXT:    pushl %edi
-; X86_WIDEN-NEXT:    pushl %esi
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    movd %xmm0, %ecx
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X86_WIDEN-NEXT:    movd %xmm0, %esi
-; X86_WIDEN-NEXT:    movl $-1840700269, %ebx # imm = 0x92492493
-; X86_WIDEN-NEXT:    movl %esi, %eax
-; X86_WIDEN-NEXT:    imull %ebx
-; X86_WIDEN-NEXT:    addl %esi, %edx
-; X86_WIDEN-NEXT:    movl %edx, %eax
-; X86_WIDEN-NEXT:    shrl $31, %eax
-; X86_WIDEN-NEXT:    sarl $2, %edx
-; X86_WIDEN-NEXT:    addl %eax, %edx
-; X86_WIDEN-NEXT:    movd %edx, %xmm0
-; X86_WIDEN-NEXT:    movl %ecx, %eax
-; X86_WIDEN-NEXT:    imull %ebx
-; X86_WIDEN-NEXT:    addl %ecx, %edx
-; X86_WIDEN-NEXT:    movl %edx, %eax
-; X86_WIDEN-NEXT:    shrl $31, %eax
-; X86_WIDEN-NEXT:    sarl $2, %edx
-; X86_WIDEN-NEXT:    addl %eax, %edx
-; X86_WIDEN-NEXT:    movd %edx, %xmm1
-; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86_WIDEN-NEXT:    movq %xmm1, (%edi)
-; X86_WIDEN-NEXT:    popl %esi
-; X86_WIDEN-NEXT:    popl %edi
-; X86_WIDEN-NEXT:    popl %ebx
+; X86_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,2454267027,2454267027,2454267027]
+; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm2
+; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
+; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm3
+; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
+; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
+; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; X86_WIDEN-NEXT:    pxor %xmm3, %xmm3
+; X86_WIDEN-NEXT:    pcmpgtd %xmm0, %xmm3
+; X86_WIDEN-NEXT:    pand %xmm1, %xmm3
+; X86_WIDEN-NEXT:    paddd %xmm0, %xmm3
+; X86_WIDEN-NEXT:    psubd %xmm3, %xmm2
+; X86_WIDEN-NEXT:    paddd %xmm0, %xmm2
+; X86_WIDEN-NEXT:    movdqa %xmm2, %xmm0
+; X86_WIDEN-NEXT:    psrld $31, %xmm0
+; X86_WIDEN-NEXT:    psrad $2, %xmm2
+; X86_WIDEN-NEXT:    paddd %xmm0, %xmm2
+; X86_WIDEN-NEXT:    movq %xmm2, (%eax)
 ; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = sdiv <2 x i32> %a, <i32 7, i32 7>
   store <2 x i32> %b, <2 x i32>* %y
   ret void
 }
 
 define void @test_srem7_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {

 ; X86-NEXT:    popl %edi
 ; X86-NEXT:    popl %ebx
 ; X86-NEXT:    popl %ebp
 ; X86-NEXT:    retl
 ;
 ; X64_WIDEN-LABEL: test_srem7_v2i32:
 ; X64_WIDEN:       # %bb.0:
 ; X64_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    cltq
-; X64_WIDEN-NEXT:    imulq $-1840700269, %rax, %rcx # imm = 0x92492493
-; X64_WIDEN-NEXT:    shrq $32, %rcx
-; X64_WIDEN-NEXT:    addl %eax, %ecx
-; X64_WIDEN-NEXT:    movl %ecx, %edx
-; X64_WIDEN-NEXT:    shrl $31, %edx
-; X64_WIDEN-NEXT:    sarl $2, %ecx
-; X64_WIDEN-NEXT:    addl %edx, %ecx
-; X64_WIDEN-NEXT:    leal (,%rcx,8), %edx
-; X64_WIDEN-NEXT:    subl %edx, %ecx
-; X64_WIDEN-NEXT:    addl %eax, %ecx
-; X64_WIDEN-NEXT:    movd %ecx, %xmm1
-; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X64_WIDEN-NEXT:    movd %xmm0, %eax
-; X64_WIDEN-NEXT:    cltq
-; X64_WIDEN-NEXT:    imulq $-1840700269, %rax, %rcx # imm = 0x92492493
-; X64_WIDEN-NEXT:    shrq $32, %rcx
-; X64_WIDEN-NEXT:    addl %eax, %ecx
-; X64_WIDEN-NEXT:    movl %ecx, %edx
-; X64_WIDEN-NEXT:    shrl $31, %edx
-; X64_WIDEN-NEXT:    sarl $2, %ecx
-; X64_WIDEN-NEXT:    addl %edx, %ecx
-; X64_WIDEN-NEXT:    leal (,%rcx,8), %edx
-; X64_WIDEN-NEXT:    subl %edx, %ecx
-; X64_WIDEN-NEXT:    addl %eax, %ecx
-; X64_WIDEN-NEXT:    movd %ecx, %xmm0
-; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X64_WIDEN-NEXT:    movq %xmm1, (%rsi)
+; X64_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,2454267027,2454267027,2454267027]
+; X64_WIDEN-NEXT:    movdqa %xmm0, %xmm2
+; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
+; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X64_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
+; X64_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; X64_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; X64_WIDEN-NEXT:    pxor %xmm3, %xmm3
+; X64_WIDEN-NEXT:    pcmpgtd %xmm0, %xmm3
+; X64_WIDEN-NEXT:    pand %xmm1, %xmm3
+; X64_WIDEN-NEXT:    paddd %xmm0, %xmm3
+; X64_WIDEN-NEXT:    psubd %xmm3, %xmm2
+; X64_WIDEN-NEXT:    paddd %xmm0, %xmm2
+; X64_WIDEN-NEXT:    movdqa %xmm2, %xmm1
+; X64_WIDEN-NEXT:    psrld $31, %xmm1
+; X64_WIDEN-NEXT:    psrad $2, %xmm2
+; X64_WIDEN-NEXT:    paddd %xmm1, %xmm2
+; X64_WIDEN-NEXT:    movdqa %xmm2, %xmm1
+; X64_WIDEN-NEXT:    pslld $3, %xmm1
+; X64_WIDEN-NEXT:    psubd %xmm1, %xmm2
+; X64_WIDEN-NEXT:    paddd %xmm0, %xmm2
+; X64_WIDEN-NEXT:    movq %xmm2, (%rsi)
 ; X64_WIDEN-NEXT:    retq
 ;
 ; X86_WIDEN-LABEL: test_srem7_v2i32:
 ; X86_WIDEN:       # %bb.0:
-; X86_WIDEN-NEXT:    pushl %ebx
-; X86_WIDEN-NEXT:    pushl %edi
-; X86_WIDEN-NEXT:    pushl %esi
 ; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86_WIDEN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X86_WIDEN-NEXT:    movd %xmm0, %ecx
-; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X86_WIDEN-NEXT:    movd %xmm0, %esi
-; X86_WIDEN-NEXT:    movl $-1840700269, %edi # imm = 0x92492493
-; X86_WIDEN-NEXT:    movl %esi, %eax
-; X86_WIDEN-NEXT:    imull %edi
-; X86_WIDEN-NEXT:    addl %esi, %edx
-; X86_WIDEN-NEXT:    movl %edx, %eax
-; X86_WIDEN-NEXT:    shrl $31, %eax
-; X86_WIDEN-NEXT:    sarl $2, %edx
-; X86_WIDEN-NEXT:    addl %eax, %edx
-; X86_WIDEN-NEXT:    leal (,%edx,8), %eax
-; X86_WIDEN-NEXT:    subl %eax, %edx
-; X86_WIDEN-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; X86_WIDEN-NEXT:    addl %esi, %edx
-; X86_WIDEN-NEXT:    movd %edx, %xmm0
-; X86_WIDEN-NEXT:    movl %ecx, %eax
-; X86_WIDEN-NEXT:    imull %edi
-; X86_WIDEN-NEXT:    addl %ecx, %edx
-; X86_WIDEN-NEXT:    movl %edx, %eax
-; X86_WIDEN-NEXT:    shrl $31, %eax
-; X86_WIDEN-NEXT:    sarl $2, %edx
-; X86_WIDEN-NEXT:    addl %eax, %edx
-; X86_WIDEN-NEXT:    leal (,%edx,8), %eax
-; X86_WIDEN-NEXT:    subl %eax, %edx
-; X86_WIDEN-NEXT:    addl %ecx, %edx
-; X86_WIDEN-NEXT:    movd %edx, %xmm1
-; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86_WIDEN-NEXT:    movq %xmm1, (%ebx)
-; X86_WIDEN-NEXT:    popl %esi
-; X86_WIDEN-NEXT:    popl %edi
-; X86_WIDEN-NEXT:    popl %ebx
+; X86_WIDEN-NEXT:    movdqa {{.*#+}} xmm1 = [2454267027,2454267027,2454267027,2454267027]
+; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm2
+; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm2
+; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; X86_WIDEN-NEXT:    movdqa %xmm0, %xmm3
+; X86_WIDEN-NEXT:    shufps {{.*#+}} xmm3 = xmm3[1,1],xmm0[2,3]
+; X86_WIDEN-NEXT:    pmuludq %xmm1, %xmm3
+; X86_WIDEN-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,3,2,3]
+; X86_WIDEN-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; X86_WIDEN-NEXT:    pxor %xmm3, %xmm3
+; X86_WIDEN-NEXT:    pcmpgtd %xmm0, %xmm3
+; X86_WIDEN-NEXT:    pand %xmm1, %xmm3
+; X86_WIDEN-NEXT:    paddd %xmm0, %xmm3
+; X86_WIDEN-NEXT:    psubd %xmm3, %xmm2
+; X86_WIDEN-NEXT:    paddd %xmm0, %xmm2
+; X86_WIDEN-NEXT:    movdqa %xmm2, %xmm1
+; X86_WIDEN-NEXT:    psrld $31, %xmm1
+; X86_WIDEN-NEXT:    psrad $2, %xmm2
+; X86_WIDEN-NEXT:    paddd %xmm1, %xmm2
+; X86_WIDEN-NEXT:    movdqa %xmm2, %xmm1
+; X86_WIDEN-NEXT:    pslld $3, %xmm1
+; X86_WIDEN-NEXT:    psubd %xmm1, %xmm2
+; X86_WIDEN-NEXT:    paddd %xmm0, %xmm2
+; X86_WIDEN-NEXT:    movq %xmm2, (%eax)
 ; X86_WIDEN-NEXT:    retl
   %a = load <2 x i32>, <2 x i32>* %x
   %b = srem <2 x i32> %a, <i32 7, i32 7>
   store <2 x i32> %b, <2 x i32>* %y
   ret void
 }
 
 define void @test_udiv_pow2_v2i32(<2 x i32>* %x, <2 x i32>* %y) nounwind {