[X86] Improve combineVectorShiftImm

[foad]

Summary

Fold (shift (shift X, C2), C1) -> (shift X, (C1 + C2)) for logical as
well as arithmetic shifts. This is needed to prevent regressions from
an upcoming funnel shift expansion change.

While we're here, fold (VSRAI -1, C) -> -1 too.

[RKSimon]

llvm/lib/Target/X86/X86ISelLowering.cpp

40880

ISD::isBuildVectorAll* leaves UNDEFs in there - we must generate the constant

[foad]

llvm/lib/Target/X86/X86ISelLowering.cpp

40880

I don't quite understand why. If there are undefs in there then we will effectively be folding (VSRAI undef, C) -> undef (for some lanes). Isn't that allowed?

[RKSimon]

llvm/lib/Target/X86/X86ISelLowering.cpp

40880

More clearly we're guaranteeing that for logical shifts that the upper bits will be zero. I'd recommend playing it safe as we've been hit by these things before (PR43159/PR43177) as SimplifyDemandedBits becomes more capable:

// Shift N0 by zero -> N0.
if (!ShiftVal)
  return N0;
// Shift zero -> zero.
if (ISD::isBuildVectorAllZeros(N0.getNode()))
  return DAG.getConstant(0, SDLoc(N), VT);
// (VSRAI -1, C) -> -1
if (!LogicalShift && ISD::isBuildVectorAllOnes(N0.getNode()))
  return DAG.getConstant(-1, SDLoc(N), VT);

[foad]

Address review comments about undef.

[foad]

Fix comments.

[RKSimon]

LGTM

[RKSimon]

reverse ping?

llvm/lib/Target/X86/X86ISelLowering.cpp

          "Unexpected shift amount type");
 
   // Out of range logical bit shifts are guaranteed to be zero.
   // Out of range arithmetic bit shifts splat the sign bit.
   unsigned ShiftVal = N->getConstantOperandVal(1);
   if (ShiftVal >= NumBitsPerElt) {
     if (LogicalShift)
       return DAG.getConstant(0, SDLoc(N), VT);
-    else
-      ShiftVal = NumBitsPerElt - 1;
+    ShiftVal = NumBitsPerElt - 1;
   }
 
-  // Shift N0 by zero -> N0.
+  // (shift X, 0) -> X
   if (!ShiftVal)
     return N0;
 
-  // Shift zero -> zero.
+  // (shift 0, C) -> 0
   if (ISD::isBuildVectorAllZeros(N0.getNode()))
+    // N0 is all zeros or undef. We guarantee that the upper bits of the result
+    // are all zeros, not undef.
     return DAG.getConstant(0, SDLoc(N), VT);
 
-  // Fold (VSRAI (VSRAI X, C1), C2) --> (VSRAI X, (C1 + C2)) with (C1 + C2)
-  // clamped to (NumBitsPerElt - 1).
-  if (Opcode == X86ISD::VSRAI && N0.getOpcode() == X86ISD::VSRAI) {
+  // (VSRAI -1, C) -> -1
+  if (!LogicalShift && ISD::isBuildVectorAllOnes(N0.getNode()))
+    // N0 is all ones or undef. We guarantee that the upper bits of the result
+    // are all ones, not undef.
+    return DAG.getConstant(-1, SDLoc(N), VT);
+
+  // (shift (shift X, C2), C1) -> (shift X, (C1 + C2))
+  if (Opcode == N0.getOpcode()) {
     unsigned ShiftVal2 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
     unsigned NewShiftVal = ShiftVal + ShiftVal2;
-    if (NewShiftVal >= NumBitsPerElt)
+    if (NewShiftVal >= NumBitsPerElt) {
+      // Out of range logical bit shifts are guaranteed to be zero.
+      // Out of range arithmetic bit shifts splat the sign bit.
+      if (LogicalShift)
+        return DAG.getConstant(0, SDLoc(N), VT);
       NewShiftVal = NumBitsPerElt - 1;
-    return DAG.getNode(X86ISD::VSRAI, SDLoc(N), VT, N0.getOperand(0),
+    }
+    return DAG.getNode(Opcode, SDLoc(N), VT, N0.getOperand(0),
                        DAG.getTargetConstant(NewShiftVal, SDLoc(N), MVT::i8));
   }
 
   // We can decode 'whole byte' logical bit shifts as shuffles.
   if (LogicalShift && (ShiftVal % 8) == 0) {
     SDValue Op(N, 0);
     if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
       return Res;

llvm/test/CodeGen/X86/midpoint-int-vec-128.ll

 ; SSE2-LABEL: vec128_i64_signed_reg_reg:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm5
 ; SSE2-NEXT:    pxor %xmm4, %xmm5
 ; SSE2-NEXT:    pxor %xmm0, %xmm4
 ; SSE2-NEXT:    movdqa %xmm4, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1,1]
 ; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [1,1]
-; SSE2-NEXT:    por %xmm2, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; SSE2-NEXT:    por %xmm4, %xmm5
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    pand %xmm5, %xmm4
 ; SSE2-NEXT:    pandn %xmm1, %xmm5
 ; SSE2-NEXT:    por %xmm4, %xmm5
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm4
-; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm4, %xmm2
-; SSE2-NEXT:    psubq %xmm5, %xmm2
-; SSE2-NEXT:    psrlq $1, %xmm2
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm1, %xmm3
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    psubq %xmm5, %xmm3
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    psrlq $32, %xmm4
-; SSE2-NEXT:    pmuludq %xmm2, %xmm4
+; SSE2-NEXT:    psrlq $1, %xmm4
+; SSE2-NEXT:    psrlq $33, %xmm3
+; SSE2-NEXT:    pmuludq %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    psrlq $32, %xmm1
-; SSE2-NEXT:    pmuludq %xmm3, %xmm1
-; SSE2-NEXT:    paddq %xmm4, %xmm1
+; SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; SSE2-NEXT:    paddq %xmm3, %xmm1
 ; SSE2-NEXT:    psllq $32, %xmm1
-; SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; SSE2-NEXT:    pmuludq %xmm2, %xmm4
 ; SSE2-NEXT:    paddq %xmm0, %xmm1
-; SSE2-NEXT:    paddq %xmm2, %xmm1
+; SSE2-NEXT:    paddq %xmm4, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: vec128_i64_signed_reg_reg:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm5
 ; SSE41-NEXT:    pxor %xmm0, %xmm5
 ; SSE41-NEXT:    pxor %xmm2, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    pcmpgtd %xmm5, %xmm3
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm6
 ; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
 ; SSE41-NEXT:    pand %xmm4, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm7, %xmm4
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [1,1]
-; SSE41-NEXT:    por %xmm4, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE41-NEXT:    por %xmm7, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [1,1]
+; SSE41-NEXT:    por %xmm3, %xmm4
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm1, %xmm5
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm5
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    psubq %xmm5, %xmm1
-; SSE41-NEXT:    psrlq $1, %xmm1
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    psrlq $1, %xmm3
+; SSE41-NEXT:    psrlq $33, %xmm1
+; SSE41-NEXT:    pmuludq %xmm4, %xmm1
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    psrlq $32, %xmm0
-; SSE41-NEXT:    pmuludq %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm1, %xmm4
-; SSE41-NEXT:    psrlq $32, %xmm4
+; SSE41-NEXT:    pmuludq %xmm3, %xmm0
+; SSE41-NEXT:    paddq %xmm1, %xmm0
+; SSE41-NEXT:    psllq $32, %xmm0
 ; SSE41-NEXT:    pmuludq %xmm3, %xmm4
-; SSE41-NEXT:    paddq %xmm0, %xmm4
-; SSE41-NEXT:    psllq $32, %xmm4
-; SSE41-NEXT:    pmuludq %xmm1, %xmm3
-; SSE41-NEXT:    paddq %xmm2, %xmm4
-; SSE41-NEXT:    paddq %xmm4, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    paddq %xmm2, %xmm0
+; SSE41-NEXT:    paddq %xmm4, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-FALLBACK-LABEL: vec128_i64_signed_reg_reg:
 ; AVX1-FALLBACK:       # %bb.0:
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpor {{.*}}(%rip), %xmm2, %xmm3
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm4
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm0, %xmm1, %xmm4
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vpsubq %xmm4, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX1-FALLBACK-NEXT:    retq
 ;
 ; AVX2-FALLBACK-LABEL: vec128_i64_signed_reg_reg:
 ; AVX2-FALLBACK:       # %bb.0:
 ; AVX2-FALLBACK-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX2-FALLBACK-NEXT:    vpor {{.*}}(%rip), %xmm2, %xmm3
 ; AVX2-FALLBACK-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm4
 ; AVX2-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm0, %xmm1, %xmm4
 ; AVX2-FALLBACK-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm1
 ; AVX2-FALLBACK-NEXT:    vpsubq %xmm4, %xmm1, %xmm1
-; AVX2-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX2-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; AVX2-FALLBACK-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; AVX2-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; AVX2-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; AVX2-FALLBACK-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX2-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX2-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX2-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX2-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX2-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX2-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX2-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX2-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX2-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX2-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX2-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX2-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX2-FALLBACK-NEXT:    retq
 ;
 ; XOP-LABEL: vec128_i64_signed_reg_reg:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpcomgtq %xmm1, %xmm0, %xmm2
 ; XOP-NEXT:    vpor {{.*}}(%rip), %xmm2, %xmm3
 ; XOP-NEXT:    vpcomltq %xmm1, %xmm0, %xmm4
 ; XOP-NEXT:    vblendvpd %xmm4, %xmm0, %xmm1, %xmm4
 ; XOP-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm1
 ; XOP-NEXT:    vpsubq %xmm4, %xmm1, %xmm1
-; XOP-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; XOP-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; XOP-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; XOP-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; XOP-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; XOP-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; XOP-NEXT:    vpsllq $32, %xmm2, %xmm2
+; XOP-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; XOP-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; XOP-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; XOP-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; XOP-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; XOP-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; XOP-NEXT:    vpsllq $32, %xmm1, %xmm1
+; XOP-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; XOP-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512F-LABEL: vec128_i64_signed_reg_reg:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
 ; AVX512F-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512F-NEXT:    vpminsq %zmm1, %zmm0, %zmm2
 ; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm1
 ; AVX512F-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsrlq $32, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; AVX512F-NEXT:    vpsrlq $32, %xmm3, %xmm4
-; AVX512F-NEXT:    vpmuludq %xmm4, %xmm1, %xmm4
-; AVX512F-NEXT:    vpaddq %xmm2, %xmm4, %xmm2
-; AVX512F-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX512F-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX512F-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX512F-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX512F-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX512F-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX512F-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX512F-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: vec128_i64_signed_reg_reg:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
 ; AVX512VL-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
 ; AVX512VL-NEXT:    vmovdqa64 %xmm2, %xmm3 {%k1}
 ; AVX512VL-NEXT:    vpminsq %xmm1, %xmm0, %xmm2
 ; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm1
 ; AVX512VL-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; AVX512VL-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; AVX512VL-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; AVX512VL-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX512VL-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX512VL-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX512VL-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX512VL-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX512VL-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX512VL-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX512VL-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-FALLBACK-LABEL: vec128_i64_signed_reg_reg:
 ; AVX512BW-FALLBACK:       # %bb.0:
 ; AVX512BW-FALLBACK-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-FALLBACK-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-FALLBACK-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512BW-FALLBACK-NEXT:    vpminsq %zmm1, %zmm0, %zmm2
 ; AVX512BW-FALLBACK-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm1
 ; AVX512BW-FALLBACK-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm2
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm1, %xmm4
-; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm2, %xmm4, %xmm2
-; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX512BW-FALLBACK-NEXT:    vzeroupper
 ; AVX512BW-FALLBACK-NEXT:    retq
   %t3 = icmp sgt <2 x i64> %a1, %a2 ; signed
   %t4 = select <2 x i1> %t3, <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 1, i64 1>
   %t5 = select <2 x i1> %t3, <2 x i64> %a2, <2 x i64> %a1
   %t6 = select <2 x i1> %t3, <2 x i64> %a1, <2 x i64> %a2
   %t7 = sub <2 x i64> %t6, %t5
   %t8 = lshr <2 x i64> %t7, <i64 1, i64 1>
   %t9 = mul nsw <2 x i64> %t8, %t4 ; signed
   %a10 = add nsw <2 x i64> %t9, %a1 ; signed
   ret <2 x i64> %a10
 }
 
 define <2 x i64> @vec128_i64_unsigned_reg_reg(<2 x i64> %a1, <2 x i64> %a2) nounwind {
 ; SSE2-LABEL: vec128_i64_unsigned_reg_reg:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm5
 ; SSE2-NEXT:    pxor %xmm4, %xmm5
 ; SSE2-NEXT:    pxor %xmm0, %xmm4
 ; SSE2-NEXT:    movdqa %xmm4, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1,1]
 ; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [1,1]
-; SSE2-NEXT:    por %xmm2, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; SSE2-NEXT:    por %xmm4, %xmm5
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    pand %xmm5, %xmm4
 ; SSE2-NEXT:    pandn %xmm1, %xmm5
 ; SSE2-NEXT:    por %xmm4, %xmm5
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm4
-; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm4, %xmm2
-; SSE2-NEXT:    psubq %xmm5, %xmm2
-; SSE2-NEXT:    psrlq $1, %xmm2
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm1, %xmm3
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    psubq %xmm5, %xmm3
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    psrlq $32, %xmm4
-; SSE2-NEXT:    pmuludq %xmm2, %xmm4
+; SSE2-NEXT:    psrlq $1, %xmm4
+; SSE2-NEXT:    psrlq $33, %xmm3
+; SSE2-NEXT:    pmuludq %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    psrlq $32, %xmm1
-; SSE2-NEXT:    pmuludq %xmm3, %xmm1
-; SSE2-NEXT:    paddq %xmm4, %xmm1
+; SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; SSE2-NEXT:    paddq %xmm3, %xmm1
 ; SSE2-NEXT:    psllq $32, %xmm1
-; SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; SSE2-NEXT:    pmuludq %xmm2, %xmm4
 ; SSE2-NEXT:    paddq %xmm0, %xmm1
-; SSE2-NEXT:    paddq %xmm2, %xmm1
+; SSE2-NEXT:    paddq %xmm4, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: vec128_i64_unsigned_reg_reg:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
 ; SSE41-NEXT:    movdqa %xmm1, %xmm5
 ; SSE41-NEXT:    pxor %xmm0, %xmm5
 ; SSE41-NEXT:    pxor %xmm2, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    pcmpgtd %xmm5, %xmm3
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm6
 ; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
 ; SSE41-NEXT:    pand %xmm4, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm7, %xmm4
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [1,1]
-; SSE41-NEXT:    por %xmm4, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE41-NEXT:    por %xmm7, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [1,1]
+; SSE41-NEXT:    por %xmm3, %xmm4
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm1, %xmm5
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm5
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
 ; SSE41-NEXT:    psubq %xmm5, %xmm1
-; SSE41-NEXT:    psrlq $1, %xmm1
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    psrlq $1, %xmm3
+; SSE41-NEXT:    psrlq $33, %xmm1
+; SSE41-NEXT:    pmuludq %xmm4, %xmm1
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    psrlq $32, %xmm0
-; SSE41-NEXT:    pmuludq %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm1, %xmm4
-; SSE41-NEXT:    psrlq $32, %xmm4
+; SSE41-NEXT:    pmuludq %xmm3, %xmm0
+; SSE41-NEXT:    paddq %xmm1, %xmm0
+; SSE41-NEXT:    psllq $32, %xmm0
 ; SSE41-NEXT:    pmuludq %xmm3, %xmm4
-; SSE41-NEXT:    paddq %xmm0, %xmm4
-; SSE41-NEXT:    psllq $32, %xmm4
-; SSE41-NEXT:    pmuludq %xmm1, %xmm3
-; SSE41-NEXT:    paddq %xmm2, %xmm4
-; SSE41-NEXT:    paddq %xmm4, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    paddq %xmm2, %xmm0
+; SSE41-NEXT:    paddq %xmm4, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-FALLBACK-LABEL: vec128_i64_unsigned_reg_reg:
 ; AVX1-FALLBACK:       # %bb.0:
 ; AVX1-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
 ; AVX1-FALLBACK-NEXT:    vpxor %xmm2, %xmm1, %xmm3
 ; AVX1-FALLBACK-NEXT:    vpxor %xmm2, %xmm0, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm4
 ; AVX1-FALLBACK-NEXT:    vpor {{.*}}(%rip), %xmm4, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm2
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm0, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm2
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm3
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm3, %xmm3
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm3
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm3
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm1, %xmm3, %xmm1
+; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX1-FALLBACK-NEXT:    retq
 ;
 ; AVX2-FALLBACK-LABEL: vec128_i64_unsigned_reg_reg:
 ; AVX2-FALLBACK:       # %bb.0:
 ; AVX2-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
 ; AVX2-FALLBACK-NEXT:    vpxor %xmm2, %xmm1, %xmm3
 ; AVX2-FALLBACK-NEXT:    vpxor %xmm2, %xmm0, %xmm2
 ; AVX2-FALLBACK-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm4
 ; AVX2-FALLBACK-NEXT:    vpor {{.*}}(%rip), %xmm4, %xmm5
 ; AVX2-FALLBACK-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
 ; AVX2-FALLBACK-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm2
 ; AVX2-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm0, %xmm1, %xmm1
 ; AVX2-FALLBACK-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
-; AVX2-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX2-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm2
-; AVX2-FALLBACK-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; AVX2-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm3
-; AVX2-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm3, %xmm3
-; AVX2-FALLBACK-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; AVX2-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX2-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX2-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX2-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; AVX2-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX2-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm3
+; AVX2-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm3
+; AVX2-FALLBACK-NEXT:    vpaddq %xmm1, %xmm3, %xmm1
+; AVX2-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX2-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
 ; AVX2-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX2-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX2-FALLBACK-NEXT:    retq
 ;
 ; XOP-LABEL: vec128_i64_unsigned_reg_reg:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpcomgtuq %xmm1, %xmm0, %xmm2
 ; XOP-NEXT:    vpor {{.*}}(%rip), %xmm2, %xmm3
 ; XOP-NEXT:    vpcomltuq %xmm1, %xmm0, %xmm4
 ; XOP-NEXT:    vblendvpd %xmm4, %xmm0, %xmm1, %xmm4
 ; XOP-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm1
 ; XOP-NEXT:    vpsubq %xmm4, %xmm1, %xmm1
-; XOP-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; XOP-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; XOP-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; XOP-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; XOP-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; XOP-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; XOP-NEXT:    vpsllq $32, %xmm2, %xmm2
+; XOP-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; XOP-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; XOP-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; XOP-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; XOP-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; XOP-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; XOP-NEXT:    vpsllq $32, %xmm1, %xmm1
+; XOP-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; XOP-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512F-LABEL: vec128_i64_unsigned_reg_reg:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vpcmpnleuq %zmm1, %zmm0, %k1
 ; AVX512F-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512F-NEXT:    vpminuq %zmm1, %zmm0, %zmm2
 ; AVX512F-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm1
 ; AVX512F-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsrlq $32, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; AVX512F-NEXT:    vpsrlq $32, %xmm3, %xmm4
-; AVX512F-NEXT:    vpmuludq %xmm4, %xmm1, %xmm4
-; AVX512F-NEXT:    vpaddq %xmm2, %xmm4, %xmm2
-; AVX512F-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX512F-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX512F-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX512F-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX512F-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX512F-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX512F-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX512F-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: vec128_i64_unsigned_reg_reg:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpcmpnleuq %xmm1, %xmm0, %k1
 ; AVX512VL-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
 ; AVX512VL-NEXT:    vmovdqa64 %xmm2, %xmm3 {%k1}
 ; AVX512VL-NEXT:    vpminuq %xmm1, %xmm0, %xmm2
 ; AVX512VL-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm1
 ; AVX512VL-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; AVX512VL-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; AVX512VL-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; AVX512VL-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX512VL-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX512VL-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX512VL-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX512VL-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX512VL-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX512VL-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX512VL-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-FALLBACK-LABEL: vec128_i64_unsigned_reg_reg:
 ; AVX512BW-FALLBACK:       # %bb.0:
 ; AVX512BW-FALLBACK-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512BW-FALLBACK-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-FALLBACK-NEXT:    vpcmpnleuq %zmm1, %zmm0, %k1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512BW-FALLBACK-NEXT:    vpminuq %zmm1, %zmm0, %zmm2
 ; AVX512BW-FALLBACK-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm1
 ; AVX512BW-FALLBACK-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm2
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm1, %xmm4
-; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm2, %xmm4, %xmm2
-; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX512BW-FALLBACK-NEXT:    vzeroupper
 ; AVX512BW-FALLBACK-NEXT:    retq
   %t3 = icmp ugt <2 x i64> %a1, %a2
   %t4 = select <2 x i1> %t3, <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 1, i64 1>
   %t5 = select <2 x i1> %t3, <2 x i64> %a2, <2 x i64> %a1
   %t6 = select <2 x i1> %t3, <2 x i64> %a1, <2 x i64> %a2
   %t7 = sub <2 x i64> %t6, %t5
   %t8 = lshr <2 x i64> %t7, <i64 1, i64 1>

 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa (%rdi), %xmm1
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    pxor %xmm4, %xmm5
 ; SSE2-NEXT:    pxor %xmm1, %xmm4
 ; SSE2-NEXT:    movdqa %xmm4, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1,1]
 ; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [1,1]
-; SSE2-NEXT:    por %xmm2, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; SSE2-NEXT:    por %xmm4, %xmm5
 ; SSE2-NEXT:    movdqa %xmm1, %xmm4
 ; SSE2-NEXT:    pand %xmm5, %xmm4
 ; SSE2-NEXT:    pandn %xmm0, %xmm5
 ; SSE2-NEXT:    por %xmm4, %xmm5
 ; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm4
-; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    por %xmm4, %xmm2
-; SSE2-NEXT:    psubq %xmm5, %xmm2
-; SSE2-NEXT:    psrlq $1, %xmm2
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    psubq %xmm5, %xmm3
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    psrlq $32, %xmm4
-; SSE2-NEXT:    pmuludq %xmm2, %xmm4
+; SSE2-NEXT:    psrlq $1, %xmm4
+; SSE2-NEXT:    psrlq $33, %xmm3
+; SSE2-NEXT:    pmuludq %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    psrlq $32, %xmm0
-; SSE2-NEXT:    pmuludq %xmm3, %xmm0
-; SSE2-NEXT:    paddq %xmm4, %xmm0
+; SSE2-NEXT:    pmuludq %xmm4, %xmm0
+; SSE2-NEXT:    paddq %xmm3, %xmm0
 ; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; SSE2-NEXT:    pmuludq %xmm2, %xmm4
 ; SSE2-NEXT:    paddq %xmm1, %xmm0
-; SSE2-NEXT:    paddq %xmm2, %xmm0
+; SSE2-NEXT:    paddq %xmm4, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: vec128_i64_signed_mem_reg:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa (%rdi), %xmm3
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]
 ; SSE41-NEXT:    pxor %xmm5, %xmm0

 ; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm5
 ; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm1, %xmm5
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm5
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
 ; SSE41-NEXT:    psubq %xmm5, %xmm1
-; SSE41-NEXT:    psrlq $1, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlq $1, %xmm0
+; SSE41-NEXT:    psrlq $33, %xmm1
+; SSE41-NEXT:    pmuludq %xmm4, %xmm1
 ; SSE41-NEXT:    movdqa %xmm4, %xmm2
 ; SSE41-NEXT:    psrlq $32, %xmm2
-; SSE41-NEXT:    pmuludq %xmm1, %xmm2
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrlq $32, %xmm0
+; SSE41-NEXT:    pmuludq %xmm0, %xmm2
+; SSE41-NEXT:    paddq %xmm1, %xmm2
+; SSE41-NEXT:    psllq $32, %xmm2
 ; SSE41-NEXT:    pmuludq %xmm4, %xmm0
+; SSE41-NEXT:    paddq %xmm3, %xmm2
 ; SSE41-NEXT:    paddq %xmm2, %xmm0
-; SSE41-NEXT:    psllq $32, %xmm0
-; SSE41-NEXT:    pmuludq %xmm4, %xmm1
-; SSE41-NEXT:    paddq %xmm3, %xmm0
-; SSE41-NEXT:    paddq %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-FALLBACK-LABEL: vec128_i64_signed_mem_reg:
 ; AVX1-FALLBACK:       # %bb.0:
 ; AVX1-FALLBACK-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpor {{.*}}(%rip), %xmm2, %xmm3
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm4
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm1, %xmm0, %xmm4
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX1-FALLBACK-NEXT:    vpsubq %xmm4, %xmm0, %xmm0
-; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm0, %xmm0
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm0, %xmm4
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm0, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsrlq $33, %xmm0, %xmm0
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm0, %xmm0
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
+; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
+; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX1-FALLBACK-NEXT:    retq
 ;
 ; AVX2-FALLBACK-LABEL: vec128_i64_signed_mem_reg:
 ; AVX2-FALLBACK:       # %bb.0:
 ; AVX2-FALLBACK-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX2-FALLBACK-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm2
 ; AVX2-FALLBACK-NEXT:    vpor {{.*}}(%rip), %xmm2, %xmm3
 ; AVX2-FALLBACK-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm4
 ; AVX2-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm1, %xmm0, %xmm4
 ; AVX2-FALLBACK-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
 ; AVX2-FALLBACK-NEXT:    vpsubq %xmm4, %xmm0, %xmm0
-; AVX2-FALLBACK-NEXT:    vpsrlq $1, %xmm0, %xmm0
-; AVX2-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; AVX2-FALLBACK-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
-; AVX2-FALLBACK-NEXT:    vpsrlq $32, %xmm0, %xmm4
-; AVX2-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; AVX2-FALLBACK-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX2-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX2-FALLBACK-NEXT:    vpsrlq $1, %xmm0, %xmm2
+; AVX2-FALLBACK-NEXT:    vpsrlq $33, %xmm0, %xmm0
 ; AVX2-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm0, %xmm0
-; AVX2-FALLBACK-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
+; AVX2-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX2-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX2-FALLBACK-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
+; AVX2-FALLBACK-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX2-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX2-FALLBACK-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX2-FALLBACK-NEXT:    retq
 ;
 ; XOP-LABEL: vec128_i64_signed_mem_reg:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vmovdqa (%rdi), %xmm1
 ; XOP-NEXT:    vpcomgtq %xmm0, %xmm1, %xmm2
 ; XOP-NEXT:    vpor {{.*}}(%rip), %xmm2, %xmm3
 ; XOP-NEXT:    vpcomltq %xmm0, %xmm1, %xmm4
 ; XOP-NEXT:    vblendvpd %xmm4, %xmm1, %xmm0, %xmm4
 ; XOP-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
 ; XOP-NEXT:    vpsubq %xmm4, %xmm0, %xmm0
-; XOP-NEXT:    vpsrlq $1, %xmm0, %xmm0
-; XOP-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; XOP-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
-; XOP-NEXT:    vpsrlq $32, %xmm0, %xmm4
-; XOP-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; XOP-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; XOP-NEXT:    vpsllq $32, %xmm2, %xmm2
+; XOP-NEXT:    vpsrlq $1, %xmm0, %xmm2
+; XOP-NEXT:    vpsrlq $33, %xmm0, %xmm0
 ; XOP-NEXT:    vpmuludq %xmm3, %xmm0, %xmm0
-; XOP-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; XOP-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; XOP-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
+; XOP-NEXT:    vpsllq $32, %xmm0, %xmm0
+; XOP-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; XOP-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512F-LABEL: vec128_i64_signed_mem_reg:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512F-NEXT:    vpcmpgtq %zmm0, %zmm1, %k1
 ; AVX512F-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512F-NEXT:    vpminsq %zmm0, %zmm1, %zmm2
 ; AVX512F-NEXT:    vpmaxsq %zmm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsrlq $1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsrlq $32, %xmm0, %xmm2
-; AVX512F-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; AVX512F-NEXT:    vpsrlq $32, %xmm3, %xmm4
-; AVX512F-NEXT:    vpmuludq %xmm4, %xmm0, %xmm4
-; AVX512F-NEXT:    vpaddq %xmm2, %xmm4, %xmm2
-; AVX512F-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX512F-NEXT:    vpsrlq $1, %xmm0, %xmm2
+; AVX512F-NEXT:    vpsrlq $33, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpmuludq %xmm3, %xmm0, %xmm0
-; AVX512F-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
+; AVX512F-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX512F-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX512F-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
+; AVX512F-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX512F-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: vec128_i64_signed_mem_reg:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512VL-NEXT:    vpcmpgtq %xmm0, %xmm1, %k1
 ; AVX512VL-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
 ; AVX512VL-NEXT:    vmovdqa64 %xmm2, %xmm3 {%k1}
 ; AVX512VL-NEXT:    vpminsq %xmm0, %xmm1, %xmm2
 ; AVX512VL-NEXT:    vpmaxsq %xmm0, %xmm1, %xmm0
 ; AVX512VL-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrlq $1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; AVX512VL-NEXT:    vpmuludq %xmm2, %xmm0, %xmm2
-; AVX512VL-NEXT:    vpsrlq $32, %xmm0, %xmm4
-; AVX512VL-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpsrlq $1, %xmm0, %xmm2
+; AVX512VL-NEXT:    vpsrlq $33, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
+; AVX512VL-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX512VL-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX512VL-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
+; AVX512VL-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-FALLBACK-LABEL: vec128_i64_signed_mem_reg:
 ; AVX512BW-FALLBACK:       # %bb.0:
 ; AVX512BW-FALLBACK-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpgtq %zmm0, %zmm1, %k1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512BW-FALLBACK-NEXT:    vpminsq %zmm0, %zmm1, %zmm2
 ; AVX512BW-FALLBACK-NEXT:    vpmaxsq %zmm0, %zmm1, %zmm0
 ; AVX512BW-FALLBACK-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %xmm0, %xmm0
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %xmm0, %xmm2
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm0, %xmm4
-; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm2, %xmm4, %xmm2
-; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %xmm0, %xmm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $33, %xmm0, %xmm0
 ; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm0, %xmm0
-; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm1, %xmm2, %xmm1
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
+; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX512BW-FALLBACK-NEXT:    vzeroupper
 ; AVX512BW-FALLBACK-NEXT:    retq
   %a1 = load <2 x i64>, <2 x i64>* %a1_addr
   %t3 = icmp sgt <2 x i64> %a1, %a2 ; signed
   %t4 = select <2 x i1> %t3, <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 1, i64 1>
   %t5 = select <2 x i1> %t3, <2 x i64> %a2, <2 x i64> %a1
   %t6 = select <2 x i1> %t3, <2 x i64> %a1, <2 x i64> %a2
   %t7 = sub <2 x i64> %t6, %t5
   %t8 = lshr <2 x i64> %t7, <i64 1, i64 1>
   %t9 = mul nsw <2 x i64> %t8, %t4 ; signed
   %a10 = add nsw <2 x i64> %t9, %a1 ; signed
   ret <2 x i64> %a10
 }
 
 define <2 x i64> @vec128_i64_signed_reg_mem(<2 x i64> %a1, <2 x i64>* %a2_addr) nounwind {
 ; SSE2-LABEL: vec128_i64_signed_reg_mem:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa (%rdi), %xmm1
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    pxor %xmm4, %xmm5
 ; SSE2-NEXT:    pxor %xmm1, %xmm4
 ; SSE2-NEXT:    movdqa %xmm5, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm4, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1,1]
 ; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [1,1]
-; SSE2-NEXT:    por %xmm2, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pand %xmm6, %xmm5
+; SSE2-NEXT:    pand %xmm7, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    por %xmm5, %xmm4
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    pand %xmm4, %xmm5
 ; SSE2-NEXT:    pandn %xmm1, %xmm4
 ; SSE2-NEXT:    por %xmm5, %xmm4
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    pand %xmm2, %xmm5
-; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm5, %xmm2
-; SSE2-NEXT:    psubq %xmm4, %xmm2
-; SSE2-NEXT:    psrlq $1, %xmm2
+; SSE2-NEXT:    pand %xmm3, %xmm5
+; SSE2-NEXT:    pandn %xmm1, %xmm3
+; SSE2-NEXT:    por %xmm5, %xmm3
+; SSE2-NEXT:    psubq %xmm4, %xmm3
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    psrlq $32, %xmm4
-; SSE2-NEXT:    pmuludq %xmm2, %xmm4
+; SSE2-NEXT:    psrlq $1, %xmm4
+; SSE2-NEXT:    psrlq $33, %xmm3
+; SSE2-NEXT:    pmuludq %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    psrlq $32, %xmm1
-; SSE2-NEXT:    pmuludq %xmm3, %xmm1
-; SSE2-NEXT:    paddq %xmm4, %xmm1
+; SSE2-NEXT:    pmuludq %xmm4, %xmm1
+; SSE2-NEXT:    paddq %xmm3, %xmm1
 ; SSE2-NEXT:    psllq $32, %xmm1
-; SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; SSE2-NEXT:    pmuludq %xmm2, %xmm4
 ; SSE2-NEXT:    paddq %xmm0, %xmm1
-; SSE2-NEXT:    paddq %xmm2, %xmm1
+; SSE2-NEXT:    paddq %xmm4, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: vec128_i64_signed_reg_mem:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    movdqa (%rdi), %xmm3
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]

 ; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm5, %xmm0
 ; SSE41-NEXT:    movdqa %xmm3, %xmm5
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
 ; SSE41-NEXT:    psubq %xmm5, %xmm3
-; SSE41-NEXT:    psrlq $1, %xmm3
-; SSE41-NEXT:    movdqa %xmm4, %xmm2
-; SSE41-NEXT:    psrlq $32, %xmm2
-; SSE41-NEXT:    pmuludq %xmm3, %xmm2
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm2
+; SSE41-NEXT:    psrlq $1, %xmm2
+; SSE41-NEXT:    psrlq $33, %xmm3
+; SSE41-NEXT:    pmuludq %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    psrlq $32, %xmm0
-; SSE41-NEXT:    pmuludq %xmm4, %xmm0
-; SSE41-NEXT:    paddq %xmm2, %xmm0
+; SSE41-NEXT:    pmuludq %xmm2, %xmm0
+; SSE41-NEXT:    paddq %xmm3, %xmm0
 ; SSE41-NEXT:    psllq $32, %xmm0
-; SSE41-NEXT:    pmuludq %xmm4, %xmm3
+; SSE41-NEXT:    pmuludq %xmm4, %xmm2
 ; SSE41-NEXT:    paddq %xmm1, %xmm0
-; SSE41-NEXT:    paddq %xmm3, %xmm0
+; SSE41-NEXT:    paddq %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-FALLBACK-LABEL: vec128_i64_signed_reg_mem:
 ; AVX1-FALLBACK:       # %bb.0:
 ; AVX1-FALLBACK-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpor {{.*}}(%rip), %xmm2, %xmm3
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm4
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm0, %xmm1, %xmm4
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vpsubq %xmm4, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX1-FALLBACK-NEXT:    retq
 ;
 ; AVX2-FALLBACK-LABEL: vec128_i64_signed_reg_mem:
 ; AVX2-FALLBACK:       # %bb.0:
 ; AVX2-FALLBACK-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX2-FALLBACK-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX2-FALLBACK-NEXT:    vpor {{.*}}(%rip), %xmm2, %xmm3
 ; AVX2-FALLBACK-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm4
 ; AVX2-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm0, %xmm1, %xmm4
 ; AVX2-FALLBACK-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm1
 ; AVX2-FALLBACK-NEXT:    vpsubq %xmm4, %xmm1, %xmm1
-; AVX2-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX2-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; AVX2-FALLBACK-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; AVX2-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; AVX2-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; AVX2-FALLBACK-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX2-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX2-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX2-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX2-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX2-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX2-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX2-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX2-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX2-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX2-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX2-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX2-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX2-FALLBACK-NEXT:    retq
 ;
 ; XOP-LABEL: vec128_i64_signed_reg_mem:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vmovdqa (%rdi), %xmm1
 ; XOP-NEXT:    vpcomgtq %xmm1, %xmm0, %xmm2
 ; XOP-NEXT:    vpor {{.*}}(%rip), %xmm2, %xmm3
 ; XOP-NEXT:    vpcomltq %xmm1, %xmm0, %xmm4
 ; XOP-NEXT:    vblendvpd %xmm4, %xmm0, %xmm1, %xmm4
 ; XOP-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm1
 ; XOP-NEXT:    vpsubq %xmm4, %xmm1, %xmm1
-; XOP-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; XOP-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; XOP-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; XOP-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; XOP-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; XOP-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; XOP-NEXT:    vpsllq $32, %xmm2, %xmm2
+; XOP-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; XOP-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; XOP-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; XOP-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; XOP-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; XOP-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; XOP-NEXT:    vpsllq $32, %xmm1, %xmm1
+; XOP-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; XOP-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512F-LABEL: vec128_i64_signed_reg_mem:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512F-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512F-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
 ; AVX512F-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512F-NEXT:    vpminsq %zmm1, %zmm0, %zmm2
 ; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm1
 ; AVX512F-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsrlq $32, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; AVX512F-NEXT:    vpsrlq $32, %xmm3, %xmm4
-; AVX512F-NEXT:    vpmuludq %xmm4, %xmm1, %xmm4
-; AVX512F-NEXT:    vpaddq %xmm2, %xmm4, %xmm2
-; AVX512F-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX512F-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX512F-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX512F-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX512F-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX512F-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX512F-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX512F-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: vec128_i64_signed_reg_mem:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512VL-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
 ; AVX512VL-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
 ; AVX512VL-NEXT:    vmovdqa64 %xmm2, %xmm3 {%k1}
 ; AVX512VL-NEXT:    vpminsq %xmm1, %xmm0, %xmm2
 ; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm1
 ; AVX512VL-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; AVX512VL-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; AVX512VL-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; AVX512VL-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX512VL-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX512VL-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX512VL-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX512VL-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX512VL-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX512VL-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX512VL-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-FALLBACK-LABEL: vec128_i64_signed_reg_mem:
 ; AVX512BW-FALLBACK:       # %bb.0:
 ; AVX512BW-FALLBACK-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512BW-FALLBACK-NEXT:    vpminsq %zmm1, %zmm0, %zmm2
 ; AVX512BW-FALLBACK-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm1
 ; AVX512BW-FALLBACK-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm2
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm1, %xmm4
-; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm2, %xmm4, %xmm2
-; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX512BW-FALLBACK-NEXT:    vzeroupper
 ; AVX512BW-FALLBACK-NEXT:    retq
   %a2 = load <2 x i64>, <2 x i64>* %a2_addr
   %t3 = icmp sgt <2 x i64> %a1, %a2 ; signed
   %t4 = select <2 x i1> %t3, <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 1, i64 1>
   %t5 = select <2 x i1> %t3, <2 x i64> %a2, <2 x i64> %a1
   %t6 = select <2 x i1> %t3, <2 x i64> %a1, <2 x i64> %a2
   %t7 = sub <2 x i64> %t6, %t5

 ; SSE2-NEXT:    movdqa (%rdi), %xmm1
 ; SSE2-NEXT:    movdqa (%rsi), %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    pxor %xmm4, %xmm5
 ; SSE2-NEXT:    pxor %xmm1, %xmm4
 ; SSE2-NEXT:    movdqa %xmm4, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1,1]
 ; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [1,1]
-; SSE2-NEXT:    por %xmm2, %xmm3
 ; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; SSE2-NEXT:    por %xmm4, %xmm5
 ; SSE2-NEXT:    movdqa %xmm1, %xmm4
 ; SSE2-NEXT:    pand %xmm5, %xmm4
 ; SSE2-NEXT:    pandn %xmm0, %xmm5
 ; SSE2-NEXT:    por %xmm4, %xmm5
 ; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm4
-; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    por %xmm4, %xmm2
-; SSE2-NEXT:    psubq %xmm5, %xmm2
-; SSE2-NEXT:    psrlq $1, %xmm2
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    psubq %xmm5, %xmm3
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    psrlq $32, %xmm4
-; SSE2-NEXT:    pmuludq %xmm2, %xmm4
+; SSE2-NEXT:    psrlq $1, %xmm4
+; SSE2-NEXT:    psrlq $33, %xmm3
+; SSE2-NEXT:    pmuludq %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    psrlq $32, %xmm0
-; SSE2-NEXT:    pmuludq %xmm3, %xmm0
-; SSE2-NEXT:    paddq %xmm4, %xmm0
+; SSE2-NEXT:    pmuludq %xmm4, %xmm0
+; SSE2-NEXT:    paddq %xmm3, %xmm0
 ; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; SSE2-NEXT:    pmuludq %xmm2, %xmm4
 ; SSE2-NEXT:    paddq %xmm1, %xmm0
-; SSE2-NEXT:    paddq %xmm2, %xmm0
+; SSE2-NEXT:    paddq %xmm4, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: vec128_i64_signed_mem_mem:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa (%rdi), %xmm3
-; SSE41-NEXT:    movdqa (%rsi), %xmm2
+; SSE41-NEXT:    movdqa (%rdi), %xmm2
+; SSE41-NEXT:    movdqa (%rsi), %xmm3
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    movdqa %xmm3, %xmm5
 ; SSE41-NEXT:    pxor %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
 ; SSE41-NEXT:    pcmpgtd %xmm5, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
 ; SSE41-NEXT:    movdqa %xmm0, %xmm6
 ; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
 ; SSE41-NEXT:    pand %xmm4, %xmm7
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE41-NEXT:    por %xmm7, %xmm1
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [1,1]
 ; SSE41-NEXT:    por %xmm1, %xmm4
 ; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm6, %xmm0
 ; SSE41-NEXT:    por %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm2, %xmm5
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm5
+; SSE41-NEXT:    movdqa %xmm3, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm5
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
-; SSE41-NEXT:    psubq %xmm5, %xmm2
-; SSE41-NEXT:    psrlq $1, %xmm2
-; SSE41-NEXT:    movdqa %xmm4, %xmm1
-; SSE41-NEXT:    psrlq $32, %xmm1
-; SSE41-NEXT:    pmuludq %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE41-NEXT:    psubq %xmm5, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm1
+; SSE41-NEXT:    psrlq $1, %xmm1
+; SSE41-NEXT:    psrlq $33, %xmm3
+; SSE41-NEXT:    pmuludq %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    psrlq $32, %xmm0
-; SSE41-NEXT:    pmuludq %xmm4, %xmm0
-; SSE41-NEXT:    paddq %xmm1, %xmm0
-; SSE41-NEXT:    psllq $32, %xmm0
-; SSE41-NEXT:    pmuludq %xmm4, %xmm2
+; SSE41-NEXT:    pmuludq %xmm1, %xmm0
 ; SSE41-NEXT:    paddq %xmm3, %xmm0
+; SSE41-NEXT:    psllq $32, %xmm0
+; SSE41-NEXT:    pmuludq %xmm4, %xmm1
 ; SSE41-NEXT:    paddq %xmm2, %xmm0
+; SSE41-NEXT:    paddq %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-FALLBACK-LABEL: vec128_i64_signed_mem_mem:
 ; AVX1-FALLBACK:       # %bb.0:
 ; AVX1-FALLBACK-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX1-FALLBACK-NEXT:    vmovdqa (%rsi), %xmm1
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpor {{.*}}(%rip), %xmm2, %xmm3
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm4
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm0, %xmm1, %xmm4
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vpsubq %xmm4, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX1-FALLBACK-NEXT:    retq
 ;
 ; AVX2-FALLBACK-LABEL: vec128_i64_signed_mem_mem:
 ; AVX2-FALLBACK:       # %bb.0:
 ; AVX2-FALLBACK-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX2-FALLBACK-NEXT:    vmovdqa (%rsi), %xmm1
 ; AVX2-FALLBACK-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
 ; AVX2-FALLBACK-NEXT:    vpor {{.*}}(%rip), %xmm2, %xmm3
 ; AVX2-FALLBACK-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm4
 ; AVX2-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm0, %xmm1, %xmm4
 ; AVX2-FALLBACK-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm1
 ; AVX2-FALLBACK-NEXT:    vpsubq %xmm4, %xmm1, %xmm1
-; AVX2-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX2-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; AVX2-FALLBACK-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; AVX2-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; AVX2-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; AVX2-FALLBACK-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX2-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX2-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX2-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX2-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX2-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX2-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX2-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX2-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX2-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX2-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX2-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX2-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX2-FALLBACK-NEXT:    retq
 ;
 ; XOP-LABEL: vec128_i64_signed_mem_mem:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vmovdqa (%rdi), %xmm0
 ; XOP-NEXT:    vmovdqa (%rsi), %xmm1
 ; XOP-NEXT:    vpcomgtq %xmm1, %xmm0, %xmm2
 ; XOP-NEXT:    vpor {{.*}}(%rip), %xmm2, %xmm3
 ; XOP-NEXT:    vpcomltq %xmm1, %xmm0, %xmm4
 ; XOP-NEXT:    vblendvpd %xmm4, %xmm0, %xmm1, %xmm4
 ; XOP-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm1
 ; XOP-NEXT:    vpsubq %xmm4, %xmm1, %xmm1
-; XOP-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; XOP-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; XOP-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; XOP-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; XOP-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; XOP-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; XOP-NEXT:    vpsllq $32, %xmm2, %xmm2
+; XOP-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; XOP-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; XOP-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; XOP-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; XOP-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; XOP-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; XOP-NEXT:    vpsllq $32, %xmm1, %xmm1
+; XOP-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; XOP-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; AVX512F-LABEL: vec128_i64_signed_mem_mem:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512F-NEXT:    vmovdqa (%rsi), %xmm1
 ; AVX512F-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
 ; AVX512F-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512F-NEXT:    vpminsq %zmm1, %zmm0, %zmm2
 ; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm1
 ; AVX512F-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsrlq $32, %xmm1, %xmm2
-; AVX512F-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; AVX512F-NEXT:    vpsrlq $32, %xmm3, %xmm4
-; AVX512F-NEXT:    vpmuludq %xmm4, %xmm1, %xmm4
-; AVX512F-NEXT:    vpaddq %xmm2, %xmm4, %xmm2
-; AVX512F-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX512F-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX512F-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX512F-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX512F-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX512F-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX512F-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX512F-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512F-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: vec128_i64_signed_mem_mem:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512VL-NEXT:    vmovdqa (%rsi), %xmm1
 ; AVX512VL-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
 ; AVX512VL-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
 ; AVX512VL-NEXT:    vmovdqa64 %xmm2, %xmm3 {%k1}
 ; AVX512VL-NEXT:    vpminsq %xmm1, %xmm0, %xmm2
 ; AVX512VL-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm1
 ; AVX512VL-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpsrlq $32, %xmm3, %xmm2
-; AVX512VL-NEXT:    vpmuludq %xmm2, %xmm1, %xmm2
-; AVX512VL-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; AVX512VL-NEXT:    vpmuludq %xmm3, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX512VL-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX512VL-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX512VL-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX512VL-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX512VL-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX512VL-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX512VL-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-FALLBACK-LABEL: vec128_i64_signed_mem_mem:
 ; AVX512BW-FALLBACK:       # %bb.0:
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa (%rsi), %xmm1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,1]
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512BW-FALLBACK-NEXT:    vpminsq %zmm1, %zmm0, %zmm2
 ; AVX512BW-FALLBACK-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm1
 ; AVX512BW-FALLBACK-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm2
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm1, %xmm4
-; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm2, %xmm4, %xmm2
-; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm1, %xmm1
-; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm4
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm4
+; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
 ; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX512BW-FALLBACK-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; AVX512BW-FALLBACK-NEXT:    vzeroupper
 ; AVX512BW-FALLBACK-NEXT:    retq
   %a1 = load <2 x i64>, <2 x i64>* %a1_addr
   %a2 = load <2 x i64>, <2 x i64>* %a2_addr
   %t3 = icmp sgt <2 x i64> %a1, %a2 ; signed
   %t4 = select <2 x i1> %t3, <2 x i64> <i64 -1, i64 -1>, <2 x i64> <i64 1, i64 1>
   %t5 = select <2 x i1> %t3, <2 x i64> %a2, <2 x i64> %a1
   %t6 = select <2 x i1> %t3, <2 x i64> %a1, <2 x i64> %a2

llvm/test/CodeGen/X86/midpoint-int-vec-256.ll

 
 ; Values come from regs
 
 define <4 x i64> @vec256_i64_signed_reg_reg(<4 x i64> %a1, <4 x i64> %a2) nounwind {
 ; AVX1-FALLBACK-LABEL: vec256_i64_signed_reg_reg:
 ; AVX1-FALLBACK:       # %bb.0:
 ; AVX1-FALLBACK-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-FALLBACK-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm4
+; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm9
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm7
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm5, %xmm0, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vpsubq %xmm7, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
-; AVX1-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm3, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpsubq %xmm6, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
+; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm6
+; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm7
+; AVX1-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
 ; AVX1-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm6
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm6
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
-; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm7
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm7, %xmm5, %xmm5
-; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm5, %xmm5
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm3, %xmm5, %xmm3
+; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm4
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm4
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm7, %xmm4
+; AVX1-FALLBACK-NEXT:    vpsrlq $33, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpor %xmm8, %xmm9, %xmm5
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm6, %xmm7
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm2, %xmm7, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm6, %xmm0
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm2, %xmm5, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; AVX1-FALLBACK-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-FALLBACK-NEXT:    retq
 ;
 ; AVX2-LABEL: vec256_i64_signed_reg_reg:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm2
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX2-NEXT:    vpor %ymm3, %ymm2, %ymm3
 ; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm4
 ; AVX2-NEXT:    vblendvpd %ymm4, %ymm0, %ymm1, %ymm4
 ; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm1
 ; AVX2-NEXT:    vpsubq %ymm4, %ymm1, %ymm1
-; AVX2-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX2-NEXT:    vpsrlq $32, %ymm3, %ymm2
-; AVX2-NEXT:    vpmuludq %ymm2, %ymm1, %ymm2
-; AVX2-NEXT:    vpsrlq $32, %ymm1, %ymm4
-; AVX2-NEXT:    vpmuludq %ymm3, %ymm4, %ymm4
-; AVX2-NEXT:    vpaddq %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX2-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX2-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX2-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX2-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX2-NEXT:    vpaddq %ymm1, %ymm4, %ymm1
+; AVX2-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX2-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; XOP-FALLBACK-LABEL: vec256_i64_signed_reg_reg:
 ; XOP-FALLBACK:       # %bb.0:
 ; XOP-FALLBACK-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; XOP-FALLBACK-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; XOP-FALLBACK-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm4
+; XOP-FALLBACK-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm9
 ; XOP-FALLBACK-NEXT:    vpcomgtq %xmm1, %xmm0, %xmm5
 ; XOP-FALLBACK-NEXT:    vpcomltq %xmm2, %xmm3, %xmm6
 ; XOP-FALLBACK-NEXT:    vpcomltq %xmm1, %xmm0, %xmm7
 ; XOP-FALLBACK-NEXT:    vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
 ; XOP-FALLBACK-NEXT:    vblendvpd %xmm5, %xmm0, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vpsubq %xmm7, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
-; XOP-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm3, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
 ; XOP-FALLBACK-NEXT:    vpsubq %xmm6, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
+; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm6
+; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm7
+; XOP-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
 ; XOP-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm6
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm6
-; XOP-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
-; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOP-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm7
-; XOP-FALLBACK-NEXT:    vpaddq %xmm7, %xmm5, %xmm5
-; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm5, %xmm5
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpaddq %xmm3, %xmm5, %xmm3
+; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm4
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm4
+; XOP-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm7, %xmm4
+; XOP-FALLBACK-NEXT:    vpsrlq $33, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vpor %xmm8, %xmm9, %xmm5
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm6, %xmm7
+; XOP-FALLBACK-NEXT:    vpaddq %xmm2, %xmm7, %xmm2
+; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm5
 ; XOP-FALLBACK-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpaddq %xmm0, %xmm6, %xmm0
+; XOP-FALLBACK-NEXT:    vpaddq %xmm2, %xmm5, %xmm2
 ; XOP-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; XOP-FALLBACK-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; XOP-FALLBACK-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOP-FALLBACK-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: vec256_i64_signed_reg_reg:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; XOPAVX1-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm4
+; XOPAVX1-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm9
 ; XOPAVX1-NEXT:    vpcomgtq %xmm1, %xmm0, %xmm5
 ; XOPAVX1-NEXT:    vpcomltq %xmm2, %xmm3, %xmm6
 ; XOPAVX1-NEXT:    vpcomltq %xmm1, %xmm0, %xmm7
 ; XOPAVX1-NEXT:    vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
 ; XOPAVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vpsubq %xmm7, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
-; XOPAVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsubq %xmm6, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsrlq $1, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsrlq $1, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpsrlq $1, %xmm2, %xmm6
+; XOPAVX1-NEXT:    vpsrlq $1, %xmm1, %xmm7
+; XOPAVX1-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
 ; XOPAVX1-NEXT:    vpor %xmm5, %xmm8, %xmm5
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
-; XOPAVX1-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm1, %xmm6
-; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm6, %xmm6
-; XOPAVX1-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
-; XOPAVX1-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm4, %xmm5
-; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm2, %xmm7
-; XOPAVX1-NEXT:    vpmuludq %xmm4, %xmm7, %xmm7
-; XOPAVX1-NEXT:    vpaddq %xmm7, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpsllq $32, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpaddq %xmm3, %xmm5, %xmm3
+; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm4
+; XOPAVX1-NEXT:    vpmuludq %xmm4, %xmm7, %xmm4
+; XOPAVX1-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; XOPAVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm7, %xmm4
+; XOPAVX1-NEXT:    vpsrlq $33, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpor %xmm8, %xmm9, %xmm5
+; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
+; XOPAVX1-NEXT:    vpmuludq %xmm7, %xmm6, %xmm7
+; XOPAVX1-NEXT:    vpaddq %xmm2, %xmm7, %xmm2
+; XOPAVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm6, %xmm5
 ; XOPAVX1-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpaddq %xmm0, %xmm6, %xmm0
+; XOPAVX1-NEXT:    vpaddq %xmm2, %xmm5, %xmm2
 ; XOPAVX1-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; XOPAVX1-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; AVX512F-LABEL: vec256_i64_signed_reg_reg:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
 ; AVX512F-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512F-NEXT:    vpminsq %zmm1, %zmm0, %zmm2
 ; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm1
 ; AVX512F-NEXT:    vpsubq %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlq $32, %ymm1, %ymm2
-; AVX512F-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlq $32, %ymm3, %ymm4
-; AVX512F-NEXT:    vpmuludq %ymm4, %ymm1, %ymm4
-; AVX512F-NEXT:    vpaddq %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX512F-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX512F-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX512F-NEXT:    vpaddq %ymm1, %ymm4, %ymm1
+; AVX512F-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: vec256_i64_signed_reg_reg:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpcmpgtq %ymm1, %ymm0, %k1
 ; AVX512VL-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX512VL-NEXT:    vmovdqa64 %ymm2, %ymm3 {%k1}
 ; AVX512VL-NEXT:    vpminsq %ymm1, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm1
 ; AVX512VL-NEXT:    vpsubq %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlq $32, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpmuludq %ymm2, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpsrlq $32, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpmuludq %ymm3, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpaddq %ymm4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX512VL-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX512VL-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX512VL-NEXT:    vpaddq %ymm1, %ymm4, %ymm1
+; AVX512VL-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-FALLBACK-LABEL: vec256_i64_signed_reg_reg:
 ; AVX512BW-FALLBACK:       # %bb.0:
 ; AVX512BW-FALLBACK-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512BW-FALLBACK-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-FALLBACK-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512BW-FALLBACK-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512BW-FALLBACK-NEXT:    vpminsq %zmm1, %zmm0, %zmm2
 ; AVX512BW-FALLBACK-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm1
 ; AVX512BW-FALLBACK-NEXT:    vpsubq %ymm2, %ymm1, %ymm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %ymm1, %ymm2
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %ymm3, %ymm4
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm4, %ymm1, %ymm4
-; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm2, %ymm4, %ymm2
-; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
-; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm1, %ymm4, %ymm1
+; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512BW-FALLBACK-NEXT:    retq
   %t3 = icmp sgt <4 x i64> %a1, %a2 ; signed
   %t4 = select <4 x i1> %t3, <4 x i64> <i64 -1, i64 -1, i64 -1, i64 -1>, <4 x i64> <i64 1, i64 1, i64 1, i64 1>
   %t5 = select <4 x i1> %t3, <4 x i64> %a2, <4 x i64> %a1
   %t6 = select <4 x i1> %t3, <4 x i64> %a1, <4 x i64> %a2
   %t7 = sub <4 x i64> %t6, %t5
   %t8 = lshr <4 x i64> %t7, <i64 1, i64 1, i64 1, i64 1>
   %t9 = mul nsw <4 x i64> %t8, %t4 ; signed

 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm6, %xmm5, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm4, %xmm7, %xmm4
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm0, %xmm1, %xmm4
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm9, %xmm0, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vpsubq %xmm4, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm5, %xmm2, %xmm3, %xmm4
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm8, %xmm2, %xmm3, %xmm3
 ; AVX1-FALLBACK-NEXT:    vpsubq %xmm4, %xmm3, %xmm3
-; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm3, %xmm3
-; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,1]
-; AVX1-FALLBACK-NEXT:    vpor %xmm4, %xmm9, %xmm5
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm6
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm6, %xmm1, %xmm6
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm7
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm7, %xmm7
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm7, %xmm6, %xmm6
-; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm3, %xmm5
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm3, %xmm7
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm7
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm7, %xmm5, %xmm5
-; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm5, %xmm5
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm3, %xmm3
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm2, %xmm5, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm3, %xmm4
+; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm5
+; AVX1-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
+; AVX1-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm10 = [1,1]
+; AVX1-FALLBACK-NEXT:    vpor %xmm10, %xmm9, %xmm7
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm1, %xmm1
+; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm7, %xmm6
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm6, %xmm5, %xmm6
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm1, %xmm6, %xmm1
+; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm5, %xmm5
+; AVX1-FALLBACK-NEXT:    vpsrlq $33, %xmm3, %xmm3
+; AVX1-FALLBACK-NEXT:    vpor %xmm10, %xmm8, %xmm6
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm6, %xmm3, %xmm3
+; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm6, %xmm7
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm4, %xmm7
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm3, %xmm7, %xmm3
+; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm3, %xmm3
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm6, %xmm4, %xmm4
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm6, %xmm0
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm2, %xmm4, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm5, %xmm0
 ; AVX1-FALLBACK-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-FALLBACK-NEXT:    retq
 ;
 ; AVX2-LABEL: vec256_i64_unsigned_reg_reg:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
 ; AVX2-NEXT:    vpxor %ymm2, %ymm1, %ymm3
 ; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm2
 ; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm2, %ymm4
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [1,1,1,1]
 ; AVX2-NEXT:    vpor %ymm5, %ymm4, %ymm5
 ; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm3, %ymm2
 ; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm2
 ; AVX2-NEXT:    vblendvpd %ymm4, %ymm0, %ymm1, %ymm1
 ; AVX2-NEXT:    vpsubq %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX2-NEXT:    vpsrlq $32, %ymm5, %ymm2
-; AVX2-NEXT:    vpmuludq %ymm2, %ymm1, %ymm2
-; AVX2-NEXT:    vpsrlq $32, %ymm1, %ymm3
-; AVX2-NEXT:    vpmuludq %ymm5, %ymm3, %ymm3
-; AVX2-NEXT:    vpaddq %ymm3, %ymm2, %ymm2
-; AVX2-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX2-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX2-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX2-NEXT:    vpmuludq %ymm5, %ymm1, %ymm1
-; AVX2-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpsrlq $32, %ymm5, %ymm3
+; AVX2-NEXT:    vpmuludq %ymm3, %ymm2, %ymm3
+; AVX2-NEXT:    vpaddq %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX2-NEXT:    vpmuludq %ymm5, %ymm2, %ymm2
 ; AVX2-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; XOP-FALLBACK-LABEL: vec256_i64_unsigned_reg_reg:
 ; XOP-FALLBACK:       # %bb.0:
 ; XOP-FALLBACK-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; XOP-FALLBACK-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; XOP-FALLBACK-NEXT:    vpcomgtuq %xmm2, %xmm3, %xmm4
+; XOP-FALLBACK-NEXT:    vpcomgtuq %xmm2, %xmm3, %xmm9
 ; XOP-FALLBACK-NEXT:    vpcomgtuq %xmm1, %xmm0, %xmm5
 ; XOP-FALLBACK-NEXT:    vpcomltuq %xmm2, %xmm3, %xmm6
 ; XOP-FALLBACK-NEXT:    vpcomltuq %xmm1, %xmm0, %xmm7
 ; XOP-FALLBACK-NEXT:    vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
 ; XOP-FALLBACK-NEXT:    vblendvpd %xmm5, %xmm0, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vpsubq %xmm7, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
-; XOP-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm3, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
 ; XOP-FALLBACK-NEXT:    vpsubq %xmm6, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
+; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm6
+; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm7
+; XOP-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
 ; XOP-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm6
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm6
-; XOP-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
-; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOP-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm7
-; XOP-FALLBACK-NEXT:    vpaddq %xmm7, %xmm5, %xmm5
-; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm5, %xmm5
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpaddq %xmm3, %xmm5, %xmm3
+; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm4
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm4
+; XOP-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm7, %xmm4
+; XOP-FALLBACK-NEXT:    vpsrlq $33, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vpor %xmm8, %xmm9, %xmm5
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm6, %xmm7
+; XOP-FALLBACK-NEXT:    vpaddq %xmm2, %xmm7, %xmm2
+; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm5
 ; XOP-FALLBACK-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpaddq %xmm0, %xmm6, %xmm0
+; XOP-FALLBACK-NEXT:    vpaddq %xmm2, %xmm5, %xmm2
 ; XOP-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; XOP-FALLBACK-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; XOP-FALLBACK-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOP-FALLBACK-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: vec256_i64_unsigned_reg_reg:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; XOPAVX1-NEXT:    vpcomgtuq %xmm2, %xmm3, %xmm4
+; XOPAVX1-NEXT:    vpcomgtuq %xmm2, %xmm3, %xmm9
 ; XOPAVX1-NEXT:    vpcomgtuq %xmm1, %xmm0, %xmm5
 ; XOPAVX1-NEXT:    vpcomltuq %xmm2, %xmm3, %xmm6
 ; XOPAVX1-NEXT:    vpcomltuq %xmm1, %xmm0, %xmm7
 ; XOPAVX1-NEXT:    vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
 ; XOPAVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vpsubq %xmm7, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
-; XOPAVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsubq %xmm6, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsrlq $1, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsrlq $1, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpsrlq $1, %xmm2, %xmm6
+; XOPAVX1-NEXT:    vpsrlq $1, %xmm1, %xmm7
+; XOPAVX1-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
 ; XOPAVX1-NEXT:    vpor %xmm5, %xmm8, %xmm5
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
-; XOPAVX1-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm1, %xmm6
-; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm6, %xmm6
-; XOPAVX1-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
-; XOPAVX1-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm4, %xmm5
-; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm2, %xmm7
-; XOPAVX1-NEXT:    vpmuludq %xmm4, %xmm7, %xmm7
-; XOPAVX1-NEXT:    vpaddq %xmm7, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpsllq $32, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpaddq %xmm3, %xmm5, %xmm3
+; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm4
+; XOPAVX1-NEXT:    vpmuludq %xmm4, %xmm7, %xmm4
+; XOPAVX1-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; XOPAVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm7, %xmm4
+; XOPAVX1-NEXT:    vpsrlq $33, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpor %xmm8, %xmm9, %xmm5
+; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
+; XOPAVX1-NEXT:    vpmuludq %xmm7, %xmm6, %xmm7
+; XOPAVX1-NEXT:    vpaddq %xmm2, %xmm7, %xmm2
+; XOPAVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm6, %xmm5
 ; XOPAVX1-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpaddq %xmm0, %xmm6, %xmm0
+; XOPAVX1-NEXT:    vpaddq %xmm2, %xmm5, %xmm2
 ; XOPAVX1-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; XOPAVX1-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; AVX512F-LABEL: vec256_i64_unsigned_reg_reg:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vpcmpnleuq %zmm1, %zmm0, %k1
 ; AVX512F-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512F-NEXT:    vpminuq %zmm1, %zmm0, %zmm2
 ; AVX512F-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm1
 ; AVX512F-NEXT:    vpsubq %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlq $32, %ymm1, %ymm2
-; AVX512F-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlq $32, %ymm3, %ymm4
-; AVX512F-NEXT:    vpmuludq %ymm4, %ymm1, %ymm4
-; AVX512F-NEXT:    vpaddq %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX512F-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX512F-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX512F-NEXT:    vpaddq %ymm1, %ymm4, %ymm1
+; AVX512F-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: vec256_i64_unsigned_reg_reg:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpcmpnleuq %ymm1, %ymm0, %k1
 ; AVX512VL-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX512VL-NEXT:    vmovdqa64 %ymm2, %ymm3 {%k1}
 ; AVX512VL-NEXT:    vpminuq %ymm1, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpmaxuq %ymm1, %ymm0, %ymm1
 ; AVX512VL-NEXT:    vpsubq %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlq $32, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpmuludq %ymm2, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpsrlq $32, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpmuludq %ymm3, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpaddq %ymm4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX512VL-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX512VL-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX512VL-NEXT:    vpaddq %ymm1, %ymm4, %ymm1
+; AVX512VL-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-FALLBACK-LABEL: vec256_i64_unsigned_reg_reg:
 ; AVX512BW-FALLBACK:       # %bb.0:
 ; AVX512BW-FALLBACK-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512BW-FALLBACK-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-FALLBACK-NEXT:    vpcmpnleuq %zmm1, %zmm0, %k1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512BW-FALLBACK-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512BW-FALLBACK-NEXT:    vpminuq %zmm1, %zmm0, %zmm2
 ; AVX512BW-FALLBACK-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm1
 ; AVX512BW-FALLBACK-NEXT:    vpsubq %ymm2, %ymm1, %ymm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %ymm1, %ymm2
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %ymm3, %ymm4
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm4, %ymm1, %ymm4
-; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm2, %ymm4, %ymm2
-; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
-; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm1, %ymm4, %ymm1
+; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512BW-FALLBACK-NEXT:    retq
   %t3 = icmp ugt <4 x i64> %a1, %a2
   %t4 = select <4 x i1> %t3, <4 x i64> <i64 -1, i64 -1, i64 -1, i64 -1>, <4 x i64> <i64 1, i64 1, i64 1, i64 1>
   %t5 = select <4 x i1> %t3, <4 x i64> %a2, <4 x i64> %a1
   %t6 = select <4 x i1> %t3, <4 x i64> %a1, <4 x i64> %a2
   %t7 = sub <4 x i64> %t6, %t5
   %t8 = lshr <4 x i64> %t7, <i64 1, i64 1, i64 1, i64 1>
   %t9 = mul <4 x i64> %t8, %t4
   %a10 = add <4 x i64> %t9, %a1
   ret <4 x i64> %a10
 }
 
 ; Values are loaded. Only check signed case.
 
 define <4 x i64> @vec256_i64_signed_mem_reg(<4 x i64>* %a1_addr, <4 x i64> %a2) nounwind {
 ; AVX1-FALLBACK-LABEL: vec256_i64_signed_mem_reg:
 ; AVX1-FALLBACK:       # %bb.0:
 ; AVX1-FALLBACK-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-FALLBACK-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX1-FALLBACK-NEXT:    vmovdqa 16(%rdi), %xmm3
-; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm4
+; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm9
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm7
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm7, %xmm1, %xmm0, %xmm7
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm5, %xmm1, %xmm0, %xmm0
 ; AVX1-FALLBACK-NEXT:    vpsubq %xmm7, %xmm0, %xmm0
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
-; AVX1-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm3, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpsubq %xmm6, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm0, %xmm0
+; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm6
+; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm0, %xmm7
+; AVX1-FALLBACK-NEXT:    vpsrlq $33, %xmm0, %xmm0
 ; AVX1-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
 ; AVX1-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm0, %xmm7
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm0, %xmm6
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm6
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
-; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm0, %xmm0
-; AVX1-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm7
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm7, %xmm5, %xmm5
-; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm5, %xmm5
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm3, %xmm5, %xmm3
+; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm4
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm4
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
+; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm7, %xmm4
+; AVX1-FALLBACK-NEXT:    vpsrlq $33, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpor %xmm8, %xmm9, %xmm5
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm6, %xmm7
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm2, %xmm7, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm1, %xmm6, %xmm1
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm2, %xmm5, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; AVX1-FALLBACK-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-FALLBACK-NEXT:    retq
 ;
 ; AVX2-LABEL: vec256_i64_signed_mem_reg:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm2
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX2-NEXT:    vpor %ymm3, %ymm2, %ymm3
 ; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm4
 ; AVX2-NEXT:    vblendvpd %ymm4, %ymm1, %ymm0, %ymm4
 ; AVX2-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsubq %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpsrlq $1, %ymm0, %ymm0
-; AVX2-NEXT:    vpsrlq $32, %ymm3, %ymm2
-; AVX2-NEXT:    vpmuludq %ymm2, %ymm0, %ymm2
-; AVX2-NEXT:    vpsrlq $32, %ymm0, %ymm4
-; AVX2-NEXT:    vpmuludq %ymm3, %ymm4, %ymm4
-; AVX2-NEXT:    vpaddq %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX2-NEXT:    vpsrlq $1, %ymm0, %ymm2
+; AVX2-NEXT:    vpsrlq $33, %ymm0, %ymm0
 ; AVX2-NEXT:    vpmuludq %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpaddq %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX2-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX2-NEXT:    vpaddq %ymm0, %ymm4, %ymm0
+; AVX2-NEXT:    vpsllq $32, %ymm0, %ymm0
+; AVX2-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; XOP-FALLBACK-LABEL: vec256_i64_signed_mem_reg:
 ; XOP-FALLBACK:       # %bb.0:
 ; XOP-FALLBACK-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; XOP-FALLBACK-NEXT:    vmovdqa (%rdi), %xmm1
 ; XOP-FALLBACK-NEXT:    vmovdqa 16(%rdi), %xmm3
-; XOP-FALLBACK-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm4
+; XOP-FALLBACK-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm9
 ; XOP-FALLBACK-NEXT:    vpcomgtq %xmm0, %xmm1, %xmm5
 ; XOP-FALLBACK-NEXT:    vpcomltq %xmm2, %xmm3, %xmm6
 ; XOP-FALLBACK-NEXT:    vpcomltq %xmm0, %xmm1, %xmm7
 ; XOP-FALLBACK-NEXT:    vblendvpd %xmm7, %xmm1, %xmm0, %xmm7
 ; XOP-FALLBACK-NEXT:    vblendvpd %xmm5, %xmm1, %xmm0, %xmm0
 ; XOP-FALLBACK-NEXT:    vpsubq %xmm7, %xmm0, %xmm0
 ; XOP-FALLBACK-NEXT:    vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
-; XOP-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm3, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
 ; XOP-FALLBACK-NEXT:    vpsubq %xmm6, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm0, %xmm0
+; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm6
+; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm0, %xmm7
+; XOP-FALLBACK-NEXT:    vpsrlq $33, %xmm0, %xmm0
 ; XOP-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
 ; XOP-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm0, %xmm7
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm0, %xmm6
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm6
-; XOP-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
-; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm0, %xmm0
-; XOP-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm7
-; XOP-FALLBACK-NEXT:    vpaddq %xmm7, %xmm5, %xmm5
-; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm5, %xmm5
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpaddq %xmm3, %xmm5, %xmm3
+; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm4
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm4
+; XOP-FALLBACK-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
+; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm0, %xmm0
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm7, %xmm4
+; XOP-FALLBACK-NEXT:    vpsrlq $33, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vpor %xmm8, %xmm9, %xmm5
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm6, %xmm7
+; XOP-FALLBACK-NEXT:    vpaddq %xmm2, %xmm7, %xmm2
+; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm5
 ; XOP-FALLBACK-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpaddq %xmm1, %xmm6, %xmm1
+; XOP-FALLBACK-NEXT:    vpaddq %xmm2, %xmm5, %xmm2
 ; XOP-FALLBACK-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; XOP-FALLBACK-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; XOP-FALLBACK-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOP-FALLBACK-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: vec256_i64_signed_mem_reg:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; XOPAVX1-NEXT:    vmovdqa (%rdi), %xmm1
 ; XOPAVX1-NEXT:    vmovdqa 16(%rdi), %xmm3
-; XOPAVX1-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm4
+; XOPAVX1-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm9
 ; XOPAVX1-NEXT:    vpcomgtq %xmm0, %xmm1, %xmm5
 ; XOPAVX1-NEXT:    vpcomltq %xmm2, %xmm3, %xmm6
 ; XOPAVX1-NEXT:    vpcomltq %xmm0, %xmm1, %xmm7
 ; XOPAVX1-NEXT:    vblendvpd %xmm7, %xmm1, %xmm0, %xmm7
 ; XOPAVX1-NEXT:    vblendvpd %xmm5, %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vpsubq %xmm7, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
-; XOPAVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsubq %xmm6, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsrlq $1, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsrlq $1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpsrlq $1, %xmm2, %xmm6
+; XOPAVX1-NEXT:    vpsrlq $1, %xmm0, %xmm7
+; XOPAVX1-NEXT:    vpsrlq $33, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
 ; XOPAVX1-NEXT:    vpor %xmm5, %xmm8, %xmm5
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
-; XOPAVX1-NEXT:    vpmuludq %xmm7, %xmm0, %xmm7
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm0, %xmm6
-; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm6, %xmm6
-; XOPAVX1-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
-; XOPAVX1-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm4, %xmm5
-; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm2, %xmm7
-; XOPAVX1-NEXT:    vpmuludq %xmm4, %xmm7, %xmm7
-; XOPAVX1-NEXT:    vpaddq %xmm7, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpsllq $32, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpaddq %xmm3, %xmm5, %xmm3
+; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm4
+; XOPAVX1-NEXT:    vpmuludq %xmm4, %xmm7, %xmm4
+; XOPAVX1-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
+; XOPAVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm7, %xmm4
+; XOPAVX1-NEXT:    vpsrlq $33, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpor %xmm8, %xmm9, %xmm5
+; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
+; XOPAVX1-NEXT:    vpmuludq %xmm7, %xmm6, %xmm7
+; XOPAVX1-NEXT:    vpaddq %xmm2, %xmm7, %xmm2
+; XOPAVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm6, %xmm5
 ; XOPAVX1-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpaddq %xmm1, %xmm6, %xmm1
+; XOPAVX1-NEXT:    vpaddq %xmm2, %xmm5, %xmm2
 ; XOPAVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; AVX512F-LABEL: vec256_i64_signed_mem_reg:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512F-NEXT:    vpcmpgtq %zmm0, %zmm1, %k1
 ; AVX512F-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512F-NEXT:    vpminsq %zmm0, %zmm1, %zmm2
 ; AVX512F-NEXT:    vpmaxsq %zmm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlq $1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlq $32, %ymm0, %ymm2
-; AVX512F-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlq $32, %ymm3, %ymm4
-; AVX512F-NEXT:    vpmuludq %ymm4, %ymm0, %ymm4
-; AVX512F-NEXT:    vpaddq %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlq $1, %ymm0, %ymm2
+; AVX512F-NEXT:    vpsrlq $33, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmuludq %ymm3, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddq %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX512F-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX512F-NEXT:    vpaddq %ymm0, %ymm4, %ymm0
+; AVX512F-NEXT:    vpsllq $32, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: vec256_i64_signed_mem_reg:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512VL-NEXT:    vpcmpgtq %ymm0, %ymm1, %k1
 ; AVX512VL-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX512VL-NEXT:    vmovdqa64 %ymm2, %ymm3 {%k1}
 ; AVX512VL-NEXT:    vpminsq %ymm0, %ymm1, %ymm2
 ; AVX512VL-NEXT:    vpmaxsq %ymm0, %ymm1, %ymm0
 ; AVX512VL-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlq $1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlq $32, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpmuludq %ymm2, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpsrlq $32, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpmuludq %ymm3, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpaddq %ymm4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsrlq $1, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlq $33, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpmuludq %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpaddq %ymm1, %ymm2, %ymm1
+; AVX512VL-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX512VL-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX512VL-NEXT:    vpaddq %ymm0, %ymm4, %ymm0
+; AVX512VL-NEXT:    vpsllq $32, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-FALLBACK-LABEL: vec256_i64_signed_mem_reg:
 ; AVX512BW-FALLBACK:       # %bb.0:
 ; AVX512BW-FALLBACK-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpgtq %zmm0, %zmm1, %k1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512BW-FALLBACK-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512BW-FALLBACK-NEXT:    vpminsq %zmm0, %zmm1, %zmm2
 ; AVX512BW-FALLBACK-NEXT:    vpmaxsq %zmm0, %zmm1, %zmm0
 ; AVX512BW-FALLBACK-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %ymm0, %ymm0
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %ymm0, %ymm2
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %ymm3, %ymm4
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm4, %ymm0, %ymm4
-; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm2, %ymm4, %ymm2
-; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %ymm0, %ymm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $33, %ymm0, %ymm0
 ; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm3, %ymm0, %ymm0
-; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm1, %ymm2, %ymm1
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm0, %ymm4, %ymm0
+; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %ymm0, %ymm0
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512BW-FALLBACK-NEXT:    retq
   %a1 = load <4 x i64>, <4 x i64>* %a1_addr
   %t3 = icmp sgt <4 x i64> %a1, %a2 ; signed
   %t4 = select <4 x i1> %t3, <4 x i64> <i64 -1, i64 -1, i64 -1, i64 -1>, <4 x i64> <i64 1, i64 1, i64 1, i64 1>
   %t5 = select <4 x i1> %t3, <4 x i64> %a2, <4 x i64> %a1
   %t6 = select <4 x i1> %t3, <4 x i64> %a1, <4 x i64> %a2
   %t7 = sub <4 x i64> %t6, %t5
   %t8 = lshr <4 x i64> %t7, <i64 1, i64 1, i64 1, i64 1>
   %t9 = mul nsw <4 x i64> %t8, %t4 ; signed
   %a10 = add nsw <4 x i64> %t9, %a1 ; signed
   ret <4 x i64> %a10
 }
 
 define <4 x i64> @vec256_i64_signed_reg_mem(<4 x i64> %a1, <4 x i64>* %a2_addr) nounwind {
 ; AVX1-FALLBACK-LABEL: vec256_i64_signed_reg_mem:
 ; AVX1-FALLBACK:       # %bb.0:
 ; AVX1-FALLBACK-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX1-FALLBACK-NEXT:    vmovdqa 16(%rdi), %xmm2
 ; AVX1-FALLBACK-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm4
+; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm9
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm7
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm5, %xmm0, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vpsubq %xmm7, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
-; AVX1-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm3, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpsubq %xmm6, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
+; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm6
+; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm7
+; AVX1-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
 ; AVX1-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm6
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm6
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
-; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm7
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm7, %xmm5, %xmm5
-; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm5, %xmm5
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm3, %xmm5, %xmm3
+; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm4
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm4
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm7, %xmm4
+; AVX1-FALLBACK-NEXT:    vpsrlq $33, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpor %xmm8, %xmm9, %xmm5
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm6, %xmm7
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm2, %xmm7, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm6, %xmm0
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm2, %xmm5, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; AVX1-FALLBACK-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-FALLBACK-NEXT:    retq
 ;
 ; AVX2-LABEL: vec256_i64_signed_reg_mem:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm2
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX2-NEXT:    vpor %ymm3, %ymm2, %ymm3
 ; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm4
 ; AVX2-NEXT:    vblendvpd %ymm4, %ymm0, %ymm1, %ymm4
 ; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm1
 ; AVX2-NEXT:    vpsubq %ymm4, %ymm1, %ymm1
-; AVX2-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX2-NEXT:    vpsrlq $32, %ymm3, %ymm2
-; AVX2-NEXT:    vpmuludq %ymm2, %ymm1, %ymm2
-; AVX2-NEXT:    vpsrlq $32, %ymm1, %ymm4
-; AVX2-NEXT:    vpmuludq %ymm3, %ymm4, %ymm4
-; AVX2-NEXT:    vpaddq %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX2-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX2-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX2-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX2-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX2-NEXT:    vpaddq %ymm1, %ymm4, %ymm1
+; AVX2-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX2-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; XOP-FALLBACK-LABEL: vec256_i64_signed_reg_mem:
 ; XOP-FALLBACK:       # %bb.0:
 ; XOP-FALLBACK-NEXT:    vmovdqa (%rdi), %xmm1
 ; XOP-FALLBACK-NEXT:    vmovdqa 16(%rdi), %xmm2
 ; XOP-FALLBACK-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; XOP-FALLBACK-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm4
+; XOP-FALLBACK-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm9
 ; XOP-FALLBACK-NEXT:    vpcomgtq %xmm1, %xmm0, %xmm5
 ; XOP-FALLBACK-NEXT:    vpcomltq %xmm2, %xmm3, %xmm6
 ; XOP-FALLBACK-NEXT:    vpcomltq %xmm1, %xmm0, %xmm7
 ; XOP-FALLBACK-NEXT:    vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
 ; XOP-FALLBACK-NEXT:    vblendvpd %xmm5, %xmm0, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vpsubq %xmm7, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
-; XOP-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm3, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
 ; XOP-FALLBACK-NEXT:    vpsubq %xmm6, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
+; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm6
+; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm7
+; XOP-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
 ; XOP-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm6
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm6
-; XOP-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
-; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOP-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm7
-; XOP-FALLBACK-NEXT:    vpaddq %xmm7, %xmm5, %xmm5
-; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm5, %xmm5
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpaddq %xmm3, %xmm5, %xmm3
+; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm4
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm4
+; XOP-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm7, %xmm4
+; XOP-FALLBACK-NEXT:    vpsrlq $33, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vpor %xmm8, %xmm9, %xmm5
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm6, %xmm7
+; XOP-FALLBACK-NEXT:    vpaddq %xmm2, %xmm7, %xmm2
+; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm5
 ; XOP-FALLBACK-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpaddq %xmm0, %xmm6, %xmm0
+; XOP-FALLBACK-NEXT:    vpaddq %xmm2, %xmm5, %xmm2
 ; XOP-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; XOP-FALLBACK-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; XOP-FALLBACK-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOP-FALLBACK-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: vec256_i64_signed_reg_mem:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovdqa (%rdi), %xmm1
 ; XOPAVX1-NEXT:    vmovdqa 16(%rdi), %xmm2
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; XOPAVX1-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm4
+; XOPAVX1-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm9
 ; XOPAVX1-NEXT:    vpcomgtq %xmm1, %xmm0, %xmm5
 ; XOPAVX1-NEXT:    vpcomltq %xmm2, %xmm3, %xmm6
 ; XOPAVX1-NEXT:    vpcomltq %xmm1, %xmm0, %xmm7
 ; XOPAVX1-NEXT:    vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
 ; XOPAVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vpsubq %xmm7, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
-; XOPAVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsubq %xmm6, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsrlq $1, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsrlq $1, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpsrlq $1, %xmm2, %xmm6
+; XOPAVX1-NEXT:    vpsrlq $1, %xmm1, %xmm7
+; XOPAVX1-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
 ; XOPAVX1-NEXT:    vpor %xmm5, %xmm8, %xmm5
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
-; XOPAVX1-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm1, %xmm6
-; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm6, %xmm6
-; XOPAVX1-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
-; XOPAVX1-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm4, %xmm5
-; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm2, %xmm7
-; XOPAVX1-NEXT:    vpmuludq %xmm4, %xmm7, %xmm7
-; XOPAVX1-NEXT:    vpaddq %xmm7, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpsllq $32, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpaddq %xmm3, %xmm5, %xmm3
+; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm4
+; XOPAVX1-NEXT:    vpmuludq %xmm4, %xmm7, %xmm4
+; XOPAVX1-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; XOPAVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm7, %xmm4
+; XOPAVX1-NEXT:    vpsrlq $33, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpor %xmm8, %xmm9, %xmm5
+; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
+; XOPAVX1-NEXT:    vpmuludq %xmm7, %xmm6, %xmm7
+; XOPAVX1-NEXT:    vpaddq %xmm2, %xmm7, %xmm2
+; XOPAVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm6, %xmm5
 ; XOPAVX1-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpaddq %xmm0, %xmm6, %xmm0
+; XOPAVX1-NEXT:    vpaddq %xmm2, %xmm5, %xmm2
 ; XOPAVX1-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; XOPAVX1-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; AVX512F-LABEL: vec256_i64_signed_reg_mem:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512F-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
 ; AVX512F-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512F-NEXT:    vpminsq %zmm1, %zmm0, %zmm2
 ; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm1
 ; AVX512F-NEXT:    vpsubq %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlq $32, %ymm1, %ymm2
-; AVX512F-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlq $32, %ymm3, %ymm4
-; AVX512F-NEXT:    vpmuludq %ymm4, %ymm1, %ymm4
-; AVX512F-NEXT:    vpaddq %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX512F-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX512F-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX512F-NEXT:    vpaddq %ymm1, %ymm4, %ymm1
+; AVX512F-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: vec256_i64_signed_reg_mem:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512VL-NEXT:    vpcmpgtq %ymm1, %ymm0, %k1
 ; AVX512VL-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX512VL-NEXT:    vmovdqa64 %ymm2, %ymm3 {%k1}
 ; AVX512VL-NEXT:    vpminsq %ymm1, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm1
 ; AVX512VL-NEXT:    vpsubq %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlq $32, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpmuludq %ymm2, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpsrlq $32, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpmuludq %ymm3, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpaddq %ymm4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX512VL-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX512VL-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX512VL-NEXT:    vpaddq %ymm1, %ymm4, %ymm1
+; AVX512VL-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-FALLBACK-LABEL: vec256_i64_signed_reg_mem:
 ; AVX512BW-FALLBACK:       # %bb.0:
 ; AVX512BW-FALLBACK-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512BW-FALLBACK-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512BW-FALLBACK-NEXT:    vpminsq %zmm1, %zmm0, %zmm2
 ; AVX512BW-FALLBACK-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm1
 ; AVX512BW-FALLBACK-NEXT:    vpsubq %ymm2, %ymm1, %ymm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %ymm1, %ymm2
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %ymm3, %ymm4
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm4, %ymm1, %ymm4
-; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm2, %ymm4, %ymm2
-; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
-; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm1, %ymm4, %ymm1
+; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512BW-FALLBACK-NEXT:    retq
   %a2 = load <4 x i64>, <4 x i64>* %a2_addr
   %t3 = icmp sgt <4 x i64> %a1, %a2 ; signed
   %t4 = select <4 x i1> %t3, <4 x i64> <i64 -1, i64 -1, i64 -1, i64 -1>, <4 x i64> <i64 1, i64 1, i64 1, i64 1>
   %t5 = select <4 x i1> %t3, <4 x i64> %a2, <4 x i64> %a1
   %t6 = select <4 x i1> %t3, <4 x i64> %a1, <4 x i64> %a2
   %t7 = sub <4 x i64> %t6, %t5
   %t8 = lshr <4 x i64> %t7, <i64 1, i64 1, i64 1, i64 1>
   %t9 = mul nsw <4 x i64> %t8, %t4 ; signed
   %a10 = add nsw <4 x i64> %t9, %a1 ; signed
   ret <4 x i64> %a10
 }
 
 define <4 x i64> @vec256_i64_signed_mem_mem(<4 x i64>* %a1_addr, <4 x i64>* %a2_addr) nounwind {
 ; AVX1-FALLBACK-LABEL: vec256_i64_signed_mem_mem:
 ; AVX1-FALLBACK:       # %bb.0:
 ; AVX1-FALLBACK-NEXT:    vmovdqa (%rsi), %xmm1
 ; AVX1-FALLBACK-NEXT:    vmovdqa 16(%rsi), %xmm2
 ; AVX1-FALLBACK-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX1-FALLBACK-NEXT:    vmovdqa 16(%rdi), %xmm3
-; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm4
+; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm9
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm7
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm5, %xmm0, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vpsubq %xmm7, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
-; AVX1-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm3, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpsubq %xmm6, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
+; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm6
+; AVX1-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm7
+; AVX1-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; AVX1-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
 ; AVX1-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm6
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm6
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
-; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; AVX1-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
-; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm7
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm7, %xmm5, %xmm5
-; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm5, %xmm5
-; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm3, %xmm5, %xmm3
+; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm4
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm4
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm7, %xmm4
+; AVX1-FALLBACK-NEXT:    vpsrlq $33, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpor %xmm8, %xmm9, %xmm5
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm6, %xmm7
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm2, %xmm7, %xmm2
+; AVX1-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX1-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm5
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm6, %xmm0
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm2, %xmm5, %xmm2
 ; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; AVX1-FALLBACK-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; AVX1-FALLBACK-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-FALLBACK-NEXT:    retq
 ;
 ; AVX2-LABEL: vec256_i64_signed_mem_mem:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX2-NEXT:    vmovdqa (%rsi), %ymm1
 ; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm2
 ; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX2-NEXT:    vpor %ymm3, %ymm2, %ymm3
 ; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm4
 ; AVX2-NEXT:    vblendvpd %ymm4, %ymm0, %ymm1, %ymm4
 ; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm1
 ; AVX2-NEXT:    vpsubq %ymm4, %ymm1, %ymm1
-; AVX2-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX2-NEXT:    vpsrlq $32, %ymm3, %ymm2
-; AVX2-NEXT:    vpmuludq %ymm2, %ymm1, %ymm2
-; AVX2-NEXT:    vpsrlq $32, %ymm1, %ymm4
-; AVX2-NEXT:    vpmuludq %ymm3, %ymm4, %ymm4
-; AVX2-NEXT:    vpaddq %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX2-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX2-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX2-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX2-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX2-NEXT:    vpaddq %ymm1, %ymm4, %ymm1
+; AVX2-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX2-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX2-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; XOP-FALLBACK-LABEL: vec256_i64_signed_mem_mem:
 ; XOP-FALLBACK:       # %bb.0:
 ; XOP-FALLBACK-NEXT:    vmovdqa (%rsi), %xmm1
 ; XOP-FALLBACK-NEXT:    vmovdqa 16(%rsi), %xmm2
 ; XOP-FALLBACK-NEXT:    vmovdqa (%rdi), %xmm0
 ; XOP-FALLBACK-NEXT:    vmovdqa 16(%rdi), %xmm3
-; XOP-FALLBACK-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm4
+; XOP-FALLBACK-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm9
 ; XOP-FALLBACK-NEXT:    vpcomgtq %xmm1, %xmm0, %xmm5
 ; XOP-FALLBACK-NEXT:    vpcomltq %xmm2, %xmm3, %xmm6
 ; XOP-FALLBACK-NEXT:    vpcomltq %xmm1, %xmm0, %xmm7
 ; XOP-FALLBACK-NEXT:    vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
 ; XOP-FALLBACK-NEXT:    vblendvpd %xmm5, %xmm0, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vpsubq %xmm7, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
-; XOP-FALLBACK-NEXT:    vblendvpd %xmm4, %xmm3, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
 ; XOP-FALLBACK-NEXT:    vpsubq %xmm6, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm1
+; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm2, %xmm6
+; XOP-FALLBACK-NEXT:    vpsrlq $1, %xmm1, %xmm7
+; XOP-FALLBACK-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; XOP-FALLBACK-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
 ; XOP-FALLBACK-NEXT:    vpor %xmm5, %xmm8, %xmm5
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm1, %xmm6
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm6
-; XOP-FALLBACK-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
-; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOP-FALLBACK-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm4, %xmm5
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
-; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm2, %xmm7
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm7
-; XOP-FALLBACK-NEXT:    vpaddq %xmm7, %xmm5, %xmm5
-; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm5, %xmm5
-; XOP-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpaddq %xmm3, %xmm5, %xmm3
+; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm4
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm4, %xmm7, %xmm4
+; XOP-FALLBACK-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm1, %xmm1
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm7, %xmm4
+; XOP-FALLBACK-NEXT:    vpsrlq $33, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vpor %xmm8, %xmm9, %xmm5
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vpsrlq $32, %xmm5, %xmm7
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm7, %xmm6, %xmm7
+; XOP-FALLBACK-NEXT:    vpaddq %xmm2, %xmm7, %xmm2
+; XOP-FALLBACK-NEXT:    vpsllq $32, %xmm2, %xmm2
+; XOP-FALLBACK-NEXT:    vpmuludq %xmm5, %xmm6, %xmm5
 ; XOP-FALLBACK-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; XOP-FALLBACK-NEXT:    vpaddq %xmm0, %xmm6, %xmm0
+; XOP-FALLBACK-NEXT:    vpaddq %xmm2, %xmm5, %xmm2
 ; XOP-FALLBACK-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; XOP-FALLBACK-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; XOP-FALLBACK-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOP-FALLBACK-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: vec256_i64_signed_mem_mem:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovdqa (%rsi), %xmm1
 ; XOPAVX1-NEXT:    vmovdqa 16(%rsi), %xmm2
 ; XOPAVX1-NEXT:    vmovdqa (%rdi), %xmm0
 ; XOPAVX1-NEXT:    vmovdqa 16(%rdi), %xmm3
-; XOPAVX1-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm4
+; XOPAVX1-NEXT:    vpcomgtq %xmm2, %xmm3, %xmm9
 ; XOPAVX1-NEXT:    vpcomgtq %xmm1, %xmm0, %xmm5
 ; XOPAVX1-NEXT:    vpcomltq %xmm2, %xmm3, %xmm6
 ; XOPAVX1-NEXT:    vpcomltq %xmm1, %xmm0, %xmm7
 ; XOPAVX1-NEXT:    vblendvpd %xmm7, %xmm0, %xmm1, %xmm7
 ; XOPAVX1-NEXT:    vblendvpd %xmm5, %xmm0, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vpsubq %xmm7, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vblendvpd %xmm6, %xmm3, %xmm2, %xmm6
-; XOPAVX1-NEXT:    vblendvpd %xmm4, %xmm3, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vblendvpd %xmm9, %xmm3, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsubq %xmm6, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsrlq $1, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsrlq $1, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpsrlq $1, %xmm2, %xmm6
+; XOPAVX1-NEXT:    vpsrlq $1, %xmm1, %xmm7
+; XOPAVX1-NEXT:    vpsrlq $33, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1]
 ; XOPAVX1-NEXT:    vpor %xmm5, %xmm8, %xmm5
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
-; XOPAVX1-NEXT:    vpmuludq %xmm7, %xmm1, %xmm7
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm1, %xmm6
-; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm6, %xmm6
-; XOPAVX1-NEXT:    vpaddq %xmm6, %xmm7, %xmm6
-; XOPAVX1-NEXT:    vpsllq $32, %xmm6, %xmm6
 ; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpor %xmm4, %xmm8, %xmm4
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm4, %xmm5
-; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm2, %xmm5
-; XOPAVX1-NEXT:    vpsrlq $32, %xmm2, %xmm7
-; XOPAVX1-NEXT:    vpmuludq %xmm4, %xmm7, %xmm7
-; XOPAVX1-NEXT:    vpaddq %xmm7, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpsllq $32, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpaddq %xmm3, %xmm5, %xmm3
+; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm4
+; XOPAVX1-NEXT:    vpmuludq %xmm4, %xmm7, %xmm4
+; XOPAVX1-NEXT:    vpaddq %xmm1, %xmm4, %xmm1
+; XOPAVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm7, %xmm4
+; XOPAVX1-NEXT:    vpsrlq $33, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpor %xmm8, %xmm9, %xmm5
+; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
+; XOPAVX1-NEXT:    vpmuludq %xmm7, %xmm6, %xmm7
+; XOPAVX1-NEXT:    vpaddq %xmm2, %xmm7, %xmm2
+; XOPAVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpmuludq %xmm5, %xmm6, %xmm5
 ; XOPAVX1-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpaddq %xmm0, %xmm6, %xmm0
+; XOPAVX1-NEXT:    vpaddq %xmm2, %xmm5, %xmm2
 ; XOPAVX1-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; XOPAVX1-NEXT:    vpaddq %xmm0, %xmm4, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; AVX512F-LABEL: vec256_i64_signed_mem_mem:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX512F-NEXT:    vmovdqa (%rsi), %ymm1
 ; AVX512F-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
 ; AVX512F-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512F-NEXT:    vpminsq %zmm1, %zmm0, %zmm2
 ; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm1
 ; AVX512F-NEXT:    vpsubq %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX512F-NEXT:    vpsrlq $32, %ymm1, %ymm2
-; AVX512F-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT:    vpsrlq $32, %ymm3, %ymm4
-; AVX512F-NEXT:    vpmuludq %ymm4, %ymm1, %ymm4
-; AVX512F-NEXT:    vpaddq %ymm2, %ymm4, %ymm2
-; AVX512F-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX512F-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX512F-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX512F-NEXT:    vpaddq %ymm1, %ymm4, %ymm1
+; AVX512F-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: vec256_i64_signed_mem_mem:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX512VL-NEXT:    vmovdqa (%rsi), %ymm1
 ; AVX512VL-NEXT:    vpcmpgtq %ymm1, %ymm0, %k1
 ; AVX512VL-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX512VL-NEXT:    vmovdqa64 %ymm2, %ymm3 {%k1}
 ; AVX512VL-NEXT:    vpminsq %ymm1, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpmaxsq %ymm1, %ymm0, %ymm1
 ; AVX512VL-NEXT:    vpsubq %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsrlq $32, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpmuludq %ymm2, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpsrlq $32, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpmuludq %ymm3, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpaddq %ymm4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX512VL-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX512VL-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX512VL-NEXT:    vpaddq %ymm1, %ymm4, %ymm1
+; AVX512VL-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-FALLBACK-LABEL: vec256_i64_signed_mem_mem:
 ; AVX512BW-FALLBACK:       # %bb.0:
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa (%rsi), %ymm1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
 ; AVX512BW-FALLBACK-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512BW-FALLBACK-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [1,1,1,1]
 ; AVX512BW-FALLBACK-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; AVX512BW-FALLBACK-NEXT:    vpminsq %zmm1, %zmm0, %zmm2
 ; AVX512BW-FALLBACK-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm1
 ; AVX512BW-FALLBACK-NEXT:    vpsubq %ymm2, %ymm1, %ymm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %ymm1, %ymm2
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
-; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %ymm3, %ymm4
-; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm4, %ymm1, %ymm4
-; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm2, %ymm4, %ymm2
-; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %ymm2, %ymm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $1, %ymm1, %ymm2
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $33, %ymm1, %ymm1
 ; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm3, %ymm1, %ymm1
-; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; AVX512BW-FALLBACK-NEXT:    vpsrlq $32, %ymm3, %ymm4
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm4, %ymm2, %ymm4
+; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm1, %ymm4, %ymm1
+; AVX512BW-FALLBACK-NEXT:    vpsllq $32, %ymm1, %ymm1
+; AVX512BW-FALLBACK-NEXT:    vpmuludq %ymm3, %ymm2, %ymm2
 ; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; AVX512BW-FALLBACK-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; AVX512BW-FALLBACK-NEXT:    retq
   %a1 = load <4 x i64>, <4 x i64>* %a1_addr
   %a2 = load <4 x i64>, <4 x i64>* %a2_addr
   %t3 = icmp sgt <4 x i64> %a1, %a2 ; signed
   %t4 = select <4 x i1> %t3, <4 x i64> <i64 -1, i64 -1, i64 -1, i64 -1>, <4 x i64> <i64 1, i64 1, i64 1, i64 1>
   %t5 = select <4 x i1> %t3, <4 x i64> %a2, <4 x i64> %a1
   %t6 = select <4 x i1> %t3, <4 x i64> %a1, <4 x i64> %a2
   %t7 = sub <4 x i64> %t6, %t5

llvm/test/CodeGen/X86/midpoint-int-vec-512.ll

 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
 ; ALL-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2
 ; ALL-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [1,1,1,1,1,1,1,1]
 ; ALL-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; ALL-NEXT:    vpminsq %zmm1, %zmm0, %zmm2
 ; ALL-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm1
 ; ALL-NEXT:    vpsubq %zmm2, %zmm1, %zmm1
-; ALL-NEXT:    vpsrlq $1, %zmm1, %zmm1
-; ALL-NEXT:    vpsrlq $32, %zmm3, %zmm2
-; ALL-NEXT:    vpmuludq %zmm2, %zmm1, %zmm2
-; ALL-NEXT:    vpsrlq $32, %zmm1, %zmm4
-; ALL-NEXT:    vpmuludq %zmm3, %zmm4, %zmm4
-; ALL-NEXT:    vpaddq %zmm4, %zmm2, %zmm2
-; ALL-NEXT:    vpsllq $32, %zmm2, %zmm2
+; ALL-NEXT:    vpsrlq $1, %zmm1, %zmm2
+; ALL-NEXT:    vpsrlq $33, %zmm1, %zmm1
 ; ALL-NEXT:    vpmuludq %zmm3, %zmm1, %zmm1
-; ALL-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
+; ALL-NEXT:    vpsrlq $32, %zmm3, %zmm4
+; ALL-NEXT:    vpmuludq %zmm4, %zmm2, %zmm4
+; ALL-NEXT:    vpaddq %zmm1, %zmm4, %zmm1
+; ALL-NEXT:    vpsllq $32, %zmm1, %zmm1
+; ALL-NEXT:    vpmuludq %zmm3, %zmm2, %zmm2
 ; ALL-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
+; ALL-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %t3 = icmp sgt <8 x i64> %a1, %a2 ; signed
   %t4 = select <8 x i1> %t3, <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
   %t5 = select <8 x i1> %t3, <8 x i64> %a2, <8 x i64> %a1
   %t6 = select <8 x i1> %t3, <8 x i64> %a1, <8 x i64> %a2
   %t7 = sub <8 x i64> %t6, %t5
   %t8 = lshr <8 x i64> %t7, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
   %t9 = mul nsw <8 x i64> %t8, %t4 ; signed
   %a10 = add nsw <8 x i64> %t9, %a1 ; signed
   ret <8 x i64> %a10
 }
 
 define <8 x i64> @vec512_i64_unsigned_reg_reg(<8 x i64> %a1, <8 x i64> %a2) nounwind {
 ; ALL-LABEL: vec512_i64_unsigned_reg_reg:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpcmpnleuq %zmm1, %zmm0, %k1
 ; ALL-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2
 ; ALL-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [1,1,1,1,1,1,1,1]
 ; ALL-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; ALL-NEXT:    vpminuq %zmm1, %zmm0, %zmm2
 ; ALL-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm1
 ; ALL-NEXT:    vpsubq %zmm2, %zmm1, %zmm1
-; ALL-NEXT:    vpsrlq $1, %zmm1, %zmm1
-; ALL-NEXT:    vpsrlq $32, %zmm3, %zmm2
-; ALL-NEXT:    vpmuludq %zmm2, %zmm1, %zmm2
-; ALL-NEXT:    vpsrlq $32, %zmm1, %zmm4
-; ALL-NEXT:    vpmuludq %zmm3, %zmm4, %zmm4
-; ALL-NEXT:    vpaddq %zmm4, %zmm2, %zmm2
-; ALL-NEXT:    vpsllq $32, %zmm2, %zmm2
+; ALL-NEXT:    vpsrlq $1, %zmm1, %zmm2
+; ALL-NEXT:    vpsrlq $33, %zmm1, %zmm1
 ; ALL-NEXT:    vpmuludq %zmm3, %zmm1, %zmm1
-; ALL-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
+; ALL-NEXT:    vpsrlq $32, %zmm3, %zmm4
+; ALL-NEXT:    vpmuludq %zmm4, %zmm2, %zmm4
+; ALL-NEXT:    vpaddq %zmm1, %zmm4, %zmm1
+; ALL-NEXT:    vpsllq $32, %zmm1, %zmm1
+; ALL-NEXT:    vpmuludq %zmm3, %zmm2, %zmm2
 ; ALL-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
+; ALL-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %t3 = icmp ugt <8 x i64> %a1, %a2
   %t4 = select <8 x i1> %t3, <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
   %t5 = select <8 x i1> %t3, <8 x i64> %a2, <8 x i64> %a1
   %t6 = select <8 x i1> %t3, <8 x i64> %a1, <8 x i64> %a2
   %t7 = sub <8 x i64> %t6, %t5
   %t8 = lshr <8 x i64> %t7, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
   %t9 = mul <8 x i64> %t8, %t4

 ; ALL-NEXT:    vmovdqa64 (%rdi), %zmm1
 ; ALL-NEXT:    vpcmpgtq %zmm0, %zmm1, %k1
 ; ALL-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2
 ; ALL-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [1,1,1,1,1,1,1,1]
 ; ALL-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; ALL-NEXT:    vpminsq %zmm0, %zmm1, %zmm2
 ; ALL-NEXT:    vpmaxsq %zmm0, %zmm1, %zmm0
 ; ALL-NEXT:    vpsubq %zmm2, %zmm0, %zmm0
-; ALL-NEXT:    vpsrlq $1, %zmm0, %zmm0
-; ALL-NEXT:    vpsrlq $32, %zmm3, %zmm2
-; ALL-NEXT:    vpmuludq %zmm2, %zmm0, %zmm2
-; ALL-NEXT:    vpsrlq $32, %zmm0, %zmm4
-; ALL-NEXT:    vpmuludq %zmm3, %zmm4, %zmm4
-; ALL-NEXT:    vpaddq %zmm4, %zmm2, %zmm2
-; ALL-NEXT:    vpsllq $32, %zmm2, %zmm2
+; ALL-NEXT:    vpsrlq $1, %zmm0, %zmm2
+; ALL-NEXT:    vpsrlq $33, %zmm0, %zmm0
 ; ALL-NEXT:    vpmuludq %zmm3, %zmm0, %zmm0
-; ALL-NEXT:    vpaddq %zmm1, %zmm2, %zmm1
+; ALL-NEXT:    vpsrlq $32, %zmm3, %zmm4
+; ALL-NEXT:    vpmuludq %zmm4, %zmm2, %zmm4
+; ALL-NEXT:    vpaddq %zmm0, %zmm4, %zmm0
+; ALL-NEXT:    vpsllq $32, %zmm0, %zmm0
+; ALL-NEXT:    vpmuludq %zmm3, %zmm2, %zmm2
 ; ALL-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
+; ALL-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %a1 = load <8 x i64>, <8 x i64>* %a1_addr
   %t3 = icmp sgt <8 x i64> %a1, %a2 ; signed
   %t4 = select <8 x i1> %t3, <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
   %t5 = select <8 x i1> %t3, <8 x i64> %a2, <8 x i64> %a1
   %t6 = select <8 x i1> %t3, <8 x i64> %a1, <8 x i64> %a2
   %t7 = sub <8 x i64> %t6, %t5
   %t8 = lshr <8 x i64> %t7, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
   %t9 = mul nsw <8 x i64> %t8, %t4 ; signed
   %a10 = add nsw <8 x i64> %t9, %a1 ; signed
   ret <8 x i64> %a10
 }
 
 define <8 x i64> @vec512_i64_signed_reg_mem(<8 x i64> %a1, <8 x i64>* %a2_addr) nounwind {
 ; ALL-LABEL: vec512_i64_signed_reg_mem:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vmovdqa64 (%rdi), %zmm1
 ; ALL-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
 ; ALL-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2
 ; ALL-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [1,1,1,1,1,1,1,1]
 ; ALL-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; ALL-NEXT:    vpminsq %zmm1, %zmm0, %zmm2
 ; ALL-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm1
 ; ALL-NEXT:    vpsubq %zmm2, %zmm1, %zmm1
-; ALL-NEXT:    vpsrlq $1, %zmm1, %zmm1
-; ALL-NEXT:    vpsrlq $32, %zmm3, %zmm2
-; ALL-NEXT:    vpmuludq %zmm2, %zmm1, %zmm2
-; ALL-NEXT:    vpsrlq $32, %zmm1, %zmm4
-; ALL-NEXT:    vpmuludq %zmm3, %zmm4, %zmm4
-; ALL-NEXT:    vpaddq %zmm4, %zmm2, %zmm2
-; ALL-NEXT:    vpsllq $32, %zmm2, %zmm2
+; ALL-NEXT:    vpsrlq $1, %zmm1, %zmm2
+; ALL-NEXT:    vpsrlq $33, %zmm1, %zmm1
 ; ALL-NEXT:    vpmuludq %zmm3, %zmm1, %zmm1
-; ALL-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
+; ALL-NEXT:    vpsrlq $32, %zmm3, %zmm4
+; ALL-NEXT:    vpmuludq %zmm4, %zmm2, %zmm4
+; ALL-NEXT:    vpaddq %zmm1, %zmm4, %zmm1
+; ALL-NEXT:    vpsllq $32, %zmm1, %zmm1
+; ALL-NEXT:    vpmuludq %zmm3, %zmm2, %zmm2
 ; ALL-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
+; ALL-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %a2 = load <8 x i64>, <8 x i64>* %a2_addr
   %t3 = icmp sgt <8 x i64> %a1, %a2 ; signed
   %t4 = select <8 x i1> %t3, <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
   %t5 = select <8 x i1> %t3, <8 x i64> %a2, <8 x i64> %a1
   %t6 = select <8 x i1> %t3, <8 x i64> %a1, <8 x i64> %a2
   %t7 = sub <8 x i64> %t6, %t5
   %t8 = lshr <8 x i64> %t7, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>

 ; ALL-NEXT:    vmovdqa64 (%rsi), %zmm1
 ; ALL-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
 ; ALL-NEXT:    vpternlogd $255, %zmm2, %zmm2, %zmm2
 ; ALL-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [1,1,1,1,1,1,1,1]
 ; ALL-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
 ; ALL-NEXT:    vpminsq %zmm1, %zmm0, %zmm2
 ; ALL-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm1
 ; ALL-NEXT:    vpsubq %zmm2, %zmm1, %zmm1
-; ALL-NEXT:    vpsrlq $1, %zmm1, %zmm1
-; ALL-NEXT:    vpsrlq $32, %zmm3, %zmm2
-; ALL-NEXT:    vpmuludq %zmm2, %zmm1, %zmm2
-; ALL-NEXT:    vpsrlq $32, %zmm1, %zmm4
-; ALL-NEXT:    vpmuludq %zmm3, %zmm4, %zmm4
-; ALL-NEXT:    vpaddq %zmm4, %zmm2, %zmm2
-; ALL-NEXT:    vpsllq $32, %zmm2, %zmm2
+; ALL-NEXT:    vpsrlq $1, %zmm1, %zmm2
+; ALL-NEXT:    vpsrlq $33, %zmm1, %zmm1
 ; ALL-NEXT:    vpmuludq %zmm3, %zmm1, %zmm1
-; ALL-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
+; ALL-NEXT:    vpsrlq $32, %zmm3, %zmm4
+; ALL-NEXT:    vpmuludq %zmm4, %zmm2, %zmm4
+; ALL-NEXT:    vpaddq %zmm1, %zmm4, %zmm1
+; ALL-NEXT:    vpsllq $32, %zmm1, %zmm1
+; ALL-NEXT:    vpmuludq %zmm3, %zmm2, %zmm2
 ; ALL-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
+; ALL-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
 ; ALL-NEXT:    retq
   %a1 = load <8 x i64>, <8 x i64>* %a1_addr
   %a2 = load <8 x i64>, <8 x i64>* %a2_addr
   %t3 = icmp sgt <8 x i64> %a1, %a2 ; signed
   %t4 = select <8 x i1> %t3, <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>, <8 x i64> <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
   %t5 = select <8 x i1> %t3, <8 x i64> %a2, <8 x i64> %a1
   %t6 = select <8 x i1> %t3, <8 x i64> %a1, <8 x i64> %a2
   %t7 = sub <8 x i64> %t6, %t5