[X86] Freeze shl(x,1) -> add(x,x) vector fold (PR50468)

[RKSimon]

Summary

Vector fold shl(x,1) ->add(freeze(x),freeze(x)) to avoid the undef issues identified in PR50468

[lebedev.ri]

What about https://github.com/llvm/llvm-project/blob/6f7f5b54c81be59ec7876649d1f9aa6b104658ec/llvm/lib/Target/X86/X86InstrCompiler.td#L1761-L1770 ?

[RKSimon]

In D106675#2962249, @lebedev.ri wrote:

What about https://github.com/llvm/llvm-project/blob/6f7f5b54c81be59ec7876649d1f9aa6b104658ec/llvm/lib/Target/X86/X86InstrCompiler.td#L1761-L1770 ?

Cheers - I've added this link to https://bugs.llvm.org/show_bug.cgi?id=50468 for reference

[RKSimon]

Rebase to see how close we are now - we're still missing a DAG equivalent of pushFreezeToPreventPoisonFromPropagating to stop freeze nodes obscuring some folds.

[RKSimon]

rebase (still WIP)

[RKSimon]

rebase (still WIP)

[RKSimon]

Move the (shl V, 1) -> (add (freeze V),(freeze V)) fold inside LowerShiftByScalarImmediate so it's only attempted on legal types.

With the recently improved freeze combines this appears to fix most of the regressions we were seeing.

[RKSimon]

llvm/test/CodeGen/X86/oddsubvector.ll

232–233

We've lost a INSERT_VECTOR_ELT -> (dependency breaking) SCALAR_TO_VECTOR optimization - still investigating this.

[RKSimon]

rebase - ready for review

[spatel]

LGTM

llvm/lib/Target/X86/X86ISelLowering.cpp

   // shl.
   // (shl V, 1) -> add V,V
   if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
     if (auto *N1SplatC = N1BV->getConstantSplatNode()) {
       assert(N0.getValueType().isVector() && "Invalid vector shift type");
       // We shift all of the values by one. In many cases we do not have
       // hardware support for this operation. This is better expressed as an ADD
       // of two values.
-      if (N1SplatC->isOne())
-        return DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, N0);
+      if (N1SplatC->isOne()) {
+        // NOTE: N0 may be undef at run-time, but (shl N0, 1) must be an even
+        // number (LSB must be 0). (add undef, undef) however can be any value.
+        // To make this safe, we must freeze N0 to ensure that register
+        // allocation uses the same register for an undefined value. This
+        // ensures that the result will still be even and preserves the original
+        // semantics.
+        SDLoc DL(N);
+        N0 = DAG.getNode(ISD::FREEZE, DL, VT, N0);
+        return DAG.getNode(ISD::ADD, DL, VT, N0, N0);
+      }
     }
 
   return SDValue();
 }
 
 static SDValue combineShiftRightArithmetic(SDNode *N, SelectionDAG &DAG,
                                            const X86Subtarget &Subtarget) {
   SDValue N0 = N->getOperand(0);

llvm/test/CodeGen/X86/combine-add.ll

 ; SSE-NEXT:    movdqu %xmm2, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: PR52039:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm0 = [10,10,10,10]
 ; AVX1-NEXT:    vpsubd 16(%rdi), %xmm0, %xmm1
 ; AVX1-NEXT:    vpsubd (%rdi), %xmm0, %xmm0
-; AVX1-NEXT:    vpaddd %xmm0, %xmm0, %xmm2
-; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpaddd %xmm1, %xmm1, %xmm3
-; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm3
-; AVX1-NEXT:    vmovdqu %xmm1, 16(%rsi)
-; AVX1-NEXT:    vmovdqu %xmm0, (%rsi)
-; AVX1-NEXT:    vmovdqu %xmm3, 16(%rdi)
-; AVX1-NEXT:    vmovdqu %xmm2, (%rdi)
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm2
+; AVX1-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpaddd %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpaddd %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vmovdqu %ymm2, (%rsi)
+; AVX1-NEXT:    vmovdqu %xmm1, 16(%rdi)
+; AVX1-NEXT:    vmovdqu %xmm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: PR52039:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [10,10,10,10,10,10,10,10]
 ; AVX2-NEXT:    vpsubd (%rdi), %ymm0, %ymm0
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [3,3,3,3,3,3,3,3]
 ; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm1

llvm/test/CodeGen/X86/oddsubvector.ll

 ; SSE42-NEXT:    paddd %xmm0, %xmm0
 ; SSE42-NEXT:    movdqa %xmm0, c+160(%rip)
 ; SSE42-NEXT:    movdqa %xmm2, c+176(%rip)
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: PR42833:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    movl b(%rip), %eax
+; AVX1-NEXT:    vmovdqu c+128(%rip), %ymm0
 ; AVX1-NEXT:    addl c+128(%rip), %eax
-; AVX1-NEXT:    vmovd %eax, %xmm0
-; AVX1-NEXT:    vmovdqa c+128(%rip), %xmm1
-; AVX1-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpaddd %xmm1, %xmm1, %xmm2
-; AVX1-NEXT:    vmovdqa c+144(%rip), %xmm3
-; AVX1-NEXT:    vpaddd %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3,4,5,6,7]
-; AVX1-NEXT:    vmovdqa d+144(%rip), %xmm2
-; AVX1-NEXT:    vpsubd c+144(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vmovd %eax, %xmm1
+; AVX1-NEXT:    vmovdqa c+128(%rip), %xmm2
+; AVX1-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpaddd %xmm0, %xmm0, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7]
+; AVX1-NEXT:    vmovdqa d+144(%rip), %xmm1
+; AVX1-NEXT:    vpsubd c+144(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vmovups %ymm0, c+128(%rip)
-; AVX1-NEXT:    vpinsrd $0, %eax, %xmm1, %xmm0
-; AVX1-NEXT:    vmovdqa d+128(%rip), %xmm1
-; AVX1-NEXT:    vpsubd %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vmovdqa d+176(%rip), %xmm1
-; AVX1-NEXT:    vmovdqa c+176(%rip), %xmm3
-; AVX1-NEXT:    vpsubd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqu c+160(%rip), %ymm0
+; AVX1-NEXT:    vpinsrd $0, %eax, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa d+128(%rip), %xmm3
+; AVX1-NEXT:    vpsubd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vmovdqa d+176(%rip), %xmm3
+; AVX1-NEXT:    vpsubd c+176(%rip), %xmm3, %xmm3
 ; AVX1-NEXT:    vmovdqa d+160(%rip), %xmm4
-; AVX1-NEXT:    vmovdqa c+160(%rip), %xmm5
-; AVX1-NEXT:    vpsubd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vmovdqa %xmm2, d+144(%rip)
+; AVX1-NEXT:    vpsubd c+160(%rip), %xmm4, %xmm4
+; AVX1-NEXT:    vmovdqa %xmm1, d+144(%rip)
 ; AVX1-NEXT:    vmovdqa %xmm4, d+160(%rip)
-; AVX1-NEXT:    vmovdqa %xmm1, d+176(%rip)
-; AVX1-NEXT:    vmovdqa %xmm0, d+128(%rip)
-; AVX1-NEXT:    vpaddd %xmm3, %xmm3, %xmm0
-; AVX1-NEXT:    vpaddd %xmm5, %xmm5, %xmm1
-; AVX1-NEXT:    vmovdqa %xmm1, c+160(%rip)
-; AVX1-NEXT:    vmovdqa %xmm0, c+176(%rip)
+; AVX1-NEXT:    vmovdqa %xmm3, d+176(%rip)
+; AVX1-NEXT:    vmovdqa %xmm2, d+128(%rip)
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpaddd %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, c+160(%rip)
+; AVX1-NEXT:    vmovdqa %xmm1, c+176(%rip)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: PR42833:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    movl b(%rip), %eax
 ; AVX2-NEXT:    vmovdqu c+128(%rip), %ymm0
 ; AVX2-NEXT:    addl c+128(%rip), %eax

 ; AVX512-NEXT:    vpaddd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqu %ymm0, c+160(%rip)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; XOP-LABEL: PR42833:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    movl b(%rip), %eax
+; XOP-NEXT:    vmovdqu c+128(%rip), %ymm0
 ; XOP-NEXT:    addl c+128(%rip), %eax
-; XOP-NEXT:    vmovd %eax, %xmm0
-; XOP-NEXT:    vmovdqa c+128(%rip), %xmm1
-; XOP-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
-; XOP-NEXT:    vpaddd %xmm1, %xmm1, %xmm2
-; XOP-NEXT:    vmovdqa c+144(%rip), %xmm3
-; XOP-NEXT:    vpaddd %xmm3, %xmm3, %xmm3
-; XOP-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; XOP-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3,4,5,6,7]
-; XOP-NEXT:    vmovdqa d+144(%rip), %xmm2
-; XOP-NEXT:    vpsubd c+144(%rip), %xmm2, %xmm2
+; XOP-NEXT:    vmovd %eax, %xmm1
+; XOP-NEXT:    vmovdqa c+128(%rip), %xmm2
+; XOP-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vpaddd %xmm0, %xmm0, %xmm3
+; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; XOP-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; XOP-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; XOP-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3,4,5,6,7]
+; XOP-NEXT:    vmovdqa d+144(%rip), %xmm1
+; XOP-NEXT:    vpsubd c+144(%rip), %xmm1, %xmm1
 ; XOP-NEXT:    vmovups %ymm0, c+128(%rip)
-; XOP-NEXT:    vpinsrd $0, %eax, %xmm1, %xmm0
-; XOP-NEXT:    vmovdqa d+128(%rip), %xmm1
-; XOP-NEXT:    vpsubd %xmm0, %xmm1, %xmm0
-; XOP-NEXT:    vmovdqa d+176(%rip), %xmm1
-; XOP-NEXT:    vmovdqa c+176(%rip), %xmm3
-; XOP-NEXT:    vpsubd %xmm3, %xmm1, %xmm1
+; XOP-NEXT:    vmovdqu c+160(%rip), %ymm0
+; XOP-NEXT:    vpinsrd $0, %eax, %xmm2, %xmm2
+; XOP-NEXT:    vmovdqa d+128(%rip), %xmm3
+; XOP-NEXT:    vpsubd %xmm2, %xmm3, %xmm2
+; XOP-NEXT:    vmovdqa d+176(%rip), %xmm3
+; XOP-NEXT:    vpsubd c+176(%rip), %xmm3, %xmm3
 ; XOP-NEXT:    vmovdqa d+160(%rip), %xmm4
-; XOP-NEXT:    vmovdqa c+160(%rip), %xmm5
-; XOP-NEXT:    vpsubd %xmm5, %xmm4, %xmm4
-; XOP-NEXT:    vmovdqa %xmm2, d+144(%rip)
+; XOP-NEXT:    vpsubd c+160(%rip), %xmm4, %xmm4
+; XOP-NEXT:    vmovdqa %xmm1, d+144(%rip)
 ; XOP-NEXT:    vmovdqa %xmm4, d+160(%rip)
-; XOP-NEXT:    vmovdqa %xmm1, d+176(%rip)
-; XOP-NEXT:    vmovdqa %xmm0, d+128(%rip)
-; XOP-NEXT:    vpaddd %xmm3, %xmm3, %xmm0
-; XOP-NEXT:    vpaddd %xmm5, %xmm5, %xmm1
-; XOP-NEXT:    vmovdqa %xmm1, c+160(%rip)
-; XOP-NEXT:    vmovdqa %xmm0, c+176(%rip)
+; XOP-NEXT:    vmovdqa %xmm3, d+176(%rip)
+; XOP-NEXT:    vmovdqa %xmm2, d+128(%rip)
+; XOP-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; XOP-NEXT:    vpaddd %xmm1, %xmm1, %xmm1
+; XOP-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; XOP-NEXT:    vmovdqa %xmm0, c+160(%rip)
+; XOP-NEXT:    vmovdqa %xmm1, c+176(%rip)
 ; XOP-NEXT:    vzeroupper
 ; XOP-NEXT:    retq
   %1 = load i32, ptr @b, align 4
   %2 = load <8 x i32>, ptr getelementptr inbounds ([49 x i32], ptr @c, i64 0, i64 32), align 16
   %3 = shufflevector <8 x i32> %2, <8 x i32> undef, <16 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   %4 = extractelement <8 x i32> %2, i32 0
   %5 = add i32 %1, %4
   %6 = insertelement <8 x i32> <i32 undef, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, i32 %5, i32 0

llvm/test/CodeGen/X86/rotate_vec.ll

; AVX512-NEXT:    retq
   %3 = ashr <4 x i32> %2, <i32 1, i32 2, i32 3, i32 4>
   %4 = and <4 x i32> %3, <i32 -32768, i32 -65536, i32 -32768, i32 -65536>
   ret <4 x i32> %4
 }
 
 define <4 x i32> @rot_v4i32_mask_ashr1(<4 x i32> %a0) {
 ; XOPAVX1-LABEL: rot_v4i32_mask_ashr1:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpsrad $25, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpshad {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; XOPAVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: rot_v4i32_mask_ashr1:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpsrad $25, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpsravd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: rot_v4i32_mask_ashr1:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrad $25, %xmm0, %xmm0
+; AVX512-NEXT:    vpsravd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %1 = ashr <4 x i32> %a0, <i32 25, i32 26, i32 27, i32 28>
   %2 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %1, <4 x i32> %1, <4 x i32> <i32 1, i32 2, i32 3, i32 4>)
   %3 = shufflevector <4 x i32> %2, <4 x i32> undef, <4 x i32> zeroinitializer
   %4 = ashr <4 x i32> %3, <i32 1, i32 2, i32 3, i32 4>

llvm/test/CodeGen/X86/udiv_fix_sat.ll

 ; X86-NEXT:    pushl %ebp
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    subl $12, %esp
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
 ; X86-NEXT:    xorl %ecx, %ecx
 ; X86-NEXT:    addl %eax, %eax
 ; X86-NEXT:    setb %cl
 ; X86-NEXT:    shldl $31, %eax, %ecx
 ; X86-NEXT:    shll $31, %eax
 ; X86-NEXT:    pushl $0
 ; X86-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NEXT:    pushl %ecx

llvm/test/CodeGen/X86/vector-gep.ll

 
 define <64 x ptr> @AGEP9(ptr %param, <64 x i32> %off) nounwind {
 ; CHECK-LABEL: AGEP9:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pushl %ebp
 ; CHECK-NEXT:    movl %esp, %ebp
 ; CHECK-NEXT:    andl $-32, %esp
 ; CHECK-NEXT:    subl $160, %esp
-; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm3
+; CHECK-NEXT:    vmovdqa 40(%ebp), %ymm3
+; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm4
 ; CHECK-NEXT:    vbroadcastss 12(%ebp), %xmm5
-; CHECK-NEXT:    vpaddd %xmm3, %xmm5, %xmm3
-; CHECK-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    vpaddd %xmm4, %xmm5, %xmm4
+; CHECK-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
 ; CHECK-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm1, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
 ; CHECK-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    vextractf128 $1, %ymm1, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
 ; CHECK-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    vpaddd %xmm2, %xmm2, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
 ; CHECK-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    vextractf128 $1, %ymm2, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
 ; CHECK-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT:    vmovdqa 40(%ebp), %xmm0
-; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm3, %xmm3, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
 ; CHECK-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT:    vmovdqa 56(%ebp), %xmm0
+; CHECK-NEXT:    vextractf128 $1, %ymm3, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
 ; CHECK-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; CHECK-NEXT:    vmovdqa 72(%ebp), %xmm0
+; CHECK-NEXT:    vmovdqa 72(%ebp), %ymm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm1
+; CHECK-NEXT:    vpaddd %xmm1, %xmm5, %xmm1
+; CHECK-NEXT:    vmovdqa %xmm1, (%esp) # 16-byte Spill
+; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
-; CHECK-NEXT:    vmovdqa %xmm0, (%esp) # 16-byte Spill
-; CHECK-NEXT:    vmovdqa 88(%ebp), %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm5, %xmm4
+; CHECK-NEXT:    vmovdqa 104(%ebp), %ymm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm3
+; CHECK-NEXT:    vpaddd %xmm3, %xmm5, %xmm3
+; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm5, %xmm2
-; CHECK-NEXT:    vmovdqa 104(%ebp), %xmm0
-; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    vpaddd %xmm0, %xmm5, %xmm1
-; CHECK-NEXT:    vmovdqa 120(%ebp), %xmm0
+; CHECK-NEXT:    vmovdqa 136(%ebp), %ymm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm7
+; CHECK-NEXT:    vpaddd %xmm7, %xmm5, %xmm7
+; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm5, %xmm0
-; CHECK-NEXT:    vmovdqa 136(%ebp), %xmm6
-; CHECK-NEXT:    vpaddd %xmm6, %xmm6, %xmm6
+; CHECK-NEXT:    vmovdqa 168(%ebp), %ymm1
+; CHECK-NEXT:    vpaddd %xmm1, %xmm1, %xmm6
 ; CHECK-NEXT:    vpaddd %xmm6, %xmm5, %xmm6
-; CHECK-NEXT:    vmovdqa 152(%ebp), %xmm7
-; CHECK-NEXT:    vpaddd %xmm7, %xmm7, %xmm7
-; CHECK-NEXT:    vpaddd %xmm7, %xmm5, %xmm7
-; CHECK-NEXT:    vmovdqa 168(%ebp), %xmm4
-; CHECK-NEXT:    vpaddd %xmm4, %xmm4, %xmm4
-; CHECK-NEXT:    vpaddd %xmm4, %xmm5, %xmm4
-; CHECK-NEXT:    vmovdqa 184(%ebp), %xmm3
-; CHECK-NEXT:    vpaddd %xmm3, %xmm3, %xmm3
-; CHECK-NEXT:    vpaddd %xmm3, %xmm5, %xmm3
+; CHECK-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; CHECK-NEXT:    vpaddd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpaddd %xmm1, %xmm5, %xmm1
 ; CHECK-NEXT:    movl 8(%ebp), %eax
-; CHECK-NEXT:    vmovdqa %xmm3, 240(%eax)
-; CHECK-NEXT:    vmovdqa %xmm4, 224(%eax)
-; CHECK-NEXT:    vmovdqa %xmm7, 208(%eax)
-; CHECK-NEXT:    vmovdqa %xmm6, 192(%eax)
-; CHECK-NEXT:    vmovdqa %xmm0, 176(%eax)
-; CHECK-NEXT:    vmovdqa %xmm1, 160(%eax)
-; CHECK-NEXT:    vmovdqa %xmm2, 144(%eax)
+; CHECK-NEXT:    vmovdqa %xmm1, 240(%eax)
+; CHECK-NEXT:    vmovdqa %xmm6, 224(%eax)
+; CHECK-NEXT:    vmovdqa %xmm0, 208(%eax)
+; CHECK-NEXT:    vmovdqa %xmm7, 192(%eax)
+; CHECK-NEXT:    vmovdqa %xmm2, 176(%eax)
+; CHECK-NEXT:    vmovdqa %xmm3, 160(%eax)
+; CHECK-NEXT:    vmovdqa %xmm4, 144(%eax)
 ; CHECK-NEXT:    vmovaps (%esp), %xmm0 # 16-byte Reload
 ; CHECK-NEXT:    vmovaps %xmm0, 128(%eax)
 ; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
 ; CHECK-NEXT:    vmovaps %xmm0, 112(%eax)
 ; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
 ; CHECK-NEXT:    vmovaps %xmm0, 96(%eax)
 ; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
 ; CHECK-NEXT:    vmovaps %xmm0, 80(%eax)