[X86][SSE] lowerAddSubToHorizontalOp - enable ymm extraction+fold

[RKSimon]

Summary

Limiting scalar hadd/hsub generation to the lowest xmm looks to be unnecessary - we will be extracting one upper xmm whatever, and we can remove a shuffle by using the hop which is inline with what shouldUseHorizontalOp expects to happen anyway.

Testing on btver2 (the main target for fast-hops) shows this is beneficial even for float ops where we have a 'shuffle' to extract the float result:
https://godbolt.org/z/0R-U-K

[spatel]

LGTM

llvm/trunk/lib/Target/X86/X86ISelLowering.cpp

static SDValue lowerAddSubToHorizontalOp(SDValue Op, SelectionDAG &DAG,
   unsigned BitWidth = VecVT.getSizeInBits();
   unsigned NumLanes = BitWidth / 128;
   unsigned NumEltsPerLane = VecVT.getVectorNumElements() / NumLanes;
   assert((BitWidth == 128 || BitWidth == 256 || BitWidth == 512) &&
          "Not expecting illegal vector widths here");
 
   // Creating a 256-bit horizontal op would be wasteful, and there is no 512-bit
   // equivalent, so extract the 256/512-bit source op to 128-bit if we can.
-  // This is free if we're extracting from the bottom lane: ymm/zmm -> xmm.
-  if (NumEltsPerLane <= LExtIndex)
-   return Op;
-
   SDLoc DL(Op);
-  if (BitWidth == 256 || BitWidth == 512)
-    X = extract128BitVector(X, 0, DAG, DL);
+  if (BitWidth == 256 || BitWidth == 512) {
+    unsigned LaneIdx = LExtIndex / NumEltsPerLane;
+    X = extract128BitVector(X, LaneIdx * NumEltsPerLane, DAG, DL);
+    LExtIndex %= NumEltsPerLane;
+  }
 
   // add (extractelt (X, 0), extractelt (X, 1)) --> extractelt (hadd X, X), 0
   // add (extractelt (X, 1), extractelt (X, 0)) --> extractelt (hadd X, X), 0
   // add (extractelt (X, 2), extractelt (X, 3)) --> extractelt (hadd X, X), 1
   // sub (extractelt (X, 0), extractelt (X, 1)) --> extractelt (hsub X, X), 0
   SDValue HOp = DAG.getNode(HOpcode, DL, X.getValueType(), X, X);
   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getSimpleValueType(), HOp,
                      DAG.getIntPtrConstant(LExtIndex / 2, DL));

llvm/trunk/test/CodeGen/X86/haddsub.ll

 ; SSE3-SLOW-NEXT:    retq
 ;
 ; SSE3-FAST-LABEL: extract_extract67_v8f32_fadd_f32:
 ; SSE3-FAST:       # %bb.0:
 ; SSE3-FAST-NEXT:    haddps %xmm1, %xmm1
 ; SSE3-FAST-NEXT:    movshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
 ; SSE3-FAST-NEXT:    retq
 ;
-; AVX-LABEL: extract_extract67_v8f32_fadd_f32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; AVX-NEXT:    vaddss %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX-SLOW-LABEL: extract_extract67_v8f32_fadd_f32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-SLOW-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract67_v8f32_fadd_f32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
   %x0 = extractelement <8 x float> %x, i32 6
   %x1 = extractelement <8 x float> %x, i32 7
   %x01 = fadd float %x0, %x1
   ret float %x01
 }
 
 define float @extract_extract01_v8f32_fadd_f32_commute(<8 x float> %x) {
 ; SSE3-SLOW-LABEL: extract_extract01_v8f32_fadd_f32_commute:

 ; SSE3-SLOW-NEXT:    retq
 ;
 ; SSE3-FAST-LABEL: extract_extract67_v8f32_fadd_f32_commute:
 ; SSE3-FAST:       # %bb.0:
 ; SSE3-FAST-NEXT:    haddps %xmm1, %xmm1
 ; SSE3-FAST-NEXT:    movshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
 ; SSE3-FAST-NEXT:    retq
 ;
-; AVX-LABEL: extract_extract67_v8f32_fadd_f32_commute:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX-SLOW-LABEL: extract_extract67_v8f32_fadd_f32_commute:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract67_v8f32_fadd_f32_commute:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
   %x0 = extractelement <8 x float> %x, i32 6
   %x1 = extractelement <8 x float> %x, i32 7
   %x01 = fadd float %x1, %x0
   ret float %x01
 }
 
 define double @extract_extract01_v4f64_fadd_f64(<4 x double> %x) {
 ; SSE3-SLOW-LABEL: extract_extract01_v4f64_fadd_f64:

 ; SSE3-SLOW-NEXT:    retq
 ;
 ; SSE3-FAST-LABEL: extract_extract23_v4f64_fadd_f64:
 ; SSE3-FAST:       # %bb.0:
 ; SSE3-FAST-NEXT:    movapd %xmm1, %xmm0
 ; SSE3-FAST-NEXT:    haddpd %xmm1, %xmm0
 ; SSE3-FAST-NEXT:    retq
 ;
-; AVX-LABEL: extract_extract23_v4f64_fadd_f64:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX-SLOW-LABEL: extract_extract23_v4f64_fadd_f64:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract23_v4f64_fadd_f64:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
   %x0 = extractelement <4 x double> %x, i32 2
   %x1 = extractelement <4 x double> %x, i32 3
   %x01 = fadd double %x0, %x1
   ret double %x01
 }
 
 define double @extract_extract01_v4f64_fadd_f64_commute(<4 x double> %x) {
 ; SSE3-SLOW-LABEL: extract_extract01_v4f64_fadd_f64_commute:

 ; SSE3-SLOW-NEXT:    retq
 ;
 ; SSE3-FAST-LABEL: extract_extract23_v4f64_fadd_f64_commute:
 ; SSE3-FAST:       # %bb.0:
 ; SSE3-FAST-NEXT:    movapd %xmm1, %xmm0
 ; SSE3-FAST-NEXT:    haddpd %xmm1, %xmm0
 ; SSE3-FAST-NEXT:    retq
 ;
-; AVX-LABEL: extract_extract23_v4f64_fadd_f64_commute:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX-SLOW-LABEL: extract_extract23_v4f64_fadd_f64_commute:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract23_v4f64_fadd_f64_commute:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
   %x0 = extractelement <4 x double> %x, i32 2
   %x1 = extractelement <4 x double> %x, i32 3
   %x01 = fadd double %x1, %x0
   ret double %x01
 }
 
 define float @extract_extract01_v8f32_fsub_f32(<8 x float> %x) {
 ; SSE3-SLOW-LABEL: extract_extract01_v8f32_fsub_f32:

 ; SSE3-SLOW-NEXT:    retq
 ;
 ; SSE3-FAST-LABEL: extract_extract45_v8f32_fsub_f32:
 ; SSE3-FAST:       # %bb.0:
 ; SSE3-FAST-NEXT:    movaps %xmm1, %xmm0
 ; SSE3-FAST-NEXT:    hsubps %xmm1, %xmm0
 ; SSE3-FAST-NEXT:    retq
 ;
-; AVX-LABEL: extract_extract45_v8f32_fsub_f32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; AVX-NEXT:    vsubss %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX-SLOW-LABEL: extract_extract45_v8f32_fsub_f32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract45_v8f32_fsub_f32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX-FAST-NEXT:    vhsubps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
   %x0 = extractelement <8 x float> %x, i32 4
   %x1 = extractelement <8 x float> %x, i32 5
   %x01 = fsub float %x0, %x1
   ret float %x01
 }
 
 ; Negative test...or get hoppy and negate?
 

llvm/trunk/test/CodeGen/X86/phaddsub-extract.ll

 ; AVX1-SLOW-NEXT:    addl %ecx, %eax
 ; AVX1-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-SLOW-NEXT:    vzeroupper
 ; AVX1-SLOW-NEXT:    retq
 ;
 ; AVX1-FAST-LABEL: extract_extract89_v16i16_add_i16:
 ; AVX1-FAST:       # %bb.0:
 ; AVX1-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-FAST-NEXT:    vmovd %xmm0, %ecx
-; AVX1-FAST-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX1-FAST-NEXT:    addl %ecx, %eax
+; AVX1-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
 ; AVX1-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-FAST-NEXT:    vzeroupper
 ; AVX1-FAST-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: extract_extract89_v16i16_add_i16:
 ; AVX2-SLOW:       # %bb.0:
 ; AVX2-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX2-SLOW-NEXT:    vmovd %xmm0, %ecx
 ; AVX2-SLOW-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX2-SLOW-NEXT:    addl %ecx, %eax
 ; AVX2-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-SLOW-NEXT:    vzeroupper
 ; AVX2-SLOW-NEXT:    retq
 ;
 ; AVX2-FAST-LABEL: extract_extract89_v16i16_add_i16:
 ; AVX2-FAST:       # %bb.0:
 ; AVX2-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-FAST-NEXT:    vmovd %xmm0, %ecx
-; AVX2-FAST-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX2-FAST-NEXT:    addl %ecx, %eax
+; AVX2-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vmovd %xmm0, %eax
 ; AVX2-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-FAST-NEXT:    vzeroupper
 ; AVX2-FAST-NEXT:    retq
 ;
 ; AVX512-SLOW-LABEL: extract_extract89_v16i16_add_i16:
 ; AVX512-SLOW:       # %bb.0:
 ; AVX512-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512-SLOW-NEXT:    vmovd %xmm0, %ecx
 ; AVX512-SLOW-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX512-SLOW-NEXT:    addl %ecx, %eax
 ; AVX512-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-SLOW-NEXT:    vzeroupper
 ; AVX512-SLOW-NEXT:    retq
 ;
 ; AVX512-FAST-LABEL: extract_extract89_v16i16_add_i16:
 ; AVX512-FAST:       # %bb.0:
 ; AVX512-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512-FAST-NEXT:    vmovd %xmm0, %ecx
-; AVX512-FAST-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX512-FAST-NEXT:    addl %ecx, %eax
+; AVX512-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX512-FAST-NEXT:    vmovd %xmm0, %eax
 ; AVX512-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-FAST-NEXT:    vzeroupper
 ; AVX512-FAST-NEXT:    retq
   %x0 = extractelement <16 x i16> %x, i32 8
   %x1 = extractelement <16 x i16> %x, i32 9
   %x01 = add i16 %x0, %x1
   ret i16 %x01
 }

 ; AVX1-SLOW-NEXT:    addl %ecx, %eax
 ; AVX1-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-SLOW-NEXT:    vzeroupper
 ; AVX1-SLOW-NEXT:    retq
 ;
 ; AVX1-FAST-LABEL: extract_extract89_v16i16_add_i16_commute:
 ; AVX1-FAST:       # %bb.0:
 ; AVX1-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-FAST-NEXT:    vmovd %xmm0, %ecx
-; AVX1-FAST-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX1-FAST-NEXT:    addl %ecx, %eax
+; AVX1-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
 ; AVX1-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-FAST-NEXT:    vzeroupper
 ; AVX1-FAST-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: extract_extract89_v16i16_add_i16_commute:
 ; AVX2-SLOW:       # %bb.0:
 ; AVX2-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX2-SLOW-NEXT:    vmovd %xmm0, %ecx
 ; AVX2-SLOW-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX2-SLOW-NEXT:    addl %ecx, %eax
 ; AVX2-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-SLOW-NEXT:    vzeroupper
 ; AVX2-SLOW-NEXT:    retq
 ;
 ; AVX2-FAST-LABEL: extract_extract89_v16i16_add_i16_commute:
 ; AVX2-FAST:       # %bb.0:
 ; AVX2-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-FAST-NEXT:    vmovd %xmm0, %ecx
-; AVX2-FAST-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX2-FAST-NEXT:    addl %ecx, %eax
+; AVX2-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vmovd %xmm0, %eax
 ; AVX2-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-FAST-NEXT:    vzeroupper
 ; AVX2-FAST-NEXT:    retq
 ;
 ; AVX512-SLOW-LABEL: extract_extract89_v16i16_add_i16_commute:
 ; AVX512-SLOW:       # %bb.0:
 ; AVX512-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512-SLOW-NEXT:    vmovd %xmm0, %ecx
 ; AVX512-SLOW-NEXT:    vpextrw $1, %xmm0, %eax
 ; AVX512-SLOW-NEXT:    addl %ecx, %eax
 ; AVX512-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-SLOW-NEXT:    vzeroupper
 ; AVX512-SLOW-NEXT:    retq
 ;
 ; AVX512-FAST-LABEL: extract_extract89_v16i16_add_i16_commute:
 ; AVX512-FAST:       # %bb.0:
 ; AVX512-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512-FAST-NEXT:    vmovd %xmm0, %ecx
-; AVX512-FAST-NEXT:    vpextrw $1, %xmm0, %eax
-; AVX512-FAST-NEXT:    addl %ecx, %eax
+; AVX512-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX512-FAST-NEXT:    vmovd %xmm0, %eax
 ; AVX512-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-FAST-NEXT:    vzeroupper
 ; AVX512-FAST-NEXT:    retq
   %x0 = extractelement <16 x i16> %x, i32 8
   %x1 = extractelement <16 x i16> %x, i32 9
   %x01 = add i16 %x1, %x0
   ret i16 %x01
 }