[x86] try to form broadcast before widening shuffle elements

[spatel]

Summary

I noticed that we weren't generating broadcasts as much I thought we would with D54271, and this is part of the problem.

Widening the shuffle elements means adding bitcasts and hiding the relationship between a splatted scalar and the vector. If we can form a broadcast, do that before going through the rest of the shuffle lowering because broadcasts should be cheap and can often be load-folded.

[RKSimon]

LGTM - Generally we've tried to avoid lowering in lowerVectorShuffle directly (although lowerVectorShuffleWithVPMOV seems to have gotten in...) - but given the nature of broadcasts this looks OK

llvm/trunk/lib/Target/X86/X86ISelLowering.cpp

static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget &Subtarget,
 
   // Try to collapse shuffles into using a vector type with fewer elements but
   // wider element types. We cap this to not form integers or floating point
   // elements wider than 64 bits, but it might be interesting to form i128
   // integers to handle flipping the low and high halves of AVX 256-bit vectors.
   SmallVector<int, 16> WidenedMask;
   if (VT.getScalarSizeInBits() < 64 && !Is1BitVector &&
       canWidenShuffleElements(ZeroableMask, WidenedMask)) {
+    // Shuffle mask widening should not interfere with a broadcast opportunity
+    // by obfuscating the operands with bitcasts.
+    // TODO: Avoid lowering directly from this top-level function: make this
+    // a query (canLowerAsBroadcast) and defer lowering to the type-based calls.
+    if (SDValue Broadcast =
+            lowerVectorShuffleAsBroadcast(DL, VT, V1, V2, Mask, Subtarget, DAG))
+      return Broadcast;
+
     MVT NewEltVT = VT.isFloatingPoint()
                        ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2)
                        : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2);
     int NewNumElts = NumElements / 2;
     MVT NewVT = MVT::getVectorVT(NewEltVT, NewNumElts);
     // Make sure that the new vector type is legal. For example, v2f64 isn't
     // legal on SSE1.
     if (DAG.getTargetLoweringInfo().isTypeLegal(NewVT)) {

llvm/trunk/test/CodeGen/X86/haddsub-undef.ll

 ;
 ; AVX1-FAST-LABEL: add_ps_007_2:
 ; AVX1-FAST:       # %bb.0:
 ; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: add_ps_007_2:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vmovddup {{.*#+}} xmm1 = xmm0[0,0]
+; AVX2-SLOW-NEXT:    vbroadcastss %xmm0, %xmm1
 ; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
 ; AVX2-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; AVX2-SLOW-NEXT:    retq
 ;
 ; AVX2-FAST-LABEL: add_ps_007_2:
 ; AVX2-FAST:       # %bb.0:
 ; AVX2-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
 ; AVX2-FAST-NEXT:    retq

 ; AVX1-FAST-LABEL: add_ps_018:
 ; AVX1-FAST:       # %bb.0:
 ; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
 ; AVX1-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX1-FAST-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: add_ps_018:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vmovddup {{.*#+}} xmm1 = xmm0[0,0]
+; AVX2-SLOW-NEXT:    vbroadcastss %xmm0, %xmm1
 ; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
 ; AVX2-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX2-SLOW-NEXT:    retq
 ;
 ; AVX2-FAST-LABEL: add_ps_018:
 ; AVX2-FAST:       # %bb.0:
 ; AVX2-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0

llvm/trunk/test/CodeGen/X86/insert-into-constant-vector.ll

 ;
 ; X64SSE4-LABEL: elt6_v8f32:
 ; X64SSE4:       # %bb.0:
 ; X64SSE4-NEXT:    movaps {{.*#+}} xmm1 = <4.0E+0,5.0E+0,u,7.0E+0>
 ; X64SSE4-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0,1],xmm0[0],xmm1[3]
 ; X64SSE4-NEXT:    movaps {{.*#+}} xmm0 = [4.2E+1,1.0E+0,2.0E+0,3.0E+0]
 ; X64SSE4-NEXT:    retq
 ;
-; X32AVX1-LABEL: elt6_v8f32:
-; X32AVX1:       # %bb.0:
-; X32AVX1-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32AVX1-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
-; X32AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32AVX1-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4,5],ymm0[6],mem[7]
-; X32AVX1-NEXT:    retl
+; X32AVX-LABEL: elt6_v8f32:
+; X32AVX:       # %bb.0:
+; X32AVX-NEXT:    vbroadcastss {{[0-9]+}}(%esp), %ymm0
+; X32AVX-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4,5],ymm0[6],mem[7]
+; X32AVX-NEXT:    retl
 ;
 ; X64AVX1-LABEL: elt6_v8f32:
 ; X64AVX1:       # %bb.0:
 ; X64AVX1-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; X64AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X64AVX1-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4,5],ymm0[6],mem[7]
 ; X64AVX1-NEXT:    retq
 ;
-; X32AVX2-LABEL: elt6_v8f32:
-; X32AVX2:       # %bb.0:
-; X32AVX2-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32AVX2-NEXT:    vbroadcastsd %xmm0, %ymm0
-; X32AVX2-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4,5],ymm0[6],mem[7]
-; X32AVX2-NEXT:    retl
-;
 ; X64AVX2-LABEL: elt6_v8f32:
 ; X64AVX2:       # %bb.0:
-; X64AVX2-NEXT:    vbroadcastsd %xmm0, %ymm0
+; X64AVX2-NEXT:    vbroadcastss %xmm0, %ymm0
 ; X64AVX2-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4,5],ymm0[6],mem[7]
 ; X64AVX2-NEXT:    retq
 ;
-; X32AVX512F-LABEL: elt6_v8f32:
-; X32AVX512F:       # %bb.0:
-; X32AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32AVX512F-NEXT:    vbroadcastsd %xmm0, %ymm0
-; X32AVX512F-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4,5],ymm0[6],mem[7]
-; X32AVX512F-NEXT:    retl
-;
 ; X64AVX512F-LABEL: elt6_v8f32:
 ; X64AVX512F:       # %bb.0:
-; X64AVX512F-NEXT:    vbroadcastsd %xmm0, %ymm0
+; X64AVX512F-NEXT:    vbroadcastss %xmm0, %ymm0
 ; X64AVX512F-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4,5],ymm0[6],mem[7]
 ; X64AVX512F-NEXT:    retq
    %ins = insertelement <8 x float> <float 42.0, float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0>, float %x, i32 6
    ret <8 x float> %ins
 }
 
 define <8 x i64> @elt5_v8i64(i64 %x) {
 ; X32SSE-LABEL: elt5_v8i64:

llvm/trunk/test/CodeGen/X86/shuffle-of-splat-multiuses.ll

; AVX2-NEXT:    retq
   %res1 = shufflevector <4 x i32> %res, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
   store <4 x i32> %res, <4 x i32>* %p
   ret <4 x i32> %res1
 }
 
 define <4 x i32> @undef_splatmask5(<4 x i32> %v, <4 x i32>* %p) nounwind {
 ; AVX2-LABEL: undef_splatmask5:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastd %xmm0, %xmm1
-; AVX2-NEXT:    vpbroadcastq %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa %xmm0, (%rdi)
-; AVX2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX2-NEXT:    vbroadcastss %xmm0, %xmm0
+; AVX2-NEXT:    vmovaps %xmm0, (%rdi)
 ; AVX2-NEXT:    retq
   %res = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 undef>
   %res1 = shufflevector <4 x i32> %res, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 undef, i32 3>
   store <4 x i32> %res, <4 x i32>* %p
   ret <4 x i32> %res1
 }

llvm/trunk/test/CodeGen/X86/vector-shuffle-128-v4.ll

 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,1,0,1]
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,2,2]
 ; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2OR512VL-LABEL: shuffle_v4i32_0142:
 ; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX2OR512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,2]
-; AVX2OR512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3]
+; AVX2OR512VL-NEXT:    vbroadcastss %xmm1, %xmm1
+; AVX2OR512VL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,2,2]
+; AVX2OR512VL-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3]
 ; AVX2OR512VL-NEXT:    retq
   %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 0, i32 1, i32 4, i32 2>
   ret <4 x i32> %shuffle
 }
 define <4 x i32> @shuffle_v4i32_0412(<4 x i32> %a, <4 x i32> %b) {
 ; SSE2-LABEL: shuffle_v4i32_0412:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,0],xmm0[0,0]

llvm/trunk/test/CodeGen/X86/vector-shuffle-combining.ll

 ;
 ; AVX1-LABEL: combine_nested_undef_test12:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: combine_nested_undef_test12:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq %xmm0, %xmm0
+; AVX2-NEXT:    vbroadcastss %xmm0, %xmm0
 ; AVX2-NEXT:    retq
   %1 = shufflevector <4 x i32> %A, <4 x i32> %B, <4 x i32> <i32 0, i32 0, i32 2, i32 4>
   %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <4 x i32> <i32 1, i32 4, i32 0, i32 4>
   ret <4 x i32> %2
 }
 
 ; The following pair of shuffles is folded into vector %A.
 define <4 x i32> @combine_nested_undef_test13(<4 x i32> %A, <4 x i32> %B) {