[X86] Widen i16 shuffle masks if vector width < 512 even with BWI

[goldstein.w.n]

Summary

{v}blend{d|ps|pd} is preferable to {v}blendw so widen so that we
can match it.

[RKSimon]

lowerVECTOR_SHUFFLE should always widen a shuffle as much as possible - my guess is canCombineAsMaskOperation is blocking it on AVX512BW targets - maybe address it there instead?

[goldstein.w.n]

move logic to cancombine

[goldstein.w.n]

In D143789#4120191, @RKSimon wrote:

lowerVECTOR_SHUFFLE should always widen a shuffle as much as possible - my guess is canCombineAsMaskOperation is blocking it on AVX512BW targets - maybe address it there instead?

Done in V2.

[RKSimon]

LGTM

[goldstein.w.n]

Rebase

[goldstein.w.n]

Rebase

[goldstein.w.n]

Rebase

[goldstein.w.n]

Rebase

[goldstein.w.n]

Rebase

[goldstein.w.n]

Rebase

[goldstein.w.n]

Rebase

[goldstein.w.n]

Rebase

[goldstein.w.n]

Rebase

[goldstein.w.n]

Rebase

[goldstein.w.n]

Rebase

[goldstein.w.n]

Rebase

[goldstein.w.n]

Rebase

llvm/lib/Target/X86/X86ISelLowering.cpp

static bool canCombineAsMaskOperation(SDValue V1, SDValue V2,
                                       const X86Subtarget &Subtarget) {
   if (!Subtarget.hasAVX512())
     return false;
 
   MVT VT = V1.getSimpleValueType().getScalarType();
   if ((VT == MVT::i16 || VT == MVT::i8) && !Subtarget.hasBWI())
     return false;
 
-  // i8 is better to be widen to i16, because there is PBLENDW for vXi16
-  // when the vector bit size is 128 or 256.
-  if (VT == MVT::i8 && V1.getSimpleValueType().getSizeInBits() < 512)
+  // If vec width < 512, widen i8/i16 even with BWI as blendd/blendps/blendpd
+  // are preferable to blendw/blendvb/masked-mov.
+  if ((VT == MVT::i16 || VT == MVT::i8) &&
+      V1.getSimpleValueType().getSizeInBits() < 512)
     return false;
 
   auto HasMaskOperation = [&](SDValue V) {
     // TODO: Currently we only check limited opcode. We probably extend
     // it to all binary operation by checking TLI.isBinOp().
     switch (V->getOpcode()) {
     default:
       return false;

llvm/test/CodeGen/X86/combine-sdiv.ll

 ;
 ; AVX1-LABEL: combine_vec_sdiv_nonuniform7:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsubw %xmm0, %xmm1, %xmm1
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: combine_vec_sdiv_nonuniform7:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpsubw %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; AVX2-NEXT:    retq
-;
-; AVX512F-LABEL: combine_vec_sdiv_nonuniform7:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpsubw %xmm0, %xmm1, %xmm1
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: combine_vec_sdiv_nonuniform7:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsubw %xmm0, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
-; AVX512BW-NEXT:    retq
+; AVX2ORLATER-LABEL: combine_vec_sdiv_nonuniform7:
+; AVX2ORLATER:       # %bb.0:
+; AVX2ORLATER-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2ORLATER-NEXT:    vpsubw %xmm0, %xmm1, %xmm1
+; AVX2ORLATER-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX2ORLATER-NEXT:    retq
 ;
 ; XOP-LABEL: combine_vec_sdiv_nonuniform7:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; XOP-NEXT:    vpsubw %xmm0, %xmm1, %xmm1
 ; XOP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
 ; XOP-NEXT:    retq
   %1 = sdiv <8 x i16> %x, <i16 -1, i16 -1, i16 -1, i16 -1, i16 1, i16 1, i16 1, i16 1>

llvm/test/CodeGen/X86/shuffle-blendw.ll

 ; X64-AVX2:       # %bb.0:
 ; X64-AVX2-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
 ; X64-AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
 ; X64-AVX2-NEXT:    retq
 ;
 ; X86-AVX512-LABEL: blendw_to_blendd_32:
 ; X86-AVX512:       # %bb.0:
 ; X86-AVX512-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
-; X86-AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7],ymm0[8,9],ymm1[10,11],ymm0[12,13],ymm1[14,15]
+; X86-AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
 ; X86-AVX512-NEXT:    retl
 ;
 ; X64-AVX512-LABEL: blendw_to_blendd_32:
 ; X64-AVX512:       # %bb.0:
 ; X64-AVX512-NEXT:    vpaddw %ymm2, %ymm0, %ymm0
-; X64-AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7],ymm0[8,9],ymm1[10,11],ymm0[12,13],ymm1[14,15]
+; X64-AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
 ; X64-AVX512-NEXT:    retq
   %x1 = add <16 x i16> %x, %z
   %shuffle = shufflevector <16 x i16> %x1, <16 x i16> %y, <16 x i32> <i32 0, i32 1, i32 18, i32 19, i32 4, i32 5, i32 22, i32 23, i32 8, i32 9, i32 26, i32 27, i32 12, i32 13, i32 30, i32 31>
   ret <16 x i16> %shuffle
 }
 
 define <8 x i16> @blendw_to_blendd_16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %z) nounwind {
 ; X86-SSE41-LABEL: blendw_to_blendd_16:

 ; X64-AVX2:       # %bb.0:
 ; X64-AVX2-NEXT:    vpaddw %xmm2, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
 ; X64-AVX2-NEXT:    retq
 ;
 ; X86-AVX512-LABEL: blendw_to_blendd_16:
 ; X86-AVX512:       # %bb.0:
 ; X86-AVX512-NEXT:    vpaddw %xmm2, %xmm0, %xmm0
-; X86-AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; X86-AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
 ; X86-AVX512-NEXT:    retl
 ;
 ; X64-AVX512-LABEL: blendw_to_blendd_16:
 ; X64-AVX512:       # %bb.0:
 ; X64-AVX512-NEXT:    vpaddw %xmm2, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4,5,6,7]
+; X64-AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
 ; X64-AVX512-NEXT:    retq
   %x1 = add <8 x i16> %x, %z
   %shuffle = shufflevector <8 x i16> %x1, <8 x i16> %y, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
   ret <8 x i16> %shuffle
 }
 
 define <16 x i16> @blendw_to_blendd_fail_32(<16 x i16> %x, <16 x i16> %y, <16 x i16> %z) nounwind {
 ; X86-SSE41-LABEL: blendw_to_blendd_fail_32: