[X86] Prefer `vpternlog` instead of `blendv` for `vselect` on masks.

[goldstein.w.n]

Summary

This rarely comes up because most vselect are lowered with actually
avx512 mask instructions, but is an improvement in the rare cases.

[craig.topper]

llvm/lib/Target/X86/X86ISelDAGToDAG.cpp

1037

Update comment

[craig.topper]

llvm/lib/Target/X86/X86ISelDAGToDAG.cpp

1044

Why is the PCMPGT check needed?

[pengfei]

llvm/lib/Target/X86/X86ISelDAGToDAG.cpp

1043

It's enough to check single hasVLX.

[RKSimon]

llvm/lib/Target/X86/X86ISelDAGToDAG.cpp

1044

Wouldn't a numsignbits check be better than PCMPGT?

[goldstein.w.n]

llvm/lib/Target/X86/X86ISelDAGToDAG.cpp

1044

Why is the PCMPGT check needed?

blendv only uses the sign-bit of the control. For vpternlog to be a replacement, the control needs to be in mask form (elements either -1/0). pcmpgt is just a check for "is the control in mask form". Maybe (sext (setcc ...)) would also work but didn't see any codegen changes from it.

1044

What is numsignbits check?

[craig.topper]

llvm/lib/Target/X86/X86ISelDAGToDAG.cpp

1044

CurDAG->computeNumSignBits or CurDAG->ComputeMaxSignificantBits

1044

I think an ISD::VSELECT is technically malformed if the elements of the condition aren't 0/-1, but we don't have a good way of proving we didn't mess that up.

[goldstein.w.n]

llvm/lib/Target/X86/X86ISelDAGToDAG.cpp

1044

I think an ISD::VSELECT is technically malformed if the elements of the condition aren't 0/-1, but we don't have a good way of proving we didn't mess that up.

Ill use CurDAG->computeNumSignBits, if we are being correct that should prove it.

[goldstein.w.n]

Use ComputeNumSignBits instead of PCMPGT check. Fix comment

[RKSimon]

llvm/test/CodeGen/X86/vselect-pcmp.ll

37 ↗

(On Diff #502256)

AVX512F basically means knights landing - and even though you'd have to use the zmm variant - vpternlogq is a LOT faster than vpblendvb on KNL

[goldstein.w.n]

llvm/test/CodeGen/X86/vselect-pcmp.ll

37 ↗

(On Diff #502256)

AVX512F basically means knights landing - and even though you'd have to use the zmm variant - vpternlogq is a LOT faster than vpblendvb on KNL

Could maybe see for ymm->zmm, but xmm->zmm will then req a vzeroupper, will require a stall for core to prepare zmm usage (if no zmm around), and increase license.

Also could potentially be dangerous if its SSE encoding around it.

[RKSimon]

LGTM - cheers

[goldstein.w.n]

Rebase

llvm/lib/Target/X86/X86ISelDAGToDAG.cpp

case X86ISD::VBROADCAST_LOAD: {
         ++I;
         MadeChange = true;
         continue;
       }
 
       break;
     }
     case ISD::VSELECT: {
       // Replace VSELECT with non-mask conditions with with BLENDV.
       if (N->getOperand(0).getValueType().getVectorElementType() == MVT::i1)
         break;
 
       assert(Subtarget->hasSSE41() && "Expected SSE4.1 support!");
-      SDValue Blendv =
-          CurDAG->getNode(X86ISD::BLENDV, SDLoc(N), N->getValueType(0),
-                          N->getOperand(0), N->getOperand(1), N->getOperand(2));
+      SDValue R;
+      if (Subtarget->hasVLX() && Subtarget->hasAVX512() &&
+          N->getOperand(0)->getOpcode() == X86ISD::PCMPGT) {
+        R = CurDAG->getNode(X86ISD::VPTERNLOG, SDLoc(N), N->getValueType(0),
+                            N->getOperand(0), N->getOperand(1), N->getOperand(2),
+                            CurDAG->getTargetConstant(0xCA, SDLoc(N), MVT::i8));
+      } else {
+        R = CurDAG->getNode(X86ISD::BLENDV, SDLoc(N), N->getValueType(0),
+                            N->getOperand(0), N->getOperand(1),
+                            N->getOperand(2));
+      }
       --I;
-      CurDAG->ReplaceAllUsesWith(N, Blendv.getNode());
+      CurDAG->ReplaceAllUsesWith(N, R.getNode());
       ++I;
       MadeChange = true;
       continue;
     }
     case ISD::FP_ROUND:
     case ISD::STRICT_FP_ROUND:
     case ISD::FP_TO_SINT:
     case ISD::FP_TO_UINT:

llvm/test/CodeGen/X86/abds-vector-128.ll

 ; AVX2-NEXT:    vpblendvb %xmm2, %xmm3, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: abd_cmp_v16i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpsubb %xmm1, %xmm0, %xmm2
 ; AVX512-NEXT:    vpsubb %xmm0, %xmm1, %xmm3
 ; AVX512-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpblendvb %xmm0, %xmm2, %xmm3, %xmm0
+; AVX512-NEXT:    vpternlogq $202, %xmm3, %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %cmp = icmp sgt <16 x i8> %a, %b
   %ab = sub <16 x i8> %a, %b
   %ba = sub <16 x i8> %b, %a
   %sel = select <16 x i1> %cmp, <16 x i8> %ab, <16 x i8> %ba
   ret <16 x i8> %sel
 }
 

llvm/test/CodeGen/X86/abds-vector-256.ll

 ; AVX2-NEXT:    vpblendvb %ymm2, %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: abd_cmp_v32i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpsubb %ymm1, %ymm0, %ymm2
 ; AVX512-NEXT:    vpsubb %ymm0, %ymm1, %ymm3
 ; AVX512-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpblendvb %ymm0, %ymm2, %ymm3, %ymm0
+; AVX512-NEXT:    vpternlogq $202, %ymm3, %ymm2, %ymm0
 ; AVX512-NEXT:    retq
   %cmp = icmp sgt <32 x i8> %a, %b
   %ab = sub <32 x i8> %a, %b
   %ba = sub <32 x i8> %b, %a
   %sel = select <32 x i1> %cmp, <32 x i8> %ab, <32 x i8> %ba
   ret <32 x i8> %sel
 }
 

llvm/test/CodeGen/X86/var-permute-256.ll

 ; AVX512-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
 ; AVX512-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: var_shuffle_v16i16:
 ; AVX512VLDQ:       # %bb.0:
 ; AVX512VLDQ-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
 ; AVX512VLDQ-NEXT:    vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512VLDQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,2,3]
+; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm2
 ; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512VLDQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VLDQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm3
+; AVX512VLDQ-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
+; AVX512VLDQ-NEXT:    vpternlogq $202, %ymm2, %ymm3, %ymm0
 ; AVX512VLDQ-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_shuffle_v16i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpermw %ymm0, %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; VLVBMI-LABEL: var_shuffle_v16i16:

 ; AVX512-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: var_shuffle_v32i8:
 ; AVX512VLDQ:       # %bb.0:
 ; AVX512VLDQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,2,3]
 ; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm2
 ; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512VLDQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm3
+; AVX512VLDQ-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
+; AVX512VLDQ-NEXT:    vpternlogq $202, %ymm3, %ymm2, %ymm0
 ; AVX512VLDQ-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_shuffle_v32i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm2
 ; AVX512VLBW-NEXT:    vpshufb %ymm1, %ymm2, %ymm2
 ; AVX512VLBW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
 ; AVX512VLBW-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %k1

 ; AVX512-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: var_shuffle_v16i16_from_v8i16:
 ; AVX512VLDQ:       # %bb.0:
 ; AVX512VLDQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX512VLDQ-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
 ; AVX512VLDQ-NEXT:    vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm2
 ; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512VLDQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm2
+; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm3
+; AVX512VLDQ-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
+; AVX512VLDQ-NEXT:    vpternlogq $202, %ymm2, %ymm3, %ymm0
 ; AVX512VLDQ-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_shuffle_v16i16_from_v8i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX512VLBW-NEXT:    vpermw %ymm0, %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;

 ; AVX512-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
 ;
 ; AVX512VLDQ-LABEL: var_shuffle_v32i8_from_v16i8:
 ; AVX512VLDQ:       # %bb.0:
 ; AVX512VLDQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm2
 ; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512VLDQ-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm3
+; AVX512VLDQ-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
+; AVX512VLDQ-NEXT:    vpternlogq $202, %ymm3, %ymm2, %ymm0
 ; AVX512VLDQ-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_shuffle_v32i8_from_v16i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX512VLBW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %k1