[X86] Don't combine (x86cmp (trunc (movmsk (bitcast X))), 0) if the truncate discards bits.

[craig.topper]

Summary

We have transform that tries turn a pmovmskb into movmskps/pd or
movmskps to movmskpd. This transform isn't valid if the truncate
discarded bits that might be set by the original movmsk.

We could fix this by inserting an AND after the new movmsk to discard
the equivalent of the truncated bits, but I've left that for later
patch.

Fixes PR52567.

[pengfei]

LGTM.

[RKSimon]

Nice catch!

llvm/lib/Target/X86/X86ISelLowering.cpp

bool IsAllOf = CmpOpcode == X86ISD::SUB && NumElts <= CmpBits &&
                  CmpVal.isMask(NumElts);
   if (!IsAnyOf && !IsAllOf)
     return SDValue();
 
   // See if we can peek through to a vector with a wider element type, if the
   // signbits extend down to all the sub-elements as well.
   // Calling MOVMSK with the wider type, avoiding the bitcast, helps expose
   // potential SimplifyDemandedBits/Elts cases.
-  if (Vec.getOpcode() == ISD::BITCAST) {
+  // If we looked through a truncate that discard bits, we can't do this
+  // transform.
+  // FIXME: We could do this transform for truncates that discarded bits by
+  // inserting an AND mask between the new MOVMSK and the CMP.
+  if (Vec.getOpcode() == ISD::BITCAST && NumElts <= CmpBits) {
     SDValue BC = peekThroughBitcasts(Vec);
     MVT BCVT = BC.getSimpleValueType();
     unsigned BCNumElts = BCVT.getVectorNumElements();
     unsigned BCNumEltBits = BCVT.getScalarSizeInBits();
     if ((BCNumEltBits == 32 || BCNumEltBits == 64) &&
         BCNumEltBits > NumEltBits &&
         DAG.ComputeNumSignBits(BC) > (BCNumEltBits - NumEltBits)) {
       SDLoc DL(EFLAGS);

llvm/test/CodeGen/X86/pr52567.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
 
 ; The and in the test below discards half the bits from vector icmp result.
-; FIXME: The generated code is using a movmskps, but fails to discard bits
-; 2 and 3 before the testl.
+; We use a testb after a pmovmskb to examine only 8 bits.
 
 define i32 @foo(<4 x float> %arg) {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %bb
 ; CHECK-NEXT:    movaps {{.*#+}} xmm1 = [1.00000005E-3,1.00000005E-3,1.00000005E-3,1.00000005E-3]
 ; CHECK-NEXT:    cmpltps %xmm0, %xmm1
-; CHECK-NEXT:    movmskps %xmm1, %ecx
+; CHECK-NEXT:    pmovmskb %xmm1, %ecx
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testl %ecx, %ecx
+; CHECK-NEXT:    testb %cl, %cl
 ; CHECK-NEXT:    sete %al
 ; CHECK-NEXT:    retq
 bb:
   %tmp = fcmp ogt <4 x float> %arg, <float 0x3F50624DE0000000, float 0x3F50624DE0000000, float 0x3F50624DE0000000, float 0x3F50624DE0000000>
   %tmp1 = sext <4 x i1> %tmp to <4 x i32>
   %tmp2 = bitcast <4 x i32> %tmp1 to <16 x i8>
   %tmp3 = icmp slt <16 x i8> %tmp2, zeroinitializer
   %tmp4 = bitcast <16 x i1> %tmp3 to i16
   %tmp5 = and i16 %tmp4, 255
   %tmp6 = icmp eq i16 %tmp5, 0
   %tmp7 = zext i1 %tmp6 to i32
   ret i32 %tmp7
 }