DAG: Lower is.fpclass fcSubnormal|fcZero to fabs(x) < smallest_normal

[arsenm]

Summary

Produces better code on x86_64 only in the unordered case. Not
sure what the exact condition should be to avoid the regression. Free
fabs might do it, or maybe requires legality checks for the alternative
integer expansion.

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

if (FPTestMask == fcNormal) {
         SDValue Abs = DAG.getNode(ISD::FABS, DL, OperandVT, Op);
         SDValue IsFinite = DAG.getSetCC(DL, ResultVT, Abs, Inf, IsFiniteOp);
         SDValue IsNormal =
             DAG.getSetCC(DL, ResultVT, Abs, SmallestNormal, IsNormalOp);
         unsigned LogicOp = IsInvertedFP ? ISD::OR : ISD::AND;
         return DAG.getNode(LogicOp, DL, ResultVT, IsFinite, IsNormal);
       }
     }
+
+    if (OrderedFPTestMask == (fcSubnormal | fcZero) && !IsOrdered) {
+      // TODO: Could handle ordered case, but it produces worse code for
+      // x86. Maybe handle ordered if fabs is free?
+
+      ISD::CondCode OrderedOp = IsInvertedFP ? ISD::SETUGE : ISD::SETOLT;
+      ISD::CondCode UnorderedOp = IsInvertedFP ? ISD::SETOGE : ISD::SETULT;
+
+      if (isCondCodeLegalOrCustom(IsOrdered ? OrderedOp : UnorderedOp,
+                                  OperandVT.getScalarType().getSimpleVT())) {
+        // (issubnormal(x) || iszero(x)) --> fabs(x) < smallest_normal
+
+        // TODO: Maybe only makes sense if fabs is free. Integer test of
+        // exponent bits seems better for x86.
+        SDValue Abs = DAG.getNode(ISD::FABS, DL, OperandVT, Op);
+        SDValue SmallestNormal = DAG.getConstantFP(
+            APFloat::getSmallestNormalized(Semantics), DL, OperandVT);
+        return DAG.getSetCC(DL, ResultVT, Abs, SmallestNormal,
+                            IsOrdered ? OrderedOp : UnorderedOp);
+      }
+    }
   }
 
   // Some checks may be represented as inversion of simpler check, for example
   // "inf|normal|subnormal|zero" => !"nan".
   bool IsInverted = false;
 
   if (FPClassTest InvertedCheck = invertFPClassTestIfSimpler(Test, false)) {
     Test = InvertedCheck;

llvm/test/CodeGen/X86/is_fpclass.ll

 ; CHECK-64-NEXT:    retq
   %class = tail call i1 @llvm.is.fpclass.f32(float %x, i32 147)  ; 0x90|0x3 = "subnormal|nan"
   ret i1 %class
 }
 
 define i1 @issubnormal_or_zero_or_nan_f(float %x) {
 ; CHECK-32-LABEL: issubnormal_or_zero_or_nan_f:
 ; CHECK-32:       # %bb.0:
-; CHECK-32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK-32-NEXT:    testl $2139095040, %eax # imm = 0x7F800000
-; CHECK-32-NEXT:    sete %cl
-; CHECK-32-NEXT:    andl $2147483647, %eax # imm = 0x7FFFFFFF
-; CHECK-32-NEXT:    cmpl $2139095041, %eax # imm = 0x7F800001
-; CHECK-32-NEXT:    setge %al
-; CHECK-32-NEXT:    orb %cl, %al
+; CHECK-32-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-32-NEXT:    fabs
+; CHECK-32-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}
+; CHECK-32-NEXT:    fxch %st(1)
+; CHECK-32-NEXT:    fucompp
+; CHECK-32-NEXT:    fnstsw %ax
+; CHECK-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; CHECK-32-NEXT:    sahf
+; CHECK-32-NEXT:    setb %al
 ; CHECK-32-NEXT:    retl
 ;
 ; CHECK-64-LABEL: issubnormal_or_zero_or_nan_f:
 ; CHECK-64:       # %bb.0:
-; CHECK-64-NEXT:    movd %xmm0, %eax
-; CHECK-64-NEXT:    testl $2139095040, %eax # imm = 0x7F800000
-; CHECK-64-NEXT:    sete %cl
-; CHECK-64-NEXT:    andl $2147483647, %eax # imm = 0x7FFFFFFF
-; CHECK-64-NEXT:    cmpl $2139095041, %eax # imm = 0x7F800001
-; CHECK-64-NEXT:    setge %al
-; CHECK-64-NEXT:    orb %cl, %al
+; CHECK-64-NEXT:    andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-64-NEXT:    ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-64-NEXT:    setb %al
 ; CHECK-64-NEXT:    retq
   %class = tail call i1 @llvm.is.fpclass.f32(float %x, i32 243)  ; 0xf0|0x3 = "subnormal|zero|nan"
   ret i1 %class
 }
 
 define i1 @issubnormal_or_zero_or_nan_f_daz(float %x) #0 {
 ; CHECK-32-LABEL: issubnormal_or_zero_or_nan_f_daz:
 ; CHECK-32:       # %bb.0:

 ; CHECK-64-NEXT:    retq
   %class = tail call i1 @llvm.is.fpclass.f32(float %x, i32 876)  ; ~(0x90|0x3) = ~"subnormal|nan"
   ret i1 %class
 }
 
 define i1 @not_issubnormal_or_zero_or_nan_f(float %x) {
 ; CHECK-32-LABEL: not_issubnormal_or_zero_or_nan_f:
 ; CHECK-32:       # %bb.0:
-; CHECK-32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK-32-NEXT:    testl $2139095040, %eax # imm = 0x7F800000
-; CHECK-32-NEXT:    setne %cl
-; CHECK-32-NEXT:    andl $2147483647, %eax # imm = 0x7FFFFFFF
-; CHECK-32-NEXT:    cmpl $2139095041, %eax # imm = 0x7F800001
-; CHECK-32-NEXT:    setl %al
-; CHECK-32-NEXT:    andb %cl, %al
+; CHECK-32-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-32-NEXT:    fabs
+; CHECK-32-NEXT:    flds {{\.?LCPI[0-9]+_[0-9]+}}
+; CHECK-32-NEXT:    fxch %st(1)
+; CHECK-32-NEXT:    fucompp
+; CHECK-32-NEXT:    fnstsw %ax
+; CHECK-32-NEXT:    # kill: def $ah killed $ah killed $ax
+; CHECK-32-NEXT:    sahf
+; CHECK-32-NEXT:    setae %al
 ; CHECK-32-NEXT:    retl
 ;
 ; CHECK-64-LABEL: not_issubnormal_or_zero_or_nan_f:
 ; CHECK-64:       # %bb.0:
-; CHECK-64-NEXT:    movd %xmm0, %eax
-; CHECK-64-NEXT:    testl $2139095040, %eax # imm = 0x7F800000
-; CHECK-64-NEXT:    setne %cl
-; CHECK-64-NEXT:    andl $2147483647, %eax # imm = 0x7FFFFFFF
-; CHECK-64-NEXT:    cmpl $2139095041, %eax # imm = 0x7F800001
-; CHECK-64-NEXT:    setl %al
-; CHECK-64-NEXT:    andb %cl, %al
+; CHECK-64-NEXT:    andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-64-NEXT:    ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-64-NEXT:    setae %al
 ; CHECK-64-NEXT:    retq
   %class = tail call i1 @llvm.is.fpclass.f32(float %x, i32 780)  ; ~(0xf0|0x3) = ~"subnormal|zero|nan"
   ret i1 %class
 }
 
 define i1 @not_issubnormal_or_zero_or_nan_f_daz(float %x) #0 {
 ; CHECK-32-LABEL: not_issubnormal_or_zero_or_nan_f_daz:
 ; CHECK-32:       # %bb.0: