InstCombine: Handle folding some negated is_fpclass mask test cases

[arsenm]

Summary

Jay requested handling these in https://reviews.llvm.org/D137811

I realized these cases that turn class into fcmp are likely broken if
input denormals are treated as 0. However, this is so poorly specified
I'm not sure it's really wrong. alive2 says they're wrong for
denormals. Assuming is_fpclass does not read canonical inputs (as
might be implied by not trapping on snans), under DAZ,
is_fpclass(denorm, zero) is false and fcmp denorm, 0 is true.

The LangRef doesn't say anything about this for llvm.is.fpclass. The
AMDGPU class instruction does not consider the FP mode, and will see
denormal inputs as-is rather than the flush the input.

The only spec I know around denormal behavior is OpenCL's
-cl-denorms-are-zero, which doesn't clarify anything. It merely states
an implementation "can choose not to flush denorms to zero". It
doesn't specify whether this means outputs can be flushed, or if
inputs can be treated as 0s or what operations this applies to.

std::fpclassify does have a FP_SUBNORMAL result type. Under clang's
fcmp based implementation, it will return 0 for a denormal under DAZ.

[foad]

llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp

854

That's also equivalent to != 0 - at least for the usual mapping of C-like comparison operators onto fp predicates.

[jcranmer-intel]

If DAZ mode is semantically "denormals are non-canonical zero", then isfpclass(denormal, Zero) arguably should be true in DAZ mode, so lowering that to fcmp 0.0 would be legal. On the other hand, the actual lowering of an unoptimized isfpclass is likely to rely on bit patterns which would ignore the current dynamic value of DAZ. On the other other hand, there's no clear indication of what isfpclass returns for noncanonical values... and on the other other other hand, leaving the result of isfpclass as unspecified behavior for noncanonical inputs is incredibly unfriendly to users.

llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp

854

They're not equivalent: !(x == 0) is true if x is NaN, and x != 0 is false if x is NaN.

[foad]

llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp

854

They are equivalent. The usual mapping is that == means oeq and != means une. See for example F.9.3 in the C spec or https://github.com/llvm/llvm-project/blob/df43ec30ab66f5af7bbf87e121e0fe26aad478b4/clang/lib/CodeGen/CGExprScalar.cpp#L865

[foad]

Seems OK. A more thorough handling of the ninf/nnan stuff could go like:

BitsToIgnore = 0;
if (isKnownNeverNaN)
  BitsToIgnore |= fcNan;
if (isKnownNeverInfinity)
  BitsToIgnore |= fcInf;

if ((Mask & ~BitsToIgnore) == 0) { convert to false }
if ((Mask | BitsToIgnore) == fcAllFlags) { convert to true }
if ((Mask & ~BitsToIgnore) == fcNan) { convert to isnan }
if ((Mask | BitsToIgnore) == (~fcNan & fcAllFlags)) { convert to !isnan }
if ((Mask & ~BitsToIgnore) == fcZero) { convert to == 0.0 }
if ((Mask | BitsToIgnore) == (~fcZero & fcAllFlags)) { convert to != 0.0 }

[arsenm]

Resplitting the patches so the pairs of negated cases are handled together

llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp

Instruction *InstCombinerImpl::foldIntrinsicIsFPClass(IntrinsicInst &II) {
 
   if (Mask == fcNan && !IsStrict) {
     // Equivalent of isnan. Replace with standard fcmp.
     Value *FCmp = Builder.CreateFCmpUNO(Src0, Src0);
     FCmp->takeName(&II);
     return replaceInstUsesWith(II, FCmp);
   }
 
+  if (Mask == (~fcNan & fcAllFlags) && !IsStrict) {
+    // Equivalent of !isnan. Replace with standard fcmp.
+    Value *FCmp = Builder.CreateFCmpORD(Src0, Src0);
+    FCmp->takeName(&II);
+    return replaceInstUsesWith(II, FCmp);
+  }
+
   if (Mask == fcZero && !IsStrict) {
     // Equivalent of == 0.
     Value *FCmp =
         Builder.CreateFCmpOEQ(Src0, ConstantFP::get(Src0->getType(), 0.0));
 
     FCmp->takeName(&II);
     return replaceInstUsesWith(II, FCmp);
   }
 
+  if (Mask == (~fcZero & fcAllFlags) && !IsStrict) {
+    // Equivalent of !(x == 0).
+    Value *FCmp =
+        Builder.CreateFCmpUNE(Src0, ConstantFP::get(Src0->getType(), 0.0));
+
+    FCmp->takeName(&II);
+    return replaceInstUsesWith(II, FCmp);
+  }
+
   // fp_class (nnan x), qnan|snan|other -> fp_class (nnan x), other
   if ((Mask & fcNan) && isKnownNeverNaN(Src0, &getTargetLibraryInfo())) {
     return replaceOperand(II, 1,
                           ConstantInt::get(Src1->getType(), Mask & ~fcNan));
   }
 
-  // TODO: Handle inverted mask cases, e.g.
   // fp_class (nnan x), ~(qnan|snan) -> true
+  if (Mask == (~fcNan & fcAllFlags) &&
+      isKnownNeverNaN(Src0, &getTargetLibraryInfo())) {
+    return replaceInstUsesWith(II, ConstantInt::get(II.getType(), true));
+  }
 
   // fp_class (ninf x), ninf|pinf|other -> fp_class (ninf x), other
   if ((Mask & fcInf) && isKnownNeverInfinity(Src0, &getTargetLibraryInfo())) {
     return replaceOperand(II, 1,
                           ConstantInt::get(Src1->getType(), Mask & ~fcInf));
   }
 
+  // fp_class (ninf x), ~(ninf|pinf) -> true
+  if (Mask == (~fcInf & fcAllFlags) &&
+      isKnownNeverInfinity(Src0, &getTargetLibraryInfo())) {
+    return replaceInstUsesWith(II, ConstantInt::get(II.getType(), true));
+  }
+
   // Clamp mask to used bits
   if ((Mask & fcAllFlags) != Mask) {
     CallInst *NewCall = Builder.CreateCall(
       II.getCalledFunction(),
       {Src0, ConstantInt::get(Src1->getType(), Mask & fcAllFlags)});
 
     NewCall->takeName(&II);
     return replaceInstUsesWith(II, NewCall);

llvm/test/Transforms/InstCombine/is_fpclass.ll

 ; CHECK-NEXT:    ret i1 [[VAL]]
 ;
   %val = call i1 @llvm.is.fpclass.f32(float %x, i32 96) ; fcZero
   ret i1 %val
 }
 
 define i1 @test_class_is_not_p0_n0_f32(float %x) {
 ; CHECK-LABEL: @test_class_is_not_p0_n0_f32(
-; CHECK-NEXT:    [[VAL:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X:%.*]], i32 927)
+; CHECK-NEXT:    [[VAL:%.*]] = fcmp une float [[X:%.*]], 0.000000e+00
 ; CHECK-NEXT:    ret i1 [[VAL]]
 ;
   %val = call i1 @llvm.is.fpclass.f32(float %x, i32 927) ; ~fcZero & fcAllFlags
   ret i1 %val
 }
 
 define i1 @test_class_is_not_p0_n0_f32_strict(float %x) {
 ; CHECK-LABEL: @test_class_is_not_p0_n0_f32_strict(
 ; CHECK-NEXT:    [[VAL:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X:%.*]], i32 927) #[[ATTR3]]
 ; CHECK-NEXT:    ret i1 [[VAL]]
 ;
   %val = call i1 @llvm.is.fpclass.f32(float %x, i32 927) strictfp ; ~fcZero & fcAllFlags
   ret i1 %val
 }
 
 define i1 @test_class_is_not_p0_n0_f32_daz(float %x) "denormal-fp-math"="ieee,preserve-sign" {
 ; CHECK-LABEL: @test_class_is_not_p0_n0_f32_daz(
-; CHECK-NEXT:    [[VAL:%.*]] = call i1 @llvm.is.fpclass.f32(float [[X:%.*]], i32 927)
+; CHECK-NEXT:    [[VAL:%.*]] = fcmp une float [[X:%.*]], 0.000000e+00
 ; CHECK-NEXT:    ret i1 [[VAL]]
 ;
   %val = call i1 @llvm.is.fpclass.f32(float %x, i32 927) ; ~fcZero & fcAllFlags
   ret i1 %val
 }
 
 define i1 @test_class_is_not_p0_n0_psub_nsub_f32_daz(float %x) "denormal-fp-math"="ieee,preserve-sign" {
 ; CHECK-LABEL: @test_class_is_not_p0_n0_psub_nsub_f32_daz(
;
   %nnan = fadd nnan float %x, 1.0
   %class = call i1 @llvm.is.fpclass.f32(float %nnan, i32 267)
   ret i1 %class
 }
 
 ; Fold test of is not nan
 define i1 @test_class_is_not_nan_nnan_src(float %x) {
 ; CHECK-LABEL: @test_class_is_not_nan_nnan_src(
-; CHECK-NEXT:    [[NNAN:%.*]] = fadd nnan float [[X:%.*]], 1.000000e+00
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[NNAN]], i32 1020)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 true
 ;
   %nnan = fadd nnan float %x, 1.0
   %class = call i1 @llvm.is.fpclass.f32(float %nnan, i32 1020) ; ~fcNan & fcAllFlags
   ret i1 %class
 }
 
 define i1 @test_class_is_not_nan_nnan_src_strict(float %x) {
 ; CHECK-LABEL: @test_class_is_not_nan_nnan_src_strict(
-; CHECK-NEXT:    [[NNAN:%.*]] = fadd nnan float [[X:%.*]], 1.000000e+00
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[NNAN]], i32 1020) #[[ATTR3]]
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 true
 ;
   %nnan = fadd nnan float %x, 1.0
   %class = call i1 @llvm.is.fpclass.f32(float %nnan, i32 1020) strictfp ; ~fcNan & fcAllFlags
   ret i1 %class
 }
 
 ; --------------------------------------------------------------------
 ; llvm.is.fpclass with ninf sources

 ;
   %ninf = fadd ninf float %x, 1.0
   %class = call i1 @llvm.is.fpclass.f32(float %ninf, i32 772)
   ret i1 %class
 }
 
 define i1 @test_class_is_not_inf_ninf_src(float %x) {
 ; CHECK-LABEL: @test_class_is_not_inf_ninf_src(
-; CHECK-NEXT:    [[NINF:%.*]] = fadd ninf float [[X:%.*]], 1.000000e+00
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[NINF]], i32 507)
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 true
 ;
   %ninf = fadd ninf float %x, 1.0
   %class = call i1 @llvm.is.fpclass.f32(float %ninf, i32 507) ; ~fcInf & fcAllFlags
   ret i1 %class
 }
 
 define i1 @test_class_is_not_inf_ninf_src_strict(float %x) {
 ; CHECK-LABEL: @test_class_is_not_inf_ninf_src_strict(
-; CHECK-NEXT:    [[NINF:%.*]] = fadd ninf float [[X:%.*]], 1.000000e+00
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f32(float [[NINF]], i32 507) #[[ATTR3]]
-; CHECK-NEXT:    ret i1 [[CLASS]]
+; CHECK-NEXT:    ret i1 true
 ;
   %ninf = fadd ninf float %x, 1.0
   %class = call i1 @llvm.is.fpclass.f32(float %ninf, i32 507) strictfp ; ~fcInf & fcAllFlags
   ret i1 %class
 }
 
 ; --------------------------------------------------------------------
 ; Negation of llvm.is.fpclass