Index: llvm/include/llvm/CodeGen/CodeGenCommonISel.h =================================================================== --- llvm/include/llvm/CodeGen/CodeGenCommonISel.h +++ llvm/include/llvm/CodeGen/CodeGenCommonISel.h @@ -219,9 +219,13 @@ /// (i.e. fewer instructions should be required to lower it). An example is the /// test "inf|normal|subnormal|zero", which is an inversion of "nan". /// \param Test The test as specified in 'is_fpclass' intrinsic invocation. +/// +/// \param UseFP The intention is to perform the comparison using floating-point +/// compare instructions which check for nan. +/// /// \returns The inverted test, or fcNone, if inversion does not produce a /// simpler test. -FPClassTest invertFPClassTestIfSimpler(FPClassTest Test); +FPClassTest invertFPClassTestIfSimpler(FPClassTest Test, bool UseFP); /// Assuming the instruction \p MI is going to be deleted, attempt to salvage /// debug users of \p MI by writing the effect of \p MI in a DIExpression. Index: llvm/lib/CodeGen/CodeGenCommonISel.cpp =================================================================== --- llvm/lib/CodeGen/CodeGenCommonISel.cpp +++ llvm/lib/CodeGen/CodeGenCommonISel.cpp @@ -173,8 +173,9 @@ return SplitPoint; } -FPClassTest llvm::invertFPClassTestIfSimpler(FPClassTest Test) { +FPClassTest llvm::invertFPClassTestIfSimpler(FPClassTest Test, bool UseFP) { FPClassTest InvertedTest = ~Test; + // Pick the direction with fewer tests // TODO: Handle more combinations of cases that can be handled together switch (static_cast(InvertedTest)) { @@ -200,6 +201,14 @@ case fcSubnormal | fcZero: case fcSubnormal | fcZero | fcNan: return InvertedTest; + case fcInf | fcNan: + // If we're trying to use fcmp, we can take advantage of the nan check + // behavior of the compare (but this is more instructions in the integer + // expansion). + return UseFP ? InvertedTest : fcNone; + case fcFinite | fcNan: + // Inversion of fcInf, which can be done in a combined check. + return fcNone; default: return fcNone; } Index: llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -8198,16 +8198,17 @@ } SDValue TargetLowering::expandIS_FPCLASS(EVT ResultVT, SDValue Op, - FPClassTest Test, SDNodeFlags Flags, - const SDLoc &DL, + const FPClassTest OrigTestMask, + SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const { EVT OperandVT = Op.getValueType(); assert(OperandVT.isFloatingPoint()); + FPClassTest Test = OrigTestMask; // Degenerated cases. if (Test == fcNone) return DAG.getBoolConstant(false, DL, ResultVT, OperandVT); - if ((Test & fcAllFlags) == fcAllFlags) + if (Test == fcAllFlags) return DAG.getBoolConstant(true, DL, ResultVT, OperandVT); // PPC double double is a pair of doubles, of which the higher part determines @@ -8218,14 +8219,6 @@ OperandVT = MVT::f64; } - // Some checks may be represented as inversion of simpler check, for example - // "inf|normal|subnormal|zero" => !"nan". - bool IsInverted = false; - if (FPClassTest InvertedCheck = invertFPClassTestIfSimpler(Test)) { - IsInverted = true; - Test = InvertedCheck; - } - // Floating-point type properties. EVT ScalarFloatVT = OperandVT.getScalarType(); const Type *FloatTy = ScalarFloatVT.getTypeForEVT(*DAG.getContext()); @@ -8236,11 +8229,20 @@ // exceptions are ignored. if (Flags.hasNoFPExcept() && isOperationLegalOrCustom(ISD::SETCC, OperandVT.getScalarType())) { - ISD::CondCode OrderedCmpOpcode = IsInverted ? ISD::SETUNE : ISD::SETOEQ; - ISD::CondCode UnorderedCmpOpcode = IsInverted ? ISD::SETONE : ISD::SETUEQ; + FPClassTest FPTestMask = Test; + bool IsInvertedFP = false; + + if (FPClassTest InvertedFPCheck = + invertFPClassTestIfSimpler(FPTestMask, true)) { + FPTestMask = InvertedFPCheck; + IsInvertedFP = true; + } + + ISD::CondCode OrderedCmpOpcode = IsInvertedFP ? ISD::SETUNE : ISD::SETOEQ; + ISD::CondCode UnorderedCmpOpcode = IsInvertedFP ? ISD::SETONE : ISD::SETUEQ; if (std::optional IsCmp0 = - isFCmpEqualZero(Test, Semantics, DAG.getMachineFunction()); + isFCmpEqualZero(FPTestMask, Semantics, DAG.getMachineFunction()); IsCmp0 && (isCondCodeLegalOrCustom( *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode, OperandVT.getScalarType().getSimpleVT()))) { @@ -8252,15 +8254,16 @@ *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode); } - if (Test == fcNan && - isCondCodeLegalOrCustom(IsInverted ? ISD::SETO : ISD::SETUO, - OperandVT.getScalarType().getSimpleVT())) { + if (FPTestMask == fcNan && + isCondCodeLegalOrCustom(IsInvertedFP ? ISD::SETO : ISD::SETUO, + OperandVT.getScalarType().getSimpleVT())) return DAG.getSetCC(DL, ResultVT, Op, Op, - IsInverted ? ISD::SETO : ISD::SETUO); - } + IsInvertedFP ? ISD::SETO : ISD::SETUO); - if (Test == fcInf && - isCondCodeLegalOrCustom(IsInverted ? ISD::SETUNE : ISD::SETOEQ, + bool IsOrderedInf = FPTestMask == fcInf; + if ((FPTestMask == fcInf || FPTestMask == (fcInf | fcNan)) && + isCondCodeLegalOrCustom(IsOrderedInf ? OrderedCmpOpcode + : UnorderedCmpOpcode, OperandVT.getScalarType().getSimpleVT()) && isOperationLegalOrCustom(ISD::FABS, OperandVT.getScalarType())) { // isinf(x) --> fabs(x) == inf @@ -8268,10 +8271,19 @@ SDValue Inf = DAG.getConstantFP(APFloat::getInf(Semantics), DL, OperandVT); return DAG.getSetCC(DL, ResultVT, Abs, Inf, - IsInverted ? ISD::SETUNE : ISD::SETOEQ); + IsOrderedInf ? OrderedCmpOpcode : UnorderedCmpOpcode); } } + // Some checks may be represented as inversion of simpler check, for example + // "inf|normal|subnormal|zero" => !"nan". + bool IsInverted = false; + + if (FPClassTest InvertedCheck = invertFPClassTestIfSimpler(Test, false)) { + Test = InvertedCheck; + IsInverted = true; + } + // In the general case use integer operations. unsigned BitSize = OperandVT.getScalarSizeInBits(); EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), BitSize); Index: llvm/test/CodeGen/X86/is_fpclass.ll =================================================================== --- llvm/test/CodeGen/X86/is_fpclass.ll +++ llvm/test/CodeGen/X86/is_fpclass.ll @@ -239,18 +239,22 @@ define i1 @isfinite_f(float %x) { ; CHECK-32-LABEL: isfinite_f: ; CHECK-32: # %bb.0: # %entry -; CHECK-32-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF -; CHECK-32-NEXT: andl {{[0-9]+}}(%esp), %eax -; CHECK-32-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 -; CHECK-32-NEXT: setl %al +; CHECK-32-NEXT: flds {{[0-9]+}}(%esp) +; CHECK-32-NEXT: fabs +; CHECK-32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; CHECK-32-NEXT: fxch %st(1) +; CHECK-32-NEXT: fucompp +; CHECK-32-NEXT: fnstsw %ax +; CHECK-32-NEXT: # kill: def $ah killed $ah killed $ax +; CHECK-32-NEXT: sahf +; CHECK-32-NEXT: setne %al ; CHECK-32-NEXT: retl ; ; CHECK-64-LABEL: isfinite_f: ; CHECK-64: # %bb.0: # %entry -; CHECK-64-NEXT: movd %xmm0, %eax -; CHECK-64-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF -; CHECK-64-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 -; CHECK-64-NEXT: setl %al +; CHECK-64-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; CHECK-64-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; CHECK-64-NEXT: setne %al ; CHECK-64-NEXT: retq entry: %0 = tail call i1 @llvm.is.fpclass.f32(float %x, i32 504) ; 0x1f8 = "finite" @@ -1137,20 +1141,22 @@ define i1 @isfinite_d(double %x) { ; CHECK-32-LABEL: isfinite_d: ; CHECK-32: # %bb.0: # %entry -; CHECK-32-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF -; CHECK-32-NEXT: andl {{[0-9]+}}(%esp), %eax -; CHECK-32-NEXT: cmpl $2146435072, %eax # imm = 0x7FF00000 -; CHECK-32-NEXT: setl %al +; CHECK-32-NEXT: fldl {{[0-9]+}}(%esp) +; CHECK-32-NEXT: fabs +; CHECK-32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; CHECK-32-NEXT: fxch %st(1) +; CHECK-32-NEXT: fucompp +; CHECK-32-NEXT: fnstsw %ax +; CHECK-32-NEXT: # kill: def $ah killed $ah killed $ax +; CHECK-32-NEXT: sahf +; CHECK-32-NEXT: setne %al ; CHECK-32-NEXT: retl ; ; CHECK-64-LABEL: isfinite_d: ; CHECK-64: # %bb.0: # %entry -; CHECK-64-NEXT: movq %xmm0, %rax -; CHECK-64-NEXT: movabsq $9223372036854775807, %rcx # imm = 0x7FFFFFFFFFFFFFFF -; CHECK-64-NEXT: andq %rax, %rcx -; CHECK-64-NEXT: movabsq $9218868437227405312, %rax # imm = 0x7FF0000000000000 -; CHECK-64-NEXT: cmpq %rax, %rcx -; CHECK-64-NEXT: setl %al +; CHECK-64-NEXT: andpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; CHECK-64-NEXT: ucomisd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; CHECK-64-NEXT: setne %al ; CHECK-64-NEXT: retq entry: %0 = tail call i1 @llvm.is.fpclass.f64(double %x, i32 504) ; 0x1f8 = "finite" @@ -1918,18 +1924,22 @@ define i1 @not_isinf_or_nan_f(float %x) { ; CHECK-32-LABEL: not_isinf_or_nan_f: ; CHECK-32: # %bb.0: # %entry -; CHECK-32-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF -; CHECK-32-NEXT: andl {{[0-9]+}}(%esp), %eax -; CHECK-32-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 -; CHECK-32-NEXT: setl %al +; CHECK-32-NEXT: flds {{[0-9]+}}(%esp) +; CHECK-32-NEXT: fabs +; CHECK-32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} +; CHECK-32-NEXT: fxch %st(1) +; CHECK-32-NEXT: fucompp +; CHECK-32-NEXT: fnstsw %ax +; CHECK-32-NEXT: # kill: def $ah killed $ah killed $ax +; CHECK-32-NEXT: sahf +; CHECK-32-NEXT: setne %al ; CHECK-32-NEXT: retl ; ; CHECK-64-LABEL: not_isinf_or_nan_f: ; CHECK-64: # %bb.0: # %entry -; CHECK-64-NEXT: movd %xmm0, %eax -; CHECK-64-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF -; CHECK-64-NEXT: cmpl $2139095040, %eax # imm = 0x7F800000 -; CHECK-64-NEXT: setl %al +; CHECK-64-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; CHECK-64-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; CHECK-64-NEXT: setne %al ; CHECK-64-NEXT: retq entry: %0 = tail call i1 @llvm.is.fpclass.f32(float %x, i32 504) ; ~(0x204|0x3) = "~(inf|nan)"