diff --git a/llvm/include/llvm/IR/FPEnv.h b/llvm/include/llvm/IR/FPEnv.h --- a/llvm/include/llvm/IR/FPEnv.h +++ b/llvm/include/llvm/IR/FPEnv.h @@ -15,6 +15,7 @@ #ifndef LLVM_IR_FLOATINGPOINT_H #define LLVM_IR_FLOATINGPOINT_H +#include "llvm/ADT/APFloat.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" #include @@ -66,5 +67,8 @@ /// input in constrained intrinsic exception behavior metadata. Optional ExceptionBehaviorToStr(fp::ExceptionBehavior); +/// Converts rounding mode represented by fp::RoundingMode to the rounding mode +/// index used by APFloat. For fp::rmDynamic it returns None. +Optional getAPFloatRoundingMode(fp::RoundingMode); } #endif diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp --- a/llvm/lib/Analysis/ConstantFolding.cpp +++ b/llvm/lib/Analysis/ConstantFolding.cpp @@ -1401,41 +1401,19 @@ // bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) { - if (Call->isNoBuiltin() || Call->isStrictFP()) + if (Call->isNoBuiltin()) return false; switch (F->getIntrinsicID()) { - case Intrinsic::fabs: - case Intrinsic::minnum: - case Intrinsic::maxnum: - case Intrinsic::minimum: - case Intrinsic::maximum: - case Intrinsic::log: - case Intrinsic::log2: - case Intrinsic::log10: - case Intrinsic::exp: - case Intrinsic::exp2: - case Intrinsic::floor: - case Intrinsic::ceil: - case Intrinsic::sqrt: - case Intrinsic::sin: - case Intrinsic::cos: - case Intrinsic::trunc: - case Intrinsic::rint: - case Intrinsic::nearbyint: - case Intrinsic::pow: - case Intrinsic::powi: + // Operations that do not operate floating-point numbers and do not depend on + // FP environment can be folded even in strictfp functions. case Intrinsic::bswap: case Intrinsic::ctpop: case Intrinsic::ctlz: case Intrinsic::cttz: case Intrinsic::fshl: case Intrinsic::fshr: - case Intrinsic::fma: - case Intrinsic::fmuladd: - case Intrinsic::copysign: case Intrinsic::launder_invariant_group: case Intrinsic::strip_invariant_group: - case Intrinsic::round: case Intrinsic::masked_load: case Intrinsic::sadd_with_overflow: case Intrinsic::uadd_with_overflow: @@ -1449,9 +1427,31 @@ case Intrinsic::usub_sat: case Intrinsic::smul_fix: case Intrinsic::smul_fix_sat: + case Intrinsic::bitreverse: + case Intrinsic::is_constant: + return true; + + // Floating point operations cannot be folded in strictfp functions in + // general case. They can be folded if FP environment is known to compiler. + case Intrinsic::minnum: + case Intrinsic::maxnum: + case Intrinsic::minimum: + case Intrinsic::maximum: + case Intrinsic::log: + case Intrinsic::log2: + case Intrinsic::log10: + case Intrinsic::exp: + case Intrinsic::exp2: + case Intrinsic::sqrt: + case Intrinsic::sin: + case Intrinsic::cos: + case Intrinsic::pow: + case Intrinsic::powi: + case Intrinsic::fma: + case Intrinsic::fmuladd: case Intrinsic::convert_from_fp16: case Intrinsic::convert_to_fp16: - case Intrinsic::bitreverse: + // The intrinsics below depend on rounding mode in MXCSR. case Intrinsic::x86_sse_cvtss2si: case Intrinsic::x86_sse_cvtss2si64: case Intrinsic::x86_sse_cvttss2si: @@ -1476,14 +1476,35 @@ case Intrinsic::x86_avx512_vcvtsd2usi64: case Intrinsic::x86_avx512_cvttsd2usi: case Intrinsic::x86_avx512_cvttsd2usi64: - case Intrinsic::is_constant: + return !Call->isStrictFP(); + + // Sign operations are actually bitwise operations, they do not raise + // exceptions even for SNANs. + case Intrinsic::fabs: + case Intrinsic::copysign: + // Non-constrained variants of rounding operations means default FP + // environment, they can be folded in any case. + case Intrinsic::ceil: + case Intrinsic::floor: + case Intrinsic::round: + case Intrinsic::trunc: + case Intrinsic::nearbyint: + case Intrinsic::rint: + // Constrained intrinsics can be folded if FP environment is known + // to compiler. + case Intrinsic::experimental_constrained_ceil: + case Intrinsic::experimental_constrained_floor: + case Intrinsic::experimental_constrained_round: + case Intrinsic::experimental_constrained_trunc: + case Intrinsic::experimental_constrained_nearbyint: + case Intrinsic::experimental_constrained_rint: return true; default: return false; case Intrinsic::not_intrinsic: break; } - if (!F->hasName()) + if (!F->hasName() || Call->isStrictFP()) return false; // In these cases, the check of the length is required. We don't want to @@ -1777,6 +1798,52 @@ return ConstantFP::get(Ty->getContext(), U); } + // Rounding operations (floor, trunc, ceil, round, rint and nearbyint) do + // not raise FP exceptions, unless the argument is signaling NaN. Large + // numbers are already integers (so overflow never occurs) and the result of + // any rounding do not require further rounding, so it is always exact. + // Infinities and quite NaNs propagate without exceptions. + + Optional RM; + switch (IntrinsicID) { + default: + break; + case Intrinsic::experimental_constrained_nearbyint: { + case Intrinsic::experimental_constrained_rint: + auto CI = cast(Call); + Optional RMOp = CI->getRoundingMode(); + if (RMOp) + RM = getAPFloatRoundingMode(*RMOp); + if (!RM) + return nullptr; + break; + } + case Intrinsic::experimental_constrained_round: + RM = APFloat::rmNearestTiesToAway; + break; + case Intrinsic::experimental_constrained_ceil: + RM = APFloat::rmTowardPositive; + break; + case Intrinsic::experimental_constrained_floor: + RM = APFloat::rmTowardNegative; + break; + case Intrinsic::experimental_constrained_trunc: + RM = APFloat::rmTowardZero; + break; + } + if (RM) { + auto CI = cast(Call); + if (U.isFinite()) { + U.roundToIntegral(*RM); + } else if (U.isSignaling()) { + Optional EB = CI->getExceptionBehavior(); + if (EB && *EB != fp::ebIgnore) + return nullptr; + U = APFloat::getQNaN(U.getSemantics()); + } + return ConstantFP::get(Ty->getContext(), U); + } + /// We only fold functions with finite arguments. Folding NaN and inf is /// likely to be aborted with an exception anyway, and some host libms /// have known errors raising exceptions. @@ -2471,7 +2538,7 @@ Constant *llvm::ConstantFoldCall(const CallBase *Call, Function *F, ArrayRef Operands, const TargetLibraryInfo *TLI) { - if (Call->isNoBuiltin() || Call->isStrictFP()) + if (Call->isNoBuiltin()) return nullptr; if (!F->hasName()) return nullptr; diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp --- a/llvm/lib/Analysis/InstructionSimplify.cpp +++ b/llvm/lib/Analysis/InstructionSimplify.cpp @@ -5333,8 +5333,11 @@ ConstantArgs.reserve(NumArgs); for (auto &Arg : Call->args()) { Constant *C = dyn_cast(&Arg); - if (!C) + if (!C) { + if (isa(Arg.get())) + continue; return nullptr; + } ConstantArgs.push_back(C); } diff --git a/llvm/lib/IR/FPEnv.cpp b/llvm/lib/IR/FPEnv.cpp --- a/llvm/lib/IR/FPEnv.cpp +++ b/llvm/lib/IR/FPEnv.cpp @@ -75,4 +75,16 @@ return ExceptStr; } -} \ No newline at end of file +Optional +getAPFloatRoundingMode(fp::RoundingMode RM) { + switch (RM) { + case fp::rmDynamic: return None; + case fp::rmToNearest: return APFloat::rmNearestTiesToEven; + case fp::rmDownward: return APFloat::rmTowardNegative; + case fp::rmUpward: return APFloat::rmTowardPositive; + case fp::rmTowardZero: return APFloat::rmTowardZero; + } + llvm_unreachable("Unexpected rounding mode"); +} + +} diff --git a/llvm/test/Transforms/InstSimplify/constfold-constrained.ll b/llvm/test/Transforms/InstSimplify/constfold-constrained.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/InstSimplify/constfold-constrained.ll @@ -0,0 +1,206 @@ +; RUN: opt < %s -instsimplify -S | FileCheck %s + + +; Verify that floor(10.1) is folded to 10.0 when the exception behavior is 'ignore'. +define double @floor_01() #0 { +entry: + %result = call double @llvm.experimental.constrained.floor.f64( + double 1.010000e+01, + metadata !"fpexcept.ignore") #0 + ret double %result + ; CHECK-LABEL: @floor_01 + ; CHECK: ret double 1.000000e+01 +} + +; Verify that floor(-10.1) is folded to -11.0 when the exception behavior is not 'ignore'. +define double @floor_02() #0 { +entry: + %result = call double @llvm.experimental.constrained.floor.f64( + double -1.010000e+01, + metadata !"fpexcept.strict") #0 + ret double %result + ; CHECK-LABEL: @floor_02 + ; CHECK: ret double -1.100000e+01 +} + +; Verify that ceil(10.1) is folded to 11.0 when the exception behavior is 'ignore'. +define double @ceil_01() #0 { +entry: + %result = call double @llvm.experimental.constrained.ceil.f64( + double 1.010000e+01, + metadata !"fpexcept.ignore") #0 + ret double %result + ; CHECK-LABEL: @ceil_01 + ; CHECK: ret double 1.100000e+01 +} + +; Verify that ceil(-10.1) is folded to -10.0 when the exception behavior is not 'ignore'. +define double @ceil_02() #0 { +entry: + %result = call double @llvm.experimental.constrained.ceil.f64( + double -1.010000e+01, + metadata !"fpexcept.strict") #0 + ret double %result + ; CHECK-LABEL: @ceil_02 + ; CHECK: ret double -1.000000e+01 +} + +; Verify that trunc(10.1) is folded to 10.0 when the exception behavior is 'ignore'. +define double @trunc_01() #0 { +entry: + %result = call double @llvm.experimental.constrained.trunc.f64( + double 1.010000e+01, + metadata !"fpexcept.ignore") #0 + ret double %result + ; CHECK-LABEL: @trunc_01 + ; CHECK: ret double 1.000000e+01 +} + +; Verify that trunc(-10.1) is folded to -10.0 when the exception behavior is NOT 'ignore'. +define double @trunc_02() #0 { +entry: + %result = call double @llvm.experimental.constrained.trunc.f64( + double -1.010000e+01, + metadata !"fpexcept.strict") #0 + ret double %result + ; CHECK-LABEL: @trunc_02 + ; CHECK: ret double -1.000000e+01 +} + +; Verify that round(10.5) is folded to 11.0 when the exception behavior is 'ignore'. +define double @round_01() #0 { +entry: + %result = call double @llvm.experimental.constrained.round.f64( + double 1.050000e+01, + metadata !"fpexcept.ignore") #0 + ret double %result + ; CHECK-LABEL: @round_01 + ; CHECK: ret double 1.100000e+01 +} + +; Verify that floor(-10.5) is folded to -11.0 when the exception behavior is NOT 'ignore'. +define double @round_02() #0 { +entry: + %result = call double @llvm.experimental.constrained.round.f64( + double -1.050000e+01, + metadata !"fpexcept.strict") #0 + ret double %result + ; CHECK-LABEL: @round_02 + ; CHECK: ret double -1.100000e+01 +} + +; Verify that nearbyint(10.5) is folded to 11.0 when the rounding mode is 'upward'. +define double @nearbyint_01() #0 { +entry: + %result = call double @llvm.experimental.constrained.nearbyint.f64( + double 1.050000e+01, + metadata !"round.upward", + metadata !"fpexcept.ignore") #0 + ret double %result + ; CHECK-LABEL: @nearbyint_01 + ; CHECK: ret double 1.100000e+01 +} + +; Verify that nearbyint(10.5) is folded to 10.0 when the rounding mode is 'downward'. +define double @nearbyint_02() #0 { +entry: + %result = call double @llvm.experimental.constrained.nearbyint.f64( + double 1.050000e+01, + metadata !"round.downward", + metadata !"fpexcept.maytrap") #0 + ret double %result + ; CHECK-LABEL: @nearbyint_02 + ; CHECK: ret double 1.000000e+01 +} + +; Verify that nearbyint(10.5) is folded to 10.0 when the rounding mode is 'towardzero'. +define double @nearbyint_03() #0 { +entry: + %result = call double @llvm.experimental.constrained.nearbyint.f64( + double 1.050000e+01, + metadata !"round.towardzero", + metadata !"fpexcept.strict") #0 + ret double %result + ; CHECK-LABEL: @nearbyint_03 + ; CHECK: ret double 1.000000e+01 +} + +; Verify that nearbyint(10.5) is folded to 10.0 when the rounding mode is 'tonearest'. +define double @nearbyint_04() #0 { +entry: + %result = call double @llvm.experimental.constrained.nearbyint.f64( + double 1.050000e+01, + metadata !"round.tonearest", + metadata !"fpexcept.strict") #0 + ret double %result + ; CHECK-LABEL: @nearbyint_04 + ; CHECK: ret double 1.000000e+01 +} + +; Verify that nearbyint(10.5) is NOT folded if the rounding mode is 'dynamic'. +define double @nearbyint_05() #0 { +entry: + %result = call double @llvm.experimental.constrained.nearbyint.f64( + double 1.050000e+01, + metadata !"round.dynamic", + metadata !"fpexcept.strict") #0 + ret double %result + ; CHECK-LABEL: @nearbyint_05 + ; CHECK: [[VAL:%.+]] = {{.*}}call double @llvm.experimental.constrained.nearbyint + ; CHECK: ret double [[VAL]] +} + +; Verify that trunc(SNAN) is NOT folded if the exception behavior mode is not 'ignore'. +define double @nonfinite_01() #0 { +entry: + %result = call double @llvm.experimental.constrained.trunc.f64( + double 0x7ff4000000000000, + metadata !"fpexcept.strict") #0 + ret double %result + ; CHECK-LABEL: @nonfinite_01 + ; CHECK: [[VAL:%.+]] = {{.*}}call double @llvm.experimental.constrained.trunc + ; CHECK: ret double [[VAL]] +} + +; Verify that trunc(SNAN) is folded to QNAN if the exception behavior mode is 'ignore'. +define double @nonfinite_02() #0 { +entry: + %result = call double @llvm.experimental.constrained.trunc.f64( + double 0x7ff4000000000000, + metadata !"fpexcept.ignore") #0 + ret double %result + ; CHECK-LABEL: @nonfinite_02 + ; CHECK: ret double 0x7FF8000000000000 +} + +; Verify that trunc(QNAN) is folded even if the exception behavior mode is not 'ignore'. +define double @nonfinite_03() #0 { +entry: + %result = call double @llvm.experimental.constrained.trunc.f64( + double 0x7ff8000000000000, + metadata !"fpexcept.strict") #0 + ret double %result + ; CHECK-LABEL: @nonfinite_03 + ; CHECK: ret double 0x7FF8000000000000 +} + +; Verify that trunc(+Inf) is folded even if the exception behavior mode is not 'ignore'. +define double @nonfinite_04() #0 { +entry: + %result = call double @llvm.experimental.constrained.trunc.f64( + double 0x7ff0000000000000, + metadata !"fpexcept.strict") #0 + ret double %result + ; CHECK-LABEL: @nonfinite_04 + ; CHECK: ret double 0x7FF0000000000000 +} + + +attributes #0 = { strictfp } + +declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata) +declare double @llvm.experimental.constrained.floor.f64(double, metadata) +declare double @llvm.experimental.constrained.ceil.f64(double, metadata) +declare double @llvm.experimental.constrained.trunc.f64(double, metadata) +declare double @llvm.experimental.constrained.round.f64(double, metadata) +