Index: llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp =================================================================== --- llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp +++ llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp @@ -1121,12 +1121,8 @@ /// Use square root in place of pow(x, +/-0.5). Value *LibCallSimplifier::replacePowWithSqrt(CallInst *Pow, IRBuilder<> &B) { - // TODO: There is some subset of 'fast' under which these transforms should - // be allowed. - if (!Pow->isFast()) - return nullptr; - Value *Sqrt, *Base = Pow->getArgOperand(0), *Expo = Pow->getArgOperand(1); + Module *Mod = Pow->getModule(); Type *Ty = Pow->getType(); const APFloat *ExpoF; @@ -1138,7 +1134,7 @@ if (Pow->hasFnAttr(Attribute::ReadNone)) { Function *SqrtFn = Intrinsic::getDeclaration(Pow->getModule(), Intrinsic::sqrt, Ty); - Sqrt = B.CreateCall(SqrtFn, Base); + Sqrt = B.CreateCall(SqrtFn, Base, "sqrt"); } // Otherwise, use the libcall for sqrt(). else if (hasUnaryFloatFn(TLI, Ty, LibFunc_sqrt, LibFunc_sqrtf, LibFunc_sqrtl)) @@ -1150,6 +1146,21 @@ else return nullptr; + // Handle signed zero base by expanding to fabs(sqrt(x)). + if (!Pow->hasNoSignedZeros()) { + Function *FAbsFn = Intrinsic::getDeclaration(Mod, Intrinsic::fabs, Ty); + Sqrt = B.CreateCall(FAbsFn, Sqrt, "abs"); + } + + // Handle non finite base by expanding to + // (x == -infinity ? +infinity : sqrt(x)). + if (!Pow->hasNoInfs()) { + Value *PosInf = ConstantFP::getInfinity(Ty), + *NegInf = ConstantFP::getInfinity(Ty, true); + Value *FCmp = B.CreateFCmpOEQ(Base, NegInf, "iseq"); + Sqrt = B.CreateSelect(FCmp, PosInf, Sqrt); + } + // If the exponent is negative, then get the reciprocal. if (ExpoF->isNegative()) Sqrt = B.CreateFDiv(ConstantFP::get(Ty, 1.0), Sqrt, "reciprocal"); @@ -1204,7 +1215,7 @@ // We enable these only with fast-math. Besides rounding differences, the // transformation changes overflow and underflow behavior quite dramatically. // Example: x = 1000, y = 0.001. - // pow(exp(x), y) = pow(inf, 0.001) = inf, whereas exp(x*y) = exp(1). + // pow(exp(x), y) = pow(inf, 0.001) = inf, whereas exp(x * y) = exp(1). auto *BaseFn = dyn_cast(Base); if (BaseFn && BaseFn->isFast() && Pow->isFast()) { LibFunc LibFn; @@ -1238,28 +1249,6 @@ if (Value *Sqrt = replacePowWithSqrt(Pow, B)) return Sqrt; - // FIXME: Correct the transforms and pull this into replacePowWithSqrt(). - ConstantFP *ExpoC = dyn_cast(Expo); - if (ExpoC && ExpoC->isExactlyValue(0.5) && - hasUnaryFloatFn(TLI, Ty, LibFunc_sqrt, LibFunc_sqrtf, LibFunc_sqrtl)) { - // Expand pow(x, 0.5) to (x == -infinity ? +infinity : fabs(sqrt(x))). - // This is faster than calling pow(), and still handles -0.0 and - // negative infinity correctly. - // TODO: In finite-only mode, this could be just fabs(sqrt(x)). - Value *PosInf = ConstantFP::getInfinity(Ty); - Value *NegInf = ConstantFP::getInfinity(Ty, true); - - // TODO: As above, we should lower to the sqrt intrinsic if the pow is an - // intrinsic, to match errno semantics. - Value *Sqrt = emitUnaryFloatFnCall(Base, TLI->getName(LibFunc_sqrt), - B, Attrs); - Function *FAbsFn = Intrinsic::getDeclaration(Module, Intrinsic::fabs, Ty); - Value *FAbs = B.CreateCall(FAbsFn, Sqrt, "abs"); - Value *FCmp = B.CreateFCmpOEQ(Base, NegInf); - Sqrt = B.CreateSelect(FCmp, PosInf, FAbs); - return Sqrt; - } - // pow(x, n) -> x * x * x * ... const APFloat *ExpoF; if (Pow->isFast() && match(Expo, m_APFloat(ExpoF))) { Index: llvm/test/Transforms/InstCombine/pow-1.ll =================================================================== --- llvm/test/Transforms/InstCombine/pow-1.ll +++ llvm/test/Transforms/InstCombine/pow-1.ll @@ -177,7 +177,7 @@ define double @test_simplify17(double %x) { ; CHECK-LABEL: @test_simplify17( %retval = call double @llvm.pow.f64(double %x, double 0.5) -; CHECK-NEXT: [[SQRT:%[a-z0-9]+]] = call double @sqrt(double %x) +; CHECK-NEXT: [[SQRT:%[a-z0-9]+]] = call double @llvm.sqrt.f64(double %x) ; CHECK-NEXT: [[FABS:%[a-z0-9]+]] = call double @llvm.fabs.f64(double [[SQRT]]) ; CHECK-NEXT: [[FCMP:%[a-z0-9]+]] = fcmp oeq double %x, 0xFFF0000000000000 ; CHECK-NEXT: [[SELECT:%[a-z0-9]+]] = select i1 [[FCMP]], double 0x7FF0000000000000, double [[FABS]] Index: llvm/test/Transforms/InstCombine/pow-sqrt.ll =================================================================== --- llvm/test/Transforms/InstCombine/pow-sqrt.ll +++ llvm/test/Transforms/InstCombine/pow-sqrt.ll @@ -1,33 +1,34 @@ ; RUN: opt < %s -instcombine -S | FileCheck %s -define double @pow_intrinsic_half_fast(double %x) { -; CHECK-LABEL: @pow_intrinsic_half_fast( -; CHECK-NEXT: [[TMP1:%.*]] = call fast double @llvm.sqrt.f64(double %x) -; CHECK-NEXT: ret double [[TMP1]] +define float @powf_intrinsic_half_fast(float %x) { +; CHECK-LABEL: @powf_intrinsic_half_fast( +; CHECK-NEXT: [[SQRT:%.*]] = call fast float @llvm.sqrt.f32(float %x) +; CHECK-NEXT: ret float [[SQRT]] ; - %pow = call fast double @llvm.pow.f64(double %x, double 5.000000e-01) - ret double %pow + %pow = call fast float @llvm.pow.f32(float %x, float 5.0e-01) + ret float %pow } define <2 x double> @pow_intrinsic_half_approx(<2 x double> %x) { ; CHECK-LABEL: @pow_intrinsic_half_approx( -; CHECK-NEXT: [[POW:%.*]] = call afn <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> ) -; CHECK-NEXT: ret <2 x double> [[POW]] +; CHECK-NEXT: [[SQRT:%.*]] = call afn <2 x double> @llvm.sqrt.v2f64(<2 x double> %x) +; CHECK-NEXT: [[TMP1:%.*]] = call afn <2 x double> @llvm.fabs.v2f64(<2 x double> [[SQRT]]) +; CHECK-NEXT: [[TMP2:%.*]] = fcmp afn oeq <2 x double> %x, +; CHECK-NEXT: [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x double> , <2 x double> [[TMP1]] +; CHECK-NEXT: ret <2 x double> [[TMP3]] ; %pow = call afn <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> ) ret <2 x double> %pow } -define double @pow_libcall_half_approx(double %x) { -; CHECK-LABEL: @pow_libcall_half_approx( -; CHECK-NEXT: [[SQRT:%.*]] = call afn double @sqrt(double %x) -; CHECK-NEXT: [[TMP1:%.*]] = call afn double @llvm.fabs.f64(double [[SQRT]]) -; CHECK-NEXT: [[TMP2:%.*]] = fcmp afn oeq double %x, 0xFFF0000000000000 -; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], double 0x7FF0000000000000, double [[TMP1]] -; CHECK-NEXT: ret double [[TMP3]] +define float @powf_libcall_half_ninf(float %x) { +; CHECK-LABEL: @powf_libcall_half_ninf( +; CHECK-NEXT: [[SQRTF:%.*]] = call ninf float @sqrtf(float %x) +; CHECK-NEXT: [[TMP1:%.*]] = call ninf float @llvm.fabs.f32(float [[SQRTF]]) +; CHECK-NEXT: ret float [[TMP1]] ; - %pow = call afn double @pow(double %x, double 5.0e-01) - ret double %pow + %pow = call ninf float @powf(float %x, float 5.0e-01) + ret float %pow } define <2 x double> @pow_intrinsic_neghalf_fast(<2 x double> %x) { @@ -40,27 +41,45 @@ ret <2 x double> %pow } -define double @pow_intrinsic_neghalf_approx(double %x) { -; CHECK-LABEL: @pow_intrinsic_neghalf_approx( -; CHECK-NEXT: [[POW:%.*]] = call afn double @llvm.pow.f64(double %x, double -5.000000e-01) -; CHECK-NEXT: ret double [[POW]] +define <2 x float> @powf_intrinsic_neghalf_ninf(<2 x float> %x) { +; CHECK-LABEL: @powf_intrinsic_neghalf_ninf( +; CHECK-NEXT: [[SQRT:%.*]] = call ninf <2 x float> @llvm.sqrt.v2f32(<2 x float> %x) +; CHECK-NEXT: [[TMP1:%.*]] = call ninf <2 x float> @llvm.fabs.v2f32(<2 x float> [[SQRT]]) +; CHECK-NEXT: [[TMP2:%.*]] = fdiv ninf <2 x float> , [[TMP1]] +; CHECK-NEXT: ret <2 x float> [[TMP2]] ; - %pow = call afn double @llvm.pow.f64(double %x, double -5.0e-01) - ret double %pow + %pow = call ninf <2 x float> @llvm.pow.v2f32(<2 x float> %x, <2 x float> ) + ret <2 x float> %pow } -define float @pow_libcall_neghalf_fast(float %x) { -; CHECK-LABEL: @pow_libcall_neghalf_fast( -; CHECK-NEXT: [[SQRTF:%.*]] = call fast float @sqrtf(float %x) -; CHECK-NEXT: [[TMP1:%.*]] = fdiv fast float 1.000000e+00, [[SQRTF]] -; CHECK-NEXT: ret float [[TMP1]] +define float @powf_libcall_neghalf_approx(float %x) { +; CHECK-LABEL: @powf_libcall_neghalf_approx( +; CHECK-NEXT: [[SQRTF:%.*]] = call afn float @sqrtf(float %x) +; CHECK-NEXT: [[TMP1:%.*]] = call afn float @llvm.fabs.f32(float [[SQRTF]]) +; CHECK-NEXT: [[TMP2:%.*]] = fcmp afn oeq float %x, 0xFFF0000000000000 +; CHECK-NEXT: [[DOTOP:%.*]] = fdiv afn float 1.000000e+00, [[TMP1]] +; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[TMP2]], float 0.000000e+00, float [[DOTOP]] +; CHECK-NEXT: ret float [[TMP3]] ; - %pow = call fast float @powf(float %x, float -5.0e-01) + %pow = call afn float @powf(float %x, float -5.0e-01) ret float %pow } +define double @pow_libcall_neghalf_fast(double %x) { +; CHECK-LABEL: @pow_libcall_neghalf_fast( +; CHECK-NEXT: [[SQRT:%.*]] = call fast double @sqrt(double %x) +; CHECK-NEXT: [[TMP1:%.*]] = fdiv fast double 1.000000e+00, [[SQRT]] +; CHECK-NEXT: ret double [[TMP1]] +; + %pow = call fast double @pow(double %x, double -5.0e-01) + ret double %pow +} + declare double @llvm.pow.f64(double, double) #0 +declare float @llvm.pow.f32(float, float) #0 declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>) #0 +declare <2 x float> @llvm.pow.v2f32(<2 x float>, <2 x float>) #0 +declare <4 x float> @llvm.pow.v4f32(<4 x float>, <4 x float>) #0 declare double @pow(double, double) declare float @powf(float, float)